389 lines
9.4 KiB
YAML
389 lines
9.4 KiB
YAML
# Python CI/CD Pipeline
|
|
# Optimized with caching, matrix testing, and deployment
|
|
|
|
name: Python CI
|
|
|
|
on:
|
|
push:
|
|
branches: [main, develop]
|
|
paths-ignore:
|
|
- '**.md'
|
|
- 'docs/**'
|
|
pull_request:
|
|
branches: [main]
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
# Security: Secret Scanning
|
|
secret-scan:
|
|
name: Secret Scanning
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: TruffleHog Secret Scan
|
|
uses: trufflesecurity/trufflehog@main
|
|
with:
|
|
path: ./
|
|
base: ${{ github.event.repository.default_branch }}
|
|
head: HEAD
|
|
|
|
- name: Gitleaks
|
|
uses: gitleaks/gitleaks-action@v2
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
# Security: SAST
|
|
sast:
|
|
name: Static Analysis (CodeQL)
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
|
|
permissions:
|
|
contents: read
|
|
security-events: write
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.11'
|
|
|
|
- name: Initialize CodeQL
|
|
uses: github/codeql-action/init@v3
|
|
with:
|
|
languages: python
|
|
queries: security-and-quality
|
|
|
|
- name: Perform CodeQL Analysis
|
|
uses: github/codeql-action/analyze@v3
|
|
|
|
lint:
|
|
name: Lint & Format Check
|
|
runs-on: ubuntu-latest
|
|
needs: [secret-scan]
|
|
timeout-minutes: 10
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.11'
|
|
cache: 'pip'
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install ruff black mypy isort
|
|
|
|
- name: Run ruff
|
|
run: ruff check .
|
|
|
|
- name: Check formatting with black
|
|
run: black --check .
|
|
|
|
- name: Check import sorting
|
|
run: isort --check-only .
|
|
|
|
- name: Type check with mypy
|
|
run: mypy .
|
|
continue-on-error: true # Don't fail on type errors initially
|
|
|
|
test:
|
|
name: Test (Python ${{ matrix.python-version }})
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 20
|
|
|
|
strategy:
|
|
matrix:
|
|
python-version: ['3.9', '3.10', '3.11', '3.12']
|
|
fail-fast: false
|
|
|
|
services:
|
|
postgres:
|
|
image: postgres:15
|
|
env:
|
|
POSTGRES_PASSWORD: postgres
|
|
POSTGRES_DB: testdb
|
|
options: >-
|
|
--health-cmd pg_isready
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
ports:
|
|
- 5432:5432
|
|
|
|
redis:
|
|
image: redis:7-alpine
|
|
options: >-
|
|
--health-cmd "redis-cli ping"
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
ports:
|
|
- 6379:6379
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
cache: 'pip'
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install -r requirements.txt
|
|
pip install -r requirements-dev.txt
|
|
|
|
- name: Run unit tests
|
|
env:
|
|
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/testdb
|
|
REDIS_URL: redis://localhost:6379
|
|
run: |
|
|
pytest tests/unit \
|
|
--cov=src \
|
|
--cov-report=xml \
|
|
--cov-report=term \
|
|
--junitxml=junit.xml \
|
|
-v
|
|
|
|
- name: Run integration tests
|
|
if: matrix.python-version == '3.11'
|
|
env:
|
|
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/testdb
|
|
REDIS_URL: redis://localhost:6379
|
|
run: |
|
|
pytest tests/integration -v
|
|
|
|
- name: Upload coverage to Codecov
|
|
if: matrix.python-version == '3.11'
|
|
uses: codecov/codecov-action@v4
|
|
with:
|
|
files: ./coverage.xml
|
|
fail_ci_if_error: false
|
|
|
|
- name: Upload test results
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: test-results-${{ matrix.python-version }}
|
|
path: junit.xml
|
|
|
|
security:
|
|
name: Security Scanning
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.11'
|
|
cache: 'pip'
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install -r requirements.txt
|
|
|
|
- name: Run bandit security scan
|
|
run: |
|
|
pip install bandit
|
|
bandit -r src/ -f json -o bandit-report.json -ll || true
|
|
bandit -r src/ -ll
|
|
continue-on-error: false
|
|
|
|
- name: Run safety check
|
|
run: |
|
|
pip install safety
|
|
safety check --json --output safety-report.json || true
|
|
safety check
|
|
continue-on-error: true
|
|
|
|
- name: pip-audit dependency scan
|
|
run: |
|
|
pip install pip-audit
|
|
pip-audit --requirement requirements.txt --format json --output pip-audit.json || true
|
|
pip-audit --requirement requirements.txt
|
|
continue-on-error: false
|
|
|
|
- name: Upload security reports
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: security-reports
|
|
path: |
|
|
bandit-report.json
|
|
safety-report.json
|
|
pip-audit.json
|
|
|
|
build:
|
|
name: Build Package
|
|
runs-on: ubuntu-latest
|
|
needs: [lint, test, sast, security]
|
|
timeout-minutes: 10
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.11'
|
|
cache: 'pip'
|
|
|
|
- name: Install build tools
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install build wheel setuptools
|
|
|
|
- name: Build package
|
|
run: python -m build
|
|
|
|
- name: Upload distribution
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: dist-${{ github.sha }}
|
|
path: dist/
|
|
retention-days: 7
|
|
|
|
e2e:
|
|
name: E2E Tests
|
|
runs-on: ubuntu-latest
|
|
needs: build
|
|
if: github.ref == 'refs/heads/main'
|
|
timeout-minutes: 30
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.11'
|
|
cache: 'pip'
|
|
|
|
- name: Download build artifacts
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: dist-${{ github.sha }}
|
|
path: dist/
|
|
|
|
- name: Install package
|
|
run: |
|
|
pip install dist/*.whl
|
|
pip install -r requirements-dev.txt
|
|
|
|
- name: Run E2E tests
|
|
run: pytest tests/e2e -v
|
|
|
|
deploy-pypi:
|
|
name: Deploy to PyPI
|
|
runs-on: ubuntu-latest
|
|
needs: [build, test]
|
|
if: startsWith(github.ref, 'refs/tags/v')
|
|
environment:
|
|
name: pypi
|
|
url: https://pypi.org/project/your-package
|
|
|
|
permissions:
|
|
id-token: write # For trusted publishing
|
|
|
|
steps:
|
|
- uses: actions/download-artifact@v4
|
|
with:
|
|
name: dist-${{ github.sha }}
|
|
path: dist/
|
|
|
|
- name: Publish to PyPI
|
|
uses: pypa/gh-action-pypi-publish@release/v1
|
|
# Uses OIDC trusted publishing - no token needed!
|
|
|
|
deploy-docker:
|
|
name: Build & Push Docker Image
|
|
runs-on: ubuntu-latest
|
|
needs: [build, test]
|
|
if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
|
|
|
|
permissions:
|
|
contents: read
|
|
packages: write
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
|
|
- name: Log in to GitHub Container Registry
|
|
uses: docker/login-action@v3
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Extract metadata
|
|
id: meta
|
|
uses: docker/metadata-action@v5
|
|
with:
|
|
images: ghcr.io/${{ github.repository }}
|
|
tags: |
|
|
type=ref,event=branch
|
|
type=semver,pattern={{version}}
|
|
type=semver,pattern={{major}}.{{minor}}
|
|
type=sha
|
|
|
|
- name: Build and push
|
|
uses: docker/build-push-action@v5
|
|
with:
|
|
context: .
|
|
push: true
|
|
tags: ${{ steps.meta.outputs.tags }}
|
|
labels: ${{ steps.meta.outputs.labels }}
|
|
cache-from: type=gha
|
|
cache-to: type=gha,mode=max
|
|
|
|
deploy-cloud:
|
|
name: Deploy to Cloud Run
|
|
runs-on: ubuntu-latest
|
|
needs: deploy-docker
|
|
if: github.ref == 'refs/heads/main'
|
|
environment:
|
|
name: production
|
|
url: https://your-app.run.app
|
|
|
|
permissions:
|
|
contents: read
|
|
id-token: write
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: google-github-actions/auth@v2
|
|
with:
|
|
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
|
|
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
|
|
|
|
- name: Deploy to Cloud Run
|
|
run: |
|
|
gcloud run deploy your-app \
|
|
--image ghcr.io/${{ github.repository }}:${{ github.sha }} \
|
|
--region us-central1 \
|
|
--platform managed \
|
|
--allow-unauthenticated
|
|
|
|
- name: Health check
|
|
run: |
|
|
URL=$(gcloud run services describe your-app --region us-central1 --format 'value(status.url)')
|
|
curl -f $URL/health || exit 1
|