473 lines
10 KiB
YAML
473 lines
10 KiB
YAML
# GitLab CI/CD Pipeline for Python
|
|
# Optimized with caching, parallel execution, and deployment
|
|
|
|
stages:
|
|
- security
|
|
- validate
|
|
- test
|
|
- build
|
|
- deploy
|
|
|
|
variables:
|
|
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
|
|
PYTHON_VERSION: "3.11"
|
|
|
|
# Global cache configuration
|
|
cache:
|
|
key:
|
|
files:
|
|
- requirements.txt
|
|
- requirements-dev.txt
|
|
paths:
|
|
- .cache/pip
|
|
- .venv/
|
|
policy: pull
|
|
|
|
# Reusable configuration
|
|
.python_template:
|
|
image: python:${PYTHON_VERSION}
|
|
before_script:
|
|
- python --version
|
|
- pip install --upgrade pip
|
|
- python -m venv .venv
|
|
- source .venv/bin/activate
|
|
- pip install -r requirements.txt
|
|
- pip install -r requirements-dev.txt
|
|
|
|
# Validation stage
|
|
lint:
|
|
extends: .python_template
|
|
stage: validate
|
|
cache:
|
|
key:
|
|
files:
|
|
- requirements.txt
|
|
- requirements-dev.txt
|
|
paths:
|
|
- .cache/pip
|
|
- .venv/
|
|
policy: pull-push
|
|
script:
|
|
- ruff check .
|
|
- black --check .
|
|
- isort --check-only .
|
|
- mypy . || true # Don't fail on type errors initially
|
|
only:
|
|
- merge_requests
|
|
- main
|
|
- develop
|
|
|
|
# Security: Secret Scanning
|
|
secret-scan:trufflehog:
|
|
stage: security
|
|
image: trufflesecurity/trufflehog:latest
|
|
script:
|
|
- trufflehog filesystem . --json --fail > trufflehog-report.json || true
|
|
- |
|
|
if [ -s trufflehog-report.json ]; then
|
|
echo "❌ Secrets detected!"
|
|
cat trufflehog-report.json
|
|
exit 1
|
|
fi
|
|
artifacts:
|
|
when: always
|
|
paths:
|
|
- trufflehog-report.json
|
|
expire_in: 30 days
|
|
allow_failure: false
|
|
only:
|
|
- merge_requests
|
|
- main
|
|
- develop
|
|
|
|
secret-scan:gitleaks:
|
|
stage: security
|
|
image: zricethezav/gitleaks:latest
|
|
script:
|
|
- gitleaks detect --source . --report-format json --report-path gitleaks-report.json
|
|
artifacts:
|
|
when: always
|
|
paths:
|
|
- gitleaks-report.json
|
|
expire_in: 30 days
|
|
allow_failure: true
|
|
only:
|
|
- merge_requests
|
|
- main
|
|
- develop
|
|
|
|
# Security: SAST
|
|
sast:semgrep:
|
|
stage: security
|
|
image: returntocorp/semgrep
|
|
script:
|
|
- semgrep scan --config=auto --sarif --output=semgrep.sarif .
|
|
- semgrep scan --config=p/owasp-top-ten --json --output=semgrep-owasp.json .
|
|
artifacts:
|
|
reports:
|
|
sast: semgrep.sarif
|
|
paths:
|
|
- semgrep.sarif
|
|
- semgrep-owasp.json
|
|
expire_in: 30 days
|
|
allow_failure: false
|
|
only:
|
|
- merge_requests
|
|
- main
|
|
- develop
|
|
|
|
security:bandit:
|
|
extends: .python_template
|
|
stage: security
|
|
script:
|
|
- pip install bandit
|
|
- bandit -r src/ -f json -o bandit-report.json -ll || true
|
|
- bandit -r src/ -ll # Fail on high severity
|
|
artifacts:
|
|
when: always
|
|
paths:
|
|
- bandit-report.json
|
|
expire_in: 30 days
|
|
allow_failure: false
|
|
only:
|
|
- merge_requests
|
|
- main
|
|
- develop
|
|
|
|
security:safety:
|
|
extends: .python_template
|
|
stage: security
|
|
script:
|
|
- pip install safety
|
|
- safety check --json --output safety-report.json || true
|
|
- safety check
|
|
artifacts:
|
|
when: always
|
|
paths:
|
|
- safety-report.json
|
|
expire_in: 30 days
|
|
allow_failure: true
|
|
only:
|
|
- merge_requests
|
|
- main
|
|
- develop
|
|
|
|
# Security: Dependency Scanning
|
|
security:pip-audit:
|
|
extends: .python_template
|
|
stage: security
|
|
script:
|
|
- pip install pip-audit
|
|
- pip-audit --requirement requirements.txt --format json --output pip-audit.json || true
|
|
- pip-audit --requirement requirements.txt # Fail on vulnerabilities
|
|
artifacts:
|
|
when: always
|
|
paths:
|
|
- pip-audit.json
|
|
expire_in: 30 days
|
|
allow_failure: false
|
|
only:
|
|
- merge_requests
|
|
- main
|
|
- develop
|
|
|
|
# Test stage with matrix
|
|
test:
|
|
extends: .python_template
|
|
stage: test
|
|
parallel:
|
|
matrix:
|
|
- PYTHON_VERSION: ["3.9", "3.10", "3.11", "3.12"]
|
|
services:
|
|
- postgres:15
|
|
- redis:7-alpine
|
|
variables:
|
|
POSTGRES_DB: testdb
|
|
POSTGRES_USER: testuser
|
|
POSTGRES_PASSWORD: testpass
|
|
DATABASE_URL: "postgresql://testuser:testpass@postgres:5432/testdb"
|
|
REDIS_URL: "redis://redis:6379"
|
|
script:
|
|
- |
|
|
pytest tests/unit \
|
|
--cov=src \
|
|
--cov-report=xml \
|
|
--cov-report=term \
|
|
--cov-report=html \
|
|
--junitxml=junit.xml \
|
|
-v
|
|
coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
|
|
artifacts:
|
|
when: always
|
|
reports:
|
|
junit: junit.xml
|
|
coverage_report:
|
|
coverage_format: cobertura
|
|
path: coverage.xml
|
|
paths:
|
|
- coverage.xml
|
|
- htmlcov/
|
|
expire_in: 30 days
|
|
|
|
integration-test:
|
|
extends: .python_template
|
|
stage: test
|
|
services:
|
|
- postgres:15
|
|
- redis:7-alpine
|
|
variables:
|
|
POSTGRES_DB: testdb
|
|
POSTGRES_USER: testuser
|
|
POSTGRES_PASSWORD: testpass
|
|
DATABASE_URL: "postgresql://testuser:testpass@postgres:5432/testdb"
|
|
REDIS_URL: "redis://redis:6379"
|
|
script:
|
|
- pytest tests/integration -v --junitxml=junit-integration.xml
|
|
artifacts:
|
|
when: always
|
|
reports:
|
|
junit: junit-integration.xml
|
|
expire_in: 7 days
|
|
only:
|
|
- main
|
|
- develop
|
|
- merge_requests
|
|
|
|
# Build stage
|
|
build:package:
|
|
extends: .python_template
|
|
stage: build
|
|
script:
|
|
- pip install build wheel setuptools
|
|
- python -m build
|
|
- ls -lh dist/
|
|
artifacts:
|
|
paths:
|
|
- dist/
|
|
expire_in: 7 days
|
|
only:
|
|
- main
|
|
- develop
|
|
- tags
|
|
|
|
build:docker:
|
|
stage: build
|
|
image: docker:24-cli
|
|
services:
|
|
- docker:24-dind
|
|
variables:
|
|
IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
|
|
before_script:
|
|
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
|
|
script:
|
|
# Pull previous image for caching
|
|
- docker pull $CI_REGISTRY_IMAGE:latest || true
|
|
|
|
# Build with cache
|
|
- |
|
|
docker build \
|
|
--cache-from $CI_REGISTRY_IMAGE:latest \
|
|
--tag $IMAGE_TAG \
|
|
--tag $CI_REGISTRY_IMAGE:latest \
|
|
--build-arg BUILDKIT_INLINE_CACHE=1 \
|
|
.
|
|
|
|
# Push images
|
|
- docker push $IMAGE_TAG
|
|
- docker push $CI_REGISTRY_IMAGE:latest
|
|
|
|
- echo "IMAGE_FULL_NAME=$IMAGE_TAG" >> build.env
|
|
artifacts:
|
|
reports:
|
|
dotenv: build.env
|
|
only:
|
|
- main
|
|
- develop
|
|
- tags
|
|
tags:
|
|
- docker
|
|
|
|
# E2E tests (only on main)
|
|
e2e-test:
|
|
extends: .python_template
|
|
stage: test
|
|
needs: [build:package]
|
|
dependencies:
|
|
- build:package
|
|
script:
|
|
- pip install dist/*.whl
|
|
- pytest tests/e2e -v
|
|
artifacts:
|
|
when: always
|
|
paths:
|
|
- test-results/
|
|
expire_in: 7 days
|
|
only:
|
|
- main
|
|
|
|
# Deploy to PyPI
|
|
deploy:pypi:
|
|
stage: deploy
|
|
image: python:3.11
|
|
needs: [build:package, test]
|
|
dependencies:
|
|
- build:package
|
|
environment:
|
|
name: pypi
|
|
url: https://pypi.org/project/your-package
|
|
before_script:
|
|
- pip install twine
|
|
script:
|
|
- twine check dist/*
|
|
- twine upload dist/* --username __token__ --password $PYPI_TOKEN
|
|
only:
|
|
- tags
|
|
when: manual
|
|
|
|
# Deploy Docker to staging
|
|
deploy:staging:
|
|
stage: deploy
|
|
image: bitnami/kubectl:latest
|
|
needs: [build:docker]
|
|
environment:
|
|
name: staging
|
|
url: https://staging.example.com
|
|
on_stop: stop:staging
|
|
before_script:
|
|
- kubectl config use-context staging-cluster
|
|
script:
|
|
- kubectl set image deployment/myapp myapp=$IMAGE_FULL_NAME --namespace=staging --record
|
|
- kubectl rollout status deployment/myapp --namespace=staging --timeout=5m
|
|
# Smoke test
|
|
- |
|
|
POD=$(kubectl get pod -n staging -l app=myapp -o jsonpath="{.items[0].metadata.name}")
|
|
kubectl exec -n staging $POD -- python -c "import sys; print(sys.version)"
|
|
kubectl exec -n staging $POD -- curl -f http://localhost:8000/health || exit 1
|
|
only:
|
|
- develop
|
|
when: manual
|
|
|
|
stop:staging:
|
|
stage: deploy
|
|
image: bitnami/kubectl:latest
|
|
environment:
|
|
name: staging
|
|
action: stop
|
|
script:
|
|
- kubectl scale deployment/myapp --replicas=0 --namespace=staging
|
|
when: manual
|
|
only:
|
|
- develop
|
|
|
|
# Deploy to production
|
|
deploy:production:
|
|
stage: deploy
|
|
image: bitnami/kubectl:latest
|
|
needs: [build:docker, e2e-test]
|
|
environment:
|
|
name: production
|
|
url: https://example.com
|
|
before_script:
|
|
- kubectl config use-context production-cluster
|
|
script:
|
|
- echo "Deploying to production..."
|
|
- kubectl set image deployment/myapp myapp=$IMAGE_FULL_NAME --namespace=production --record
|
|
- kubectl rollout status deployment/myapp --namespace=production --timeout=5m
|
|
|
|
# Health check
|
|
- sleep 10
|
|
- |
|
|
for i in {1..10}; do
|
|
POD=$(kubectl get pod -n production -l app=myapp -o jsonpath="{.items[0].metadata.name}")
|
|
if kubectl exec -n production $POD -- curl -f http://localhost:8000/health; then
|
|
echo "Health check passed"
|
|
exit 0
|
|
fi
|
|
echo "Attempt $i failed, retrying..."
|
|
sleep 10
|
|
done
|
|
echo "Health check failed"
|
|
exit 1
|
|
only:
|
|
- main
|
|
when: manual
|
|
|
|
# Deploy to Cloud Run (Google Cloud)
|
|
deploy:cloudrun:
|
|
stage: deploy
|
|
image: google/cloud-sdk:alpine
|
|
needs: [build:docker]
|
|
environment:
|
|
name: production
|
|
url: https://your-app.run.app
|
|
before_script:
|
|
- echo $GCP_SERVICE_KEY | base64 -d > ${HOME}/gcp-key.json
|
|
- gcloud auth activate-service-account --key-file ${HOME}/gcp-key.json
|
|
- gcloud config set project $GCP_PROJECT_ID
|
|
script:
|
|
- |
|
|
gcloud run deploy your-app \
|
|
--image $IMAGE_FULL_NAME \
|
|
--region us-central1 \
|
|
--platform managed \
|
|
--allow-unauthenticated
|
|
|
|
# Health check
|
|
- |
|
|
URL=$(gcloud run services describe your-app --region us-central1 --format 'value(status.url)')
|
|
curl -f $URL/health || exit 1
|
|
only:
|
|
- main
|
|
when: manual
|
|
|
|
# Create release
|
|
release:
|
|
stage: deploy
|
|
image: registry.gitlab.com/gitlab-org/release-cli:latest
|
|
needs: [deploy:production]
|
|
script:
|
|
- echo "Creating release"
|
|
release:
|
|
tag_name: $CI_COMMIT_TAG
|
|
description: |
|
|
Python Package: https://pypi.org/project/your-package/$CI_COMMIT_TAG
|
|
Docker Image: $IMAGE_FULL_NAME
|
|
|
|
Changes in this release:
|
|
$CI_COMMIT_MESSAGE
|
|
only:
|
|
- tags
|
|
|
|
# GitLab built-in security templates
|
|
include:
|
|
- template: Security/Dependency-Scanning.gitlab-ci.yml
|
|
- template: Security/SAST.gitlab-ci.yml
|
|
|
|
# Override GitLab template stages
|
|
dependency_scanning:
|
|
stage: security
|
|
|
|
sast:
|
|
stage: security
|
|
|
|
# Workflow rules
|
|
workflow:
|
|
rules:
|
|
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
|
- if: '$CI_COMMIT_BRANCH == "main"'
|
|
- if: '$CI_COMMIT_BRANCH == "develop"'
|
|
- if: '$CI_COMMIT_TAG'
|
|
|
|
# Interruptible jobs
|
|
.interruptible_template:
|
|
interruptible: true
|
|
|
|
lint:
|
|
extends: [.python_template, .interruptible_template]
|
|
|
|
test:
|
|
extends: [.python_template, .interruptible_template]
|
|
|
|
integration-test:
|
|
extends: [.python_template, .interruptible_template]
|