Initial commit
This commit is contained in:
Zhongwei Li
472
assets/templates/gitlab-ci/python-ci.yml
Normal file
472
assets/templates/gitlab-ci/python-ci.yml
Normal file
@@ -0,0 +1,472 @@
|
||||
# GitLab CI/CD Pipeline for Python
|
||||
# Optimized with caching, parallel execution, and deployment
|
||||
|
||||
stages:
|
||||
- security
|
||||
- validate
|
||||
- test
|
||||
- build
|
||||
- deploy
|
||||
|
||||
variables:
|
||||
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
|
||||
PYTHON_VERSION: "3.11"
|
||||
|
||||
# Global cache configuration
|
||||
cache:
|
||||
key:
|
||||
files:
|
||||
- requirements.txt
|
||||
- requirements-dev.txt
|
||||
paths:
|
||||
- .cache/pip
|
||||
- .venv/
|
||||
policy: pull
|
||||
|
||||
# Reusable configuration
|
||||
.python_template:
|
||||
image: python:${PYTHON_VERSION}
|
||||
before_script:
|
||||
- python --version
|
||||
- pip install --upgrade pip
|
||||
- python -m venv .venv
|
||||
- source .venv/bin/activate
|
||||
- pip install -r requirements.txt
|
||||
- pip install -r requirements-dev.txt
|
||||
|
||||
# Validation stage
|
||||
lint:
|
||||
extends: .python_template
|
||||
stage: validate
|
||||
cache:
|
||||
key:
|
||||
files:
|
||||
- requirements.txt
|
||||
- requirements-dev.txt
|
||||
paths:
|
||||
- .cache/pip
|
||||
- .venv/
|
||||
policy: pull-push
|
||||
script:
|
||||
- ruff check .
|
||||
- black --check .
|
||||
- isort --check-only .
|
||||
- mypy . || true # Don't fail on type errors initially
|
||||
only:
|
||||
- merge_requests
|
||||
- main
|
||||
- develop
|
||||
|
||||
# Security: Secret Scanning
|
||||
secret-scan:trufflehog:
|
||||
stage: security
|
||||
image: trufflesecurity/trufflehog:latest
|
||||
script:
|
||||
- trufflehog filesystem . --json --fail > trufflehog-report.json || true
|
||||
- |
|
||||
if [ -s trufflehog-report.json ]; then
|
||||
echo "❌ Secrets detected!"
|
||||
cat trufflehog-report.json
|
||||
exit 1
|
||||
fi
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- trufflehog-report.json
|
||||
expire_in: 30 days
|
||||
allow_failure: false
|
||||
only:
|
||||
- merge_requests
|
||||
- main
|
||||
- develop
|
||||
|
||||
secret-scan:gitleaks:
|
||||
stage: security
|
||||
image: zricethezav/gitleaks:latest
|
||||
script:
|
||||
- gitleaks detect --source . --report-format json --report-path gitleaks-report.json
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- gitleaks-report.json
|
||||
expire_in: 30 days
|
||||
allow_failure: true
|
||||
only:
|
||||
- merge_requests
|
||||
- main
|
||||
- develop
|
||||
|
||||
# Security: SAST
|
||||
sast:semgrep:
|
||||
stage: security
|
||||
image: returntocorp/semgrep
|
||||
script:
|
||||
- semgrep scan --config=auto --sarif --output=semgrep.sarif .
|
||||
- semgrep scan --config=p/owasp-top-ten --json --output=semgrep-owasp.json .
|
||||
artifacts:
|
||||
reports:
|
||||
sast: semgrep.sarif
|
||||
paths:
|
||||
- semgrep.sarif
|
||||
- semgrep-owasp.json
|
||||
expire_in: 30 days
|
||||
allow_failure: false
|
||||
only:
|
||||
- merge_requests
|
||||
- main
|
||||
- develop
|
||||
|
||||
security:bandit:
|
||||
extends: .python_template
|
||||
stage: security
|
||||
script:
|
||||
- pip install bandit
|
||||
- bandit -r src/ -f json -o bandit-report.json -ll || true
|
||||
- bandit -r src/ -ll # Fail on high severity
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- bandit-report.json
|
||||
expire_in: 30 days
|
||||
allow_failure: false
|
||||
only:
|
||||
- merge_requests
|
||||
- main
|
||||
- develop
|
||||
|
||||
security:safety:
|
||||
extends: .python_template
|
||||
stage: security
|
||||
script:
|
||||
- pip install safety
|
||||
- safety check --json --output safety-report.json || true
|
||||
- safety check
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- safety-report.json
|
||||
expire_in: 30 days
|
||||
allow_failure: true
|
||||
only:
|
||||
- merge_requests
|
||||
- main
|
||||
- develop
|
||||
|
||||
# Security: Dependency Scanning
|
||||
security:pip-audit:
|
||||
extends: .python_template
|
||||
stage: security
|
||||
script:
|
||||
- pip install pip-audit
|
||||
- pip-audit --requirement requirements.txt --format json --output pip-audit.json || true
|
||||
- pip-audit --requirement requirements.txt # Fail on vulnerabilities
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- pip-audit.json
|
||||
expire_in: 30 days
|
||||
allow_failure: false
|
||||
only:
|
||||
- merge_requests
|
||||
- main
|
||||
- develop
|
||||
|
||||
# Test stage with matrix
|
||||
test:
|
||||
extends: .python_template
|
||||
stage: test
|
||||
parallel:
|
||||
matrix:
|
||||
- PYTHON_VERSION: ["3.9", "3.10", "3.11", "3.12"]
|
||||
services:
|
||||
- postgres:15
|
||||
- redis:7-alpine
|
||||
variables:
|
||||
POSTGRES_DB: testdb
|
||||
POSTGRES_USER: testuser
|
||||
POSTGRES_PASSWORD: testpass
|
||||
DATABASE_URL: "postgresql://testuser:testpass@postgres:5432/testdb"
|
||||
REDIS_URL: "redis://redis:6379"
|
||||
script:
|
||||
- |
|
||||
pytest tests/unit \
|
||||
--cov=src \
|
||||
--cov-report=xml \
|
||||
--cov-report=term \
|
||||
--cov-report=html \
|
||||
--junitxml=junit.xml \
|
||||
-v
|
||||
coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
|
||||
artifacts:
|
||||
when: always
|
||||
reports:
|
||||
junit: junit.xml
|
||||
coverage_report:
|
||||
coverage_format: cobertura
|
||||
path: coverage.xml
|
||||
paths:
|
||||
- coverage.xml
|
||||
- htmlcov/
|
||||
expire_in: 30 days
|
||||
|
||||
integration-test:
|
||||
extends: .python_template
|
||||
stage: test
|
||||
services:
|
||||
- postgres:15
|
||||
- redis:7-alpine
|
||||
variables:
|
||||
POSTGRES_DB: testdb
|
||||
POSTGRES_USER: testuser
|
||||
POSTGRES_PASSWORD: testpass
|
||||
DATABASE_URL: "postgresql://testuser:testpass@postgres:5432/testdb"
|
||||
REDIS_URL: "redis://redis:6379"
|
||||
script:
|
||||
- pytest tests/integration -v --junitxml=junit-integration.xml
|
||||
artifacts:
|
||||
when: always
|
||||
reports:
|
||||
junit: junit-integration.xml
|
||||
expire_in: 7 days
|
||||
only:
|
||||
- main
|
||||
- develop
|
||||
- merge_requests
|
||||
|
||||
# Build stage
|
||||
build:package:
|
||||
extends: .python_template
|
||||
stage: build
|
||||
script:
|
||||
- pip install build wheel setuptools
|
||||
- python -m build
|
||||
- ls -lh dist/
|
||||
artifacts:
|
||||
paths:
|
||||
- dist/
|
||||
expire_in: 7 days
|
||||
only:
|
||||
- main
|
||||
- develop
|
||||
- tags
|
||||
|
||||
build:docker:
|
||||
stage: build
|
||||
image: docker:24-cli
|
||||
services:
|
||||
- docker:24-dind
|
||||
variables:
|
||||
IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
|
||||
before_script:
|
||||
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
|
||||
script:
|
||||
# Pull previous image for caching
|
||||
- docker pull $CI_REGISTRY_IMAGE:latest || true
|
||||
|
||||
# Build with cache
|
||||
- |
|
||||
docker build \
|
||||
--cache-from $CI_REGISTRY_IMAGE:latest \
|
||||
--tag $IMAGE_TAG \
|
||||
--tag $CI_REGISTRY_IMAGE:latest \
|
||||
--build-arg BUILDKIT_INLINE_CACHE=1 \
|
||||
.
|
||||
|
||||
# Push images
|
||||
- docker push $IMAGE_TAG
|
||||
- docker push $CI_REGISTRY_IMAGE:latest
|
||||
|
||||
- echo "IMAGE_FULL_NAME=$IMAGE_TAG" >> build.env
|
||||
artifacts:
|
||||
reports:
|
||||
dotenv: build.env
|
||||
only:
|
||||
- main
|
||||
- develop
|
||||
- tags
|
||||
tags:
|
||||
- docker
|
||||
|
||||
# E2E tests (only on main)
|
||||
e2e-test:
|
||||
extends: .python_template
|
||||
stage: test
|
||||
needs: [build:package]
|
||||
dependencies:
|
||||
- build:package
|
||||
script:
|
||||
- pip install dist/*.whl
|
||||
- pytest tests/e2e -v
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- test-results/
|
||||
expire_in: 7 days
|
||||
only:
|
||||
- main
|
||||
|
||||
# Deploy to PyPI
|
||||
deploy:pypi:
|
||||
stage: deploy
|
||||
image: python:3.11
|
||||
needs: [build:package, test]
|
||||
dependencies:
|
||||
- build:package
|
||||
environment:
|
||||
name: pypi
|
||||
url: https://pypi.org/project/your-package
|
||||
before_script:
|
||||
- pip install twine
|
||||
script:
|
||||
- twine check dist/*
|
||||
- twine upload dist/* --username __token__ --password $PYPI_TOKEN
|
||||
only:
|
||||
- tags
|
||||
when: manual
|
||||
|
||||
# Deploy Docker to staging
|
||||
deploy:staging:
|
||||
stage: deploy
|
||||
image: bitnami/kubectl:latest
|
||||
needs: [build:docker]
|
||||
environment:
|
||||
name: staging
|
||||
url: https://staging.example.com
|
||||
on_stop: stop:staging
|
||||
before_script:
|
||||
- kubectl config use-context staging-cluster
|
||||
script:
|
||||
- kubectl set image deployment/myapp myapp=$IMAGE_FULL_NAME --namespace=staging --record
|
||||
- kubectl rollout status deployment/myapp --namespace=staging --timeout=5m
|
||||
# Smoke test
|
||||
- |
|
||||
POD=$(kubectl get pod -n staging -l app=myapp -o jsonpath="{.items[0].metadata.name}")
|
||||
kubectl exec -n staging $POD -- python -c "import sys; print(sys.version)"
|
||||
kubectl exec -n staging $POD -- curl -f http://localhost:8000/health || exit 1
|
||||
only:
|
||||
- develop
|
||||
when: manual
|
||||
|
||||
stop:staging:
|
||||
stage: deploy
|
||||
image: bitnami/kubectl:latest
|
||||
environment:
|
||||
name: staging
|
||||
action: stop
|
||||
script:
|
||||
- kubectl scale deployment/myapp --replicas=0 --namespace=staging
|
||||
when: manual
|
||||
only:
|
||||
- develop
|
||||
|
||||
# Deploy to production
|
||||
deploy:production:
|
||||
stage: deploy
|
||||
image: bitnami/kubectl:latest
|
||||
needs: [build:docker, e2e-test]
|
||||
environment:
|
||||
name: production
|
||||
url: https://example.com
|
||||
before_script:
|
||||
- kubectl config use-context production-cluster
|
||||
script:
|
||||
- echo "Deploying to production..."
|
||||
- kubectl set image deployment/myapp myapp=$IMAGE_FULL_NAME --namespace=production --record
|
||||
- kubectl rollout status deployment/myapp --namespace=production --timeout=5m
|
||||
|
||||
# Health check
|
||||
- sleep 10
|
||||
- |
|
||||
for i in {1..10}; do
|
||||
POD=$(kubectl get pod -n production -l app=myapp -o jsonpath="{.items[0].metadata.name}")
|
||||
if kubectl exec -n production $POD -- curl -f http://localhost:8000/health; then
|
||||
echo "Health check passed"
|
||||
exit 0
|
||||
fi
|
||||
echo "Attempt $i failed, retrying..."
|
||||
sleep 10
|
||||
done
|
||||
echo "Health check failed"
|
||||
exit 1
|
||||
only:
|
||||
- main
|
||||
when: manual
|
||||
|
||||
# Deploy to Cloud Run (Google Cloud)
|
||||
deploy:cloudrun:
|
||||
stage: deploy
|
||||
image: google/cloud-sdk:alpine
|
||||
needs: [build:docker]
|
||||
environment:
|
||||
name: production
|
||||
url: https://your-app.run.app
|
||||
before_script:
|
||||
- echo $GCP_SERVICE_KEY | base64 -d > ${HOME}/gcp-key.json
|
||||
- gcloud auth activate-service-account --key-file ${HOME}/gcp-key.json
|
||||
- gcloud config set project $GCP_PROJECT_ID
|
||||
script:
|
||||
- |
|
||||
gcloud run deploy your-app \
|
||||
--image $IMAGE_FULL_NAME \
|
||||
--region us-central1 \
|
||||
--platform managed \
|
||||
--allow-unauthenticated
|
||||
|
||||
# Health check
|
||||
- |
|
||||
URL=$(gcloud run services describe your-app --region us-central1 --format 'value(status.url)')
|
||||
curl -f $URL/health || exit 1
|
||||
only:
|
||||
- main
|
||||
when: manual
|
||||
|
||||
# Create release
|
||||
release:
|
||||
stage: deploy
|
||||
image: registry.gitlab.com/gitlab-org/release-cli:latest
|
||||
needs: [deploy:production]
|
||||
script:
|
||||
- echo "Creating release"
|
||||
release:
|
||||
tag_name: $CI_COMMIT_TAG
|
||||
description: |
|
||||
Python Package: https://pypi.org/project/your-package/$CI_COMMIT_TAG
|
||||
Docker Image: $IMAGE_FULL_NAME
|
||||
|
||||
Changes in this release:
|
||||
$CI_COMMIT_MESSAGE
|
||||
only:
|
||||
- tags
|
||||
|
||||
# GitLab built-in security templates
|
||||
include:
|
||||
- template: Security/Dependency-Scanning.gitlab-ci.yml
|
||||
- template: Security/SAST.gitlab-ci.yml
|
||||
|
||||
# Override GitLab template stages
|
||||
dependency_scanning:
|
||||
stage: security
|
||||
|
||||
sast:
|
||||
stage: security
|
||||
|
||||
# Workflow rules
|
||||
workflow:
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
||||
- if: '$CI_COMMIT_BRANCH == "main"'
|
||||
- if: '$CI_COMMIT_BRANCH == "develop"'
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
|
||||
# Interruptible jobs
|
||||
.interruptible_template:
|
||||
interruptible: true
|
||||
|
||||
lint:
|
||||
extends: [.python_template, .interruptible_template]
|
||||
|
||||
test:
|
||||
extends: [.python_template, .interruptible_template]
|
||||
|
||||
integration-test:
|
||||
extends: [.python_template, .interruptible_template]
|
||||
Reference in New Issue
Block a user