gh-ahmedasmar-devops-claude-skills-ci-cd/assets/templates/gitlab-ci/python-ci.yml

# GitLab CI/CD Pipeline for Python
# Optimized with caching, parallel execution, and deployment

stages:
  - security
  - validate
  - test
  - build
  - deploy

variables:
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
  PYTHON_VERSION: "3.11"

# Global cache configuration
cache:
  key:
    files:
      - requirements.txt
      - requirements-dev.txt
  paths:
    - .cache/pip
    - .venv/
  policy: pull

# Reusable configuration
.python_template:
  image: python:${PYTHON_VERSION}
  before_script:
    - python --version
    - pip install --upgrade pip
    - python -m venv .venv
    - source .venv/bin/activate
    - pip install -r requirements.txt
    - pip install -r requirements-dev.txt

# Validation stage
lint:
  extends: .python_template
  stage: validate
  cache:
    key:
      files:
        - requirements.txt
        - requirements-dev.txt
    paths:
      - .cache/pip
      - .venv/
    policy: pull-push
  script:
    - ruff check .
    - black --check .
    - isort --check-only .
    - mypy . || true  # Don't fail on type errors initially
  only:
    - merge_requests
    - main
    - develop

# Security: Secret Scanning
secret-scan:trufflehog:
  stage: security
  image: trufflesecurity/trufflehog:latest
  script:
    - trufflehog filesystem . --json --fail > trufflehog-report.json || true
    - |
      if [ -s trufflehog-report.json ]; then
        echo "❌ Secrets detected!"
        cat trufflehog-report.json
        exit 1
      fi
  artifacts:
    when: always
    paths:
      - trufflehog-report.json
    expire_in: 30 days
  allow_failure: false
  only:
    - merge_requests
    - main
    - develop

secret-scan:gitleaks:
  stage: security
  image: zricethezav/gitleaks:latest
  script:
    - gitleaks detect --source . --report-format json --report-path gitleaks-report.json
  artifacts:
    when: always
    paths:
      - gitleaks-report.json
    expire_in: 30 days
  allow_failure: true
  only:
    - merge_requests
    - main
    - develop

# Security: SAST
sast:semgrep:
  stage: security
  image: returntocorp/semgrep
  script:
    - semgrep scan --config=auto --sarif --output=semgrep.sarif .
    - semgrep scan --config=p/owasp-top-ten --json --output=semgrep-owasp.json .
  artifacts:
    reports:
      sast: semgrep.sarif
    paths:
      - semgrep.sarif
      - semgrep-owasp.json
    expire_in: 30 days
  allow_failure: false
  only:
    - merge_requests
    - main
    - develop

security:bandit:
  extends: .python_template
  stage: security
  script:
    - pip install bandit
    - bandit -r src/ -f json -o bandit-report.json -ll || true
    - bandit -r src/ -ll  # Fail on high severity
  artifacts:
    when: always
    paths:
      - bandit-report.json
    expire_in: 30 days
  allow_failure: false
  only:
    - merge_requests
    - main
    - develop

security:safety:
  extends: .python_template
  stage: security
  script:
    - pip install safety
    - safety check --json --output safety-report.json || true
    - safety check
  artifacts:
    when: always
    paths:
      - safety-report.json
    expire_in: 30 days
  allow_failure: true
  only:
    - merge_requests
    - main
    - develop

# Security: Dependency Scanning
security:pip-audit:
  extends: .python_template
  stage: security
  script:
    - pip install pip-audit
    - pip-audit --requirement requirements.txt --format json --output pip-audit.json || true
    - pip-audit --requirement requirements.txt  # Fail on vulnerabilities
  artifacts:
    when: always
    paths:
      - pip-audit.json
    expire_in: 30 days
  allow_failure: false
  only:
    - merge_requests
    - main
    - develop

# Test stage with matrix
test:
  extends: .python_template
  stage: test
  parallel:
    matrix:
      - PYTHON_VERSION: ["3.9", "3.10", "3.11", "3.12"]
  services:
    - postgres:15
    - redis:7-alpine
  variables:
    POSTGRES_DB: testdb
    POSTGRES_USER: testuser
    POSTGRES_PASSWORD: testpass
    DATABASE_URL: "postgresql://testuser:testpass@postgres:5432/testdb"
    REDIS_URL: "redis://redis:6379"
  script:
    - |
      pytest tests/unit \
        --cov=src \
        --cov-report=xml \
        --cov-report=term \
        --cov-report=html \
        --junitxml=junit.xml \
        -v
  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
  artifacts:
    when: always
    reports:
      junit: junit.xml
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml
    paths:
      - coverage.xml
      - htmlcov/
    expire_in: 30 days

integration-test:
  extends: .python_template
  stage: test
  services:
    - postgres:15
    - redis:7-alpine
  variables:
    POSTGRES_DB: testdb
    POSTGRES_USER: testuser
    POSTGRES_PASSWORD: testpass
    DATABASE_URL: "postgresql://testuser:testpass@postgres:5432/testdb"
    REDIS_URL: "redis://redis:6379"
  script:
    - pytest tests/integration -v --junitxml=junit-integration.xml
  artifacts:
    when: always
    reports:
      junit: junit-integration.xml
    expire_in: 7 days
  only:
    - main
    - develop
    - merge_requests

# Build stage
build:package:
  extends: .python_template
  stage: build
  script:
    - pip install build wheel setuptools
    - python -m build
    - ls -lh dist/
  artifacts:
    paths:
      - dist/
    expire_in: 7 days
  only:
    - main
    - develop
    - tags

build:docker:
  stage: build
  image: docker:24-cli
  services:
    - docker:24-dind
  variables:
    IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
  before_script:
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
  script:
    # Pull previous image for caching
    - docker pull $CI_REGISTRY_IMAGE:latest || true

    # Build with cache
    - |
      docker build \
        --cache-from $CI_REGISTRY_IMAGE:latest \
        --tag $IMAGE_TAG \
        --tag $CI_REGISTRY_IMAGE:latest \
        --build-arg BUILDKIT_INLINE_CACHE=1 \
        .

    # Push images
    - docker push $IMAGE_TAG
    - docker push $CI_REGISTRY_IMAGE:latest

    - echo "IMAGE_FULL_NAME=$IMAGE_TAG" >> build.env
  artifacts:
    reports:
      dotenv: build.env
  only:
    - main
    - develop
    - tags
  tags:
    - docker

# E2E tests (only on main)
e2e-test:
  extends: .python_template
  stage: test
  needs: [build:package]
  dependencies:
    - build:package
  script:
    - pip install dist/*.whl
    - pytest tests/e2e -v
  artifacts:
    when: always
    paths:
      - test-results/
    expire_in: 7 days
  only:
    - main

# Deploy to PyPI
deploy:pypi:
  stage: deploy
  image: python:3.11
  needs: [build:package, test]
  dependencies:
    - build:package
  environment:
    name: pypi
    url: https://pypi.org/project/your-package
  before_script:
    - pip install twine
  script:
    - twine check dist/*
    - twine upload dist/* --username __token__ --password $PYPI_TOKEN
  only:
    - tags
  when: manual

# Deploy Docker to staging
deploy:staging:
  stage: deploy
  image: bitnami/kubectl:latest
  needs: [build:docker]
  environment:
    name: staging
    url: https://staging.example.com
    on_stop: stop:staging
  before_script:
    - kubectl config use-context staging-cluster
  script:
    - kubectl set image deployment/myapp myapp=$IMAGE_FULL_NAME --namespace=staging --record
    - kubectl rollout status deployment/myapp --namespace=staging --timeout=5m
    # Smoke test
    - |
      POD=$(kubectl get pod -n staging -l app=myapp -o jsonpath="{.items[0].metadata.name}")
      kubectl exec -n staging $POD -- python -c "import sys; print(sys.version)"
      kubectl exec -n staging $POD -- curl -f http://localhost:8000/health || exit 1
  only:
    - develop
  when: manual

stop:staging:
  stage: deploy
  image: bitnami/kubectl:latest
  environment:
    name: staging
    action: stop
  script:
    - kubectl scale deployment/myapp --replicas=0 --namespace=staging
  when: manual
  only:
    - develop

# Deploy to production
deploy:production:
  stage: deploy
  image: bitnami/kubectl:latest
  needs: [build:docker, e2e-test]
  environment:
    name: production
    url: https://example.com
  before_script:
    - kubectl config use-context production-cluster
  script:
    - echo "Deploying to production..."
    - kubectl set image deployment/myapp myapp=$IMAGE_FULL_NAME --namespace=production --record
    - kubectl rollout status deployment/myapp --namespace=production --timeout=5m

    # Health check
    - sleep 10
    - |
      for i in {1..10}; do
        POD=$(kubectl get pod -n production -l app=myapp -o jsonpath="{.items[0].metadata.name}")
        if kubectl exec -n production $POD -- curl -f http://localhost:8000/health; then
          echo "Health check passed"
          exit 0
        fi
        echo "Attempt $i failed, retrying..."
        sleep 10
      done
      echo "Health check failed"
      exit 1
  only:
    - main
  when: manual

# Deploy to Cloud Run (Google Cloud)
deploy:cloudrun:
  stage: deploy
  image: google/cloud-sdk:alpine
  needs: [build:docker]
  environment:
    name: production
    url: https://your-app.run.app
  before_script:
    - echo $GCP_SERVICE_KEY | base64 -d > ${HOME}/gcp-key.json
    - gcloud auth activate-service-account --key-file ${HOME}/gcp-key.json
    - gcloud config set project $GCP_PROJECT_ID
  script:
    - |
      gcloud run deploy your-app \
        --image $IMAGE_FULL_NAME \
        --region us-central1 \
        --platform managed \
        --allow-unauthenticated

    # Health check
    - |
      URL=$(gcloud run services describe your-app --region us-central1 --format 'value(status.url)')
      curl -f $URL/health || exit 1
  only:
    - main
  when: manual

# Create release
release:
  stage: deploy
  image: registry.gitlab.com/gitlab-org/release-cli:latest
  needs: [deploy:production]
  script:
    - echo "Creating release"
  release:
    tag_name: $CI_COMMIT_TAG
    description: |
      Python Package: https://pypi.org/project/your-package/$CI_COMMIT_TAG
      Docker Image: $IMAGE_FULL_NAME

      Changes in this release:
      $CI_COMMIT_MESSAGE
  only:
    - tags

# GitLab built-in security templates
include:
  - template: Security/Dependency-Scanning.gitlab-ci.yml
  - template: Security/SAST.gitlab-ci.yml

# Override GitLab template stages
dependency_scanning:
  stage: security

sast:
  stage: security

# Workflow rules
workflow:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_COMMIT_BRANCH == "main"'
    - if: '$CI_COMMIT_BRANCH == "develop"'
    - if: '$CI_COMMIT_TAG'

# Interruptible jobs
.interruptible_template:
  interruptible: true

lint:
  extends: [.python_template, .interruptible_template]

test:
  extends: [.python_template, .interruptible_template]

integration-test:
  extends: [.python_template, .interruptible_template]