zhongwei/gh-ahmedasmar-devops-claude-skills-ci-cd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
Zhongwei Li
2025-11-29 17:51:12 +08:00
commit 1878d01517
21 changed files with 8728 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

164
assets/templates/github-actions/docker-build.yml Normal file
View File

@@ -0,0 +1,164 @@
# Docker Build & Push Pipeline
# Multi-platform build with caching and security scanning
name: Docker Build
on:
push:
branches: [main]
tags: ['v*']
pull_request:
branches: [main]
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
jobs:
build:
name: Build & Push Docker Image
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
packages: write
security-events: write # For uploading SARIF
steps:
- uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to Container Registry
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=sha,prefix={{branch}}-
- name: Build and push Docker image
id: build
uses: docker/build-push-action@v5
with:
context: .
platforms: linux/amd64,linux/arm64
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
format: 'sarif'
output: 'trivy-results.sarif'
severity: 'CRITICAL,HIGH'
- name: Upload Trivy results to GitHub Security
if: always()
uses: github/codeql-action/upload-sarif@v3
with:
sarif_file: 'trivy-results.sarif'
- name: Run Grype vulnerability scanner
uses: anchore/scan-action@v3
with:
image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
fail-build: true
severity-cutoff: high
- name: Generate SBOM
uses: anchore/sbom-action@v0
with:
image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
format: spdx-json
output-file: sbom.spdx.json
- name: Upload SBOM
uses: actions/upload-artifact@v4
with:
name: sbom
path: sbom.spdx.json
sign:
name: Sign Container Image
runs-on: ubuntu-latest
needs: build
if: github.event_name != 'pull_request'
permissions:
contents: read
packages: write
id-token: write
steps:
- uses: actions/checkout@v4
- name: Install Cosign
uses: sigstore/cosign-installer@v3
- name: Log in to Container Registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Sign the images
env:
DIGEST: ${{ needs.build.outputs.digest }}
run: |
cosign sign --yes ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${DIGEST}
deploy:
name: Deploy to Kubernetes
runs-on: ubuntu-latest
needs: [build, sign]
if: github.ref == 'refs/heads/main'
environment:
name: production
url: https://example.com
steps:
- uses: actions/checkout@v4
- name: Configure kubectl
uses: azure/k8s-set-context@v3
with:
method: kubeconfig
kubeconfig: ${{ secrets.KUBE_CONFIG }}
- name: Deploy to Kubernetes
run: |
kubectl set image deployment/myapp \
myapp=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} \
--record
- name: Verify deployment
run: |
kubectl rollout status deployment/myapp --timeout=5m
- name: Run smoke tests
run: |
POD=$(kubectl get pod -l app=myapp -o jsonpath="{.items[0].metadata.name}")
kubectl exec $POD -- curl -f http://localhost:8080/health

420
assets/templates/github-actions/go-ci.yml Normal file
View File

@@ -0,0 +1,420 @@
# Go CI/CD Pipeline
# Optimized with caching, matrix testing, and deployment
name: Go CI
on:
push:
branches: [main, develop]
paths-ignore:
- '**.md'
- 'docs/**'
pull_request:
branches: [main]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
GO_VERSION: '1.22'
jobs:
# Security: Secret Scanning
secret-scan:
name: Secret Scanning
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: TruffleHog Secret Scan
uses: trufflesecurity/trufflehog@main
with:
path: ./
base: ${{ github.event.repository.default_branch }}
head: HEAD
- name: Gitleaks
uses: gitleaks/gitleaks-action@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Security: SAST
sast:
name: Static Analysis (CodeQL)
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
security-events: write
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: go
queries: security-and-quality
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
lint:
name: Lint
runs-on: ubuntu-latest
needs: [secret-scan]
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache: true
- name: Run golangci-lint
uses: golangci/golangci-lint-action@v4
with:
version: latest
args: --timeout=5m
- name: Check formatting
run: |
if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then
echo "Please run: gofmt -s -w ."
gofmt -s -l .
exit 1
fi
- name: Check go mod tidy
run: |
go mod tidy
git diff --exit-code go.mod go.sum
test:
name: Test (Go ${{ matrix.go-version }}, ${{ matrix.os }})
runs-on: ${{ matrix.os }}
timeout-minutes: 20
strategy:
matrix:
go-version: ['1.21', '1.22']
os: [ubuntu-latest, macos-latest]
fail-fast: false
services:
postgres:
image: postgres:15
env:
POSTGRES_PASSWORD: postgres
POSTGRES_DB: testdb
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
redis:
image: redis:7-alpine
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 6379:6379
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
cache: true
- name: Download dependencies
run: go mod download
- name: Run unit tests
env:
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/testdb?sslmode=disable
REDIS_URL: redis://localhost:6379
run: |
go test -v -race -coverprofile=coverage.out -covermode=atomic ./...
- name: Upload coverage to Codecov
if: matrix.go-version == '1.22' && matrix.os == 'ubuntu-latest'
uses: codecov/codecov-action@v4
with:
files: ./coverage.out
fail_ci_if_error: false
- name: Run benchmarks
if: matrix.go-version == '1.22' && matrix.os == 'ubuntu-latest'
run: go test -bench=. -benchmem ./... | tee benchmark.txt
- name: Upload benchmark results
if: matrix.go-version == '1.22' && matrix.os == 'ubuntu-latest'
uses: actions/upload-artifact@v4
with:
name: benchmark-results
path: benchmark.txt
security:
name: Security Scanning
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache: true
- name: Run Gosec
uses: securego/gosec@master
with:
args: '-fmt json -out gosec-report.json ./...'
continue-on-error: true
- name: Run govulncheck
run: |
go install golang.org/x/vuln/cmd/govulncheck@latest
govulncheck ./...
- name: Upload security reports
if: always()
uses: actions/upload-artifact@v4
with:
name: security-reports
path: gosec-report.json
build:
name: Build
runs-on: ubuntu-latest
needs: [lint, test, sast, security]
timeout-minutes: 15
strategy:
matrix:
goos: [linux, darwin, windows]
goarch: [amd64, arm64]
exclude:
- goos: windows
goarch: arm64
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache: true
- name: Build binary
env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
CGO_ENABLED: 0
run: |
OUTPUT="myapp-${{ matrix.goos }}-${{ matrix.goarch }}"
if [ "${{ matrix.goos }}" = "windows" ]; then
OUTPUT="${OUTPUT}.exe"
fi
go build \
-ldflags="-s -w -X main.version=${{ github.sha }} -X main.buildTime=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
-o $OUTPUT \
./cmd/myapp
ls -lh $OUTPUT
- name: Upload binary
uses: actions/upload-artifact@v4
with:
name: myapp-${{ matrix.goos }}-${{ matrix.goarch }}
path: myapp-*
retention-days: 7
integration-test:
name: Integration Tests
runs-on: ubuntu-latest
needs: build
if: github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
timeout-minutes: 30
services:
postgres:
image: postgres:15
env:
POSTGRES_PASSWORD: postgres
POSTGRES_DB: testdb
options: >-
--health-cmd pg_isready
--health-interval 10s
ports:
- 5432:5432
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache: true
- name: Download Linux binary
uses: actions/download-artifact@v4
with:
name: myapp-linux-amd64
- name: Make binary executable
run: chmod +x myapp-linux-amd64
- name: Run integration tests
env:
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/testdb?sslmode=disable
BINARY_PATH: ./myapp-linux-amd64
run: go test -v -tags=integration ./tests/integration/...
docker:
name: Build Docker Image
runs-on: ubuntu-latest
needs: [build, test]
if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ghcr.io/${{ github.repository }}
tags: |
type=ref,event=branch
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=sha
- name: Build and push
uses: docker/build-push-action@v5
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: |
VERSION=${{ github.sha }}
BUILD_TIME=${{ github.event.head_commit.timestamp }}
deploy:
name: Deploy to Production
runs-on: ubuntu-latest
needs: [docker, integration-test]
if: github.ref == 'refs/heads/main'
environment:
name: production
url: https://api.example.com
permissions:
contents: read
id-token: write
steps:
- uses: actions/checkout@v4
- uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
- name: Deploy to Cloud Run
run: |
gcloud run deploy myapp \
--image ghcr.io/${{ github.repository }}:${{ github.sha }} \
--region us-central1 \
--platform managed \
--allow-unauthenticated \
--memory 512Mi \
--cpu 1 \
--max-instances 10
- name: Health check
run: |
URL=$(gcloud run services describe myapp --region us-central1 --format 'value(status.url)')
for i in {1..10}; do
if curl -f $URL/health; then
echo "Health check passed"
exit 0
fi
echo "Attempt $i failed, retrying..."
sleep 10
done
exit 1
release:
name: Create Release
runs-on: ubuntu-latest
needs: [build]
if: startsWith(github.ref, 'refs/tags/v')
permissions:
contents: write
steps:
- uses: actions/checkout@v4
- name: Download all artifacts
uses: actions/download-artifact@v4
with:
path: artifacts/
- name: Create checksums
run: |
cd artifacts
for dir in myapp-*; do
cd $dir
sha256sum * > checksums.txt
cd ..
done
- name: Create GitHub Release
uses: softprops/action-gh-release@v1
with:
files: artifacts/**/*
generate_release_notes: true
draft: false
prerelease: false

313
assets/templates/github-actions/node-ci.yml Normal file
View File

@@ -0,0 +1,313 @@
# Node.js CI/CD Pipeline
# Optimized workflow with caching, matrix testing, and deployment
name: Node.js CI
on:
push:
branches: [main, develop]
paths-ignore:
- '**.md'
- 'docs/**'
pull_request:
branches: [main]
# Cancel in-progress runs for same workflow
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
# Security: Secret Scanning
secret-scan:
name: Secret Scanning
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: TruffleHog Secret Scan
uses: trufflesecurity/trufflehog@main
with:
path: ./
base: ${{ github.event.repository.default_branch }}
head: HEAD
- name: Gitleaks
uses: gitleaks/gitleaks-action@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Security: SAST
sast:
name: Static Analysis
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
security-events: write
steps:
- uses: actions/checkout@v4
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: javascript
queries: security-and-quality
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
- name: Run Semgrep
uses: returntocorp/semgrep-action@v1
with:
config: >-
p/security-audit
p/owasp-top-ten
# Security: Dependency Scanning
dependency-scan:
name: Dependency Security
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- name: Install dependencies
run: npm ci
- name: npm audit
run: |
npm audit --audit-level=moderate --json > npm-audit.json || true
npm audit --audit-level=high
continue-on-error: false
- name: Upload audit results
if: always()
uses: actions/upload-artifact@v4
with:
name: npm-audit-report
path: npm-audit.json
lint:
name: Lint
runs-on: ubuntu-latest
needs: [secret-scan]
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- name: Install dependencies
run: npm ci
- name: Run linter
run: npm run lint
- name: Check formatting
run: npm run format:check
test:
name: Test (Node ${{ matrix.node }})
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
matrix:
node: [18, 20, 22]
fail-fast: false
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node }}
cache: 'npm'
- name: Install dependencies
run: npm ci
- name: Run unit tests
run: npm run test:unit
- name: Run integration tests
run: npm run test:integration
if: matrix.node == 20 # Only run on one version
- name: Upload coverage
uses: codecov/codecov-action@v4
if: matrix.node == 20
with:
files: ./coverage/lcov.info
fail_ci_if_error: false
build:
name: Build
runs-on: ubuntu-latest
needs: [lint, test, sast, dependency-scan]
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- name: Install dependencies
run: npm ci
- name: Build application
run: npm run build
- name: Upload build artifacts
uses: actions/upload-artifact@v4
with:
name: dist-${{ github.sha }}
path: dist/
retention-days: 7
e2e:
name: E2E Tests
runs-on: ubuntu-latest
needs: build
if: github.ref == 'refs/heads/main'
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- name: Install dependencies
run: npm ci
- name: Download build artifacts
uses: actions/download-artifact@v4
with:
name: dist-${{ github.sha }}
path: dist/
- name: Run E2E tests
run: npm run test:e2e
- name: Upload test results
if: always()
uses: actions/upload-artifact@v4
with:
name: e2e-results
path: test-results/
deploy-staging:
name: Deploy to Staging
runs-on: ubuntu-latest
needs: [build, test]
if: github.ref == 'refs/heads/develop'
environment:
name: staging
url: https://staging.example.com
permissions:
contents: read
id-token: write # For OIDC
steps:
- uses: actions/checkout@v4
- name: Download build artifacts
uses: actions/download-artifact@v4
with:
name: dist-${{ github.sha }}
path: dist/
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
aws-region: us-east-1
- name: Deploy to S3
run: |
aws s3 sync dist/ s3://${{ secrets.STAGING_BUCKET }}
aws cloudfront create-invalidation --distribution-id ${{ secrets.STAGING_CF_DIST }} --paths "/*"
- name: Smoke tests
run: |
sleep 10
curl -f https://staging.example.com/health || exit 1
deploy-production:
name: Deploy to Production
runs-on: ubuntu-latest
needs: [e2e]
if: github.ref == 'refs/heads/main'
environment:
name: production
url: https://example.com
permissions:
contents: read
id-token: write
steps:
- uses: actions/checkout@v4
- name: Download build artifacts
uses: actions/download-artifact@v4
with:
name: dist-${{ github.sha }}
path: dist/
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
aws-region: us-east-1
- name: Deploy to S3
run: |
aws s3 sync dist/ s3://${{ secrets.PRODUCTION_BUCKET }}
aws cloudfront create-invalidation --distribution-id ${{ secrets.PRODUCTION_CF_DIST }} --paths "/*"
- name: Health check
run: |
for i in {1..10}; do
if curl -f https://example.com/health; then
echo "Health check passed"
exit 0
fi
echo "Attempt $i failed, retrying..."
sleep 10
done
echo "Health check failed"
exit 1
- name: Create deployment record
run: |
echo "Deployed version: ${{ github.sha }}"
echo "Deployment time: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
# Optionally create release with gh CLI:
# gh release create v${{ github.run_number }} \
# --title "Release v${{ github.run_number }}" \
# --notes "Deployed commit ${{ github.sha }}"

388
assets/templates/github-actions/python-ci.yml Normal file
View File

@@ -0,0 +1,388 @@
# Python CI/CD Pipeline
# Optimized with caching, matrix testing, and deployment
name: Python CI
on:
push:
branches: [main, develop]
paths-ignore:
- '**.md'
- 'docs/**'
pull_request:
branches: [main]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
# Security: Secret Scanning
secret-scan:
name: Secret Scanning
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: TruffleHog Secret Scan
uses: trufflesecurity/trufflehog@main
with:
path: ./
base: ${{ github.event.repository.default_branch }}
head: HEAD
- name: Gitleaks
uses: gitleaks/gitleaks-action@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Security: SAST
sast:
name: Static Analysis (CodeQL)
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
security-events: write
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: python
queries: security-and-quality
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
lint:
name: Lint & Format Check
runs-on: ubuntu-latest
needs: [secret-scan]
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.11'
cache: 'pip'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install ruff black mypy isort
- name: Run ruff
run: ruff check .
- name: Check formatting with black
run: black --check .
- name: Check import sorting
run: isort --check-only .
- name: Type check with mypy
run: mypy .
continue-on-error: true # Don't fail on type errors initially
test:
name: Test (Python ${{ matrix.python-version }})
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
matrix:
python-version: ['3.9', '3.10', '3.11', '3.12']
fail-fast: false
services:
postgres:
image: postgres:15
env:
POSTGRES_PASSWORD: postgres
POSTGRES_DB: testdb
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
redis:
image: redis:7-alpine
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 6379:6379
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
pip install -r requirements-dev.txt
- name: Run unit tests
env:
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/testdb
REDIS_URL: redis://localhost:6379
run: |
pytest tests/unit \
--cov=src \
--cov-report=xml \
--cov-report=term \
--junitxml=junit.xml \
-v
- name: Run integration tests
if: matrix.python-version == '3.11'
env:
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/testdb
REDIS_URL: redis://localhost:6379
run: |
pytest tests/integration -v
- name: Upload coverage to Codecov
if: matrix.python-version == '3.11'
uses: codecov/codecov-action@v4
with:
files: ./coverage.xml
fail_ci_if_error: false
- name: Upload test results
if: always()
uses: actions/upload-artifact@v4
with:
name: test-results-${{ matrix.python-version }}
path: junit.xml
security:
name: Security Scanning
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.11'
cache: 'pip'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Run bandit security scan
run: |
pip install bandit
bandit -r src/ -f json -o bandit-report.json -ll || true
bandit -r src/ -ll
continue-on-error: false
- name: Run safety check
run: |
pip install safety
safety check --json --output safety-report.json || true
safety check
continue-on-error: true
- name: pip-audit dependency scan
run: |
pip install pip-audit
pip-audit --requirement requirements.txt --format json --output pip-audit.json || true
pip-audit --requirement requirements.txt
continue-on-error: false
- name: Upload security reports
if: always()
uses: actions/upload-artifact@v4
with:
name: security-reports
path: |
bandit-report.json
safety-report.json
pip-audit.json
build:
name: Build Package
runs-on: ubuntu-latest
needs: [lint, test, sast, security]
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.11'
cache: 'pip'
- name: Install build tools
run: |
python -m pip install --upgrade pip
pip install build wheel setuptools
- name: Build package
run: python -m build
- name: Upload distribution
uses: actions/upload-artifact@v4
with:
name: dist-${{ github.sha }}
path: dist/
retention-days: 7
e2e:
name: E2E Tests
runs-on: ubuntu-latest
needs: build
if: github.ref == 'refs/heads/main'
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.11'
cache: 'pip'
- name: Download build artifacts
uses: actions/download-artifact@v4
with:
name: dist-${{ github.sha }}
path: dist/
- name: Install package
run: |
pip install dist/*.whl
pip install -r requirements-dev.txt
- name: Run E2E tests
run: pytest tests/e2e -v
deploy-pypi:
name: Deploy to PyPI
runs-on: ubuntu-latest
needs: [build, test]
if: startsWith(github.ref, 'refs/tags/v')
environment:
name: pypi
url: https://pypi.org/project/your-package
permissions:
id-token: write # For trusted publishing
steps:
- uses: actions/download-artifact@v4
with:
name: dist-${{ github.sha }}
path: dist/
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
# Uses OIDC trusted publishing - no token needed!
deploy-docker:
name: Build & Push Docker Image
runs-on: ubuntu-latest
needs: [build, test]
if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ghcr.io/${{ github.repository }}
tags: |
type=ref,event=branch
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=sha
- name: Build and push
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
deploy-cloud:
name: Deploy to Cloud Run
runs-on: ubuntu-latest
needs: deploy-docker
if: github.ref == 'refs/heads/main'
environment:
name: production
url: https://your-app.run.app
permissions:
contents: read
id-token: write
steps:
- uses: actions/checkout@v4
- uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
- name: Deploy to Cloud Run
run: |
gcloud run deploy your-app \
--image ghcr.io/${{ github.repository }}:${{ github.sha }} \
--region us-central1 \
--platform managed \
--allow-unauthenticated
- name: Health check
run: |
URL=$(gcloud run services describe your-app --region us-central1 --format 'value(status.url)')
curl -f $URL/health || exit 1

416
assets/templates/github-actions/security-scan.yml Normal file
View File

@@ -0,0 +1,416 @@
# Complete DevSecOps Security Scanning Pipeline
# SAST, DAST, SCA, Container Scanning, Secret Scanning
name: Security Scanning
on:
push:
branches: [main, develop]
pull_request:
branches: [main]
schedule:
- cron: '0 2 * * 1' # Weekly full scan on Monday 2 AM
concurrency:
group: security-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
# Stage 1: Secret Scanning
secret-scan:
name: Secret Scanning
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # Full history for secret scanning
- name: TruffleHog Secret Scan
uses: trufflesecurity/trufflehog@main
with:
path: ./
base: ${{ github.event.repository.default_branch }}
head: HEAD
- name: Gitleaks
uses: gitleaks/gitleaks-action@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Stage 2: SAST (Static Application Security Testing)
sast-codeql:
name: CodeQL Analysis
runs-on: ubuntu-latest
needs: secret-scan
timeout-minutes: 30
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
language: ['javascript', 'python']
steps:
- uses: actions/checkout@v4
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
queries: security-and-quality
- name: Autobuild
uses: github/codeql-action/autobuild@v3
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
with:
category: "/language:${{matrix.language}}"
sast-semgrep:
name: Semgrep SAST
runs-on: ubuntu-latest
needs: secret-scan
timeout-minutes: 15
permissions:
contents: read
security-events: write
steps:
- uses: actions/checkout@v4
- name: Run Semgrep
uses: returntocorp/semgrep-action@v1
with:
config: >-
p/security-audit
p/owasp-top-ten
p/cwe-top-25
publishToken: ${{ secrets.SEMGREP_APP_TOKEN }}
# Stage 3: SCA (Software Composition Analysis)
sca-dependencies:
name: Dependency Scanning
runs-on: ubuntu-latest
needs: secret-scan
timeout-minutes: 15
permissions:
contents: read
pull-requests: write
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
- name: Install dependencies
run: npm ci
- name: npm audit
run: |
npm audit --audit-level=moderate --json > npm-audit.json
npm audit --audit-level=high
continue-on-error: true
- name: Dependency Review (PR only)
if: github.event_name == 'pull_request'
uses: actions/dependency-review-action@v4
with:
fail-on-severity: high
- name: Snyk Security Scan
uses: snyk/actions/node@master
env:
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
with:
args: --severity-threshold=high --json-file-output=snyk-report.json
continue-on-error: true
- name: Upload scan results
if: always()
uses: actions/upload-artifact@v4
with:
name: sca-reports
path: |
npm-audit.json
snyk-report.json
# Stage 4: Build Application
build:
name: Build Application
runs-on: ubuntu-latest
needs: [sast-codeql, sast-semgrep, sca-dependencies]
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- run: npm ci
- run: npm run build
- uses: actions/upload-artifact@v4
with:
name: build-output
path: dist/
# Stage 5: Container Security Scanning
container-scan:
name: Container Security
runs-on: ubuntu-latest
needs: build
timeout-minutes: 20
permissions:
contents: read
security-events: write
packages: write
steps:
- uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build Docker image
uses: docker/build-push-action@v5
with:
context: .
load: true
tags: myapp:${{ github.sha }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: myapp:${{ github.sha }}
format: 'sarif'
output: 'trivy-results.sarif'
severity: 'CRITICAL,HIGH'
exit-code: '1'
- name: Upload Trivy results to GitHub Security
if: always()
uses: github/codeql-action/upload-sarif@v3
with:
sarif_file: 'trivy-results.sarif'
- name: Run Grype vulnerability scanner
uses: anchore/scan-action@v3
id: grype
with:
image: myapp:${{ github.sha }}
fail-build: true
severity-cutoff: high
output-format: sarif
- name: Upload Grype results
if: always()
uses: github/codeql-action/upload-sarif@v3
with:
sarif_file: ${{ steps.grype.outputs.sarif }}
- name: Generate SBOM with Syft
uses: anchore/sbom-action@v0
with:
image: myapp:${{ github.sha }}
format: spdx-json
output-file: sbom.spdx.json
- name: Upload SBOM
uses: actions/upload-artifact@v4
with:
name: sbom
path: sbom.spdx.json
# Stage 6: DAST (Dynamic Application Security Testing)
dast-baseline:
name: DAST Baseline Scan
runs-on: ubuntu-latest
needs: container-scan
if: github.ref == 'refs/heads/main' || github.event_name == 'schedule'
timeout-minutes: 30
permissions:
contents: read
issues: write
services:
app:
image: myapp:latest
ports:
- 8080:8080
options: --health-cmd "curl -f http://localhost:8080/health" --health-interval 10s
steps:
- uses: actions/checkout@v4
- name: Wait for application
run: |
timeout 60 bash -c 'until curl -f http://localhost:8080/health; do sleep 2; done'
- name: OWASP ZAP Baseline Scan
uses: zaproxy/action-baseline@v0.10.0
with:
target: 'http://localhost:8080'
rules_file_name: '.zap/rules.tsv'
cmd_options: '-a'
fail_action: true
- name: Upload ZAP report
if: always()
uses: actions/upload-artifact@v4
with:
name: zap-baseline-report
path: report_html.html
dast-full-scan:
name: DAST Full Scan
runs-on: ubuntu-latest
needs: container-scan
if: github.event_name == 'schedule'
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
- name: OWASP ZAP Full Scan
uses: zaproxy/action-full-scan@v0.10.0
with:
target: 'https://staging.example.com'
rules_file_name: '.zap/rules.tsv'
allow_issue_writing: false
- name: Upload ZAP report
if: always()
uses: actions/upload-artifact@v4
with:
name: zap-full-scan-report
path: report_html.html
# Stage 7: License Compliance
license-check:
name: License Compliance
runs-on: ubuntu-latest
needs: build
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
- run: npm ci
- name: Check licenses
run: |
npx license-checker --production \
--onlyAllow "MIT;Apache-2.0;BSD-2-Clause;BSD-3-Clause;ISC;0BSD" \
--json --out license-report.json
- name: Upload license report
uses: actions/upload-artifact@v4
with:
name: license-report
path: license-report.json
# Stage 8: Security Gate
security-gate:
name: Security Quality Gate
runs-on: ubuntu-latest
needs: [sast-codeql, sast-semgrep, sca-dependencies, container-scan, license-check]
if: always()
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- name: Download all artifacts
uses: actions/download-artifact@v4
- name: Evaluate Security Posture
run: |
echo "## 🔒 Security Scan Summary" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
# Check job statuses
echo "### Scan Results" >> $GITHUB_STEP_SUMMARY
echo "- ✅ Secret Scanning: Complete" >> $GITHUB_STEP_SUMMARY
echo "- ✅ SAST (CodeQL): Complete" >> $GITHUB_STEP_SUMMARY
echo "- ✅ SAST (Semgrep): Complete" >> $GITHUB_STEP_SUMMARY
echo "- ✅ SCA (Dependencies): Complete" >> $GITHUB_STEP_SUMMARY
echo "- ✅ Container Scanning: Complete" >> $GITHUB_STEP_SUMMARY
echo "- ✅ License Compliance: Complete" >> $GITHUB_STEP_SUMMARY
# Parse results and determine if we can proceed
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Security Gate Status" >> $GITHUB_STEP_SUMMARY
if [ "${{ needs.sast-codeql.result }}" == "failure" ] || \
[ "${{ needs.container-scan.result }}" == "failure" ]; then
echo "❌ **Security gate FAILED** - Critical vulnerabilities found" >> $GITHUB_STEP_SUMMARY
exit 1
else
echo "✅ **Security gate PASSED** - No critical issues detected" >> $GITHUB_STEP_SUMMARY
fi
# Stage 9: Security Report
security-report:
name: Generate Security Report
runs-on: ubuntu-latest
needs: security-gate
if: always() && github.ref == 'refs/heads/main'
steps:
- uses: actions/checkout@v4
- name: Download all artifacts
uses: actions/download-artifact@v4
- name: Create unified security report
run: |
cat << EOF > security-report.md
# Security Scan Report - $(date +%Y-%m-%d)
## Summary
- **Repository:** ${{ github.repository }}
- **Branch:** ${{ github.ref_name }}
- **Commit:** ${{ github.sha }}
- **Scan Date:** $(date -u +"%Y-%m-%d %H:%M:%S UTC")
## Scans Performed
1. Secret Scanning (TruffleHog, Gitleaks)
2. SAST (CodeQL, Semgrep)
3. SCA (npm audit, Snyk)
4. Container Scanning (Trivy, Grype)
5. License Compliance
## Status
All security scans completed. See artifacts for detailed reports.
EOF
cat security-report.md >> $GITHUB_STEP_SUMMARY
- name: Upload security report
uses: actions/upload-artifact@v4
with:
name: security-report
path: security-report.md
retention-days: 90

298
assets/templates/gitlab-ci/docker-build.yml Normal file
View File

@@ -0,0 +1,298 @@
# GitLab CI/CD Docker Build Pipeline
# Multi-platform build with DinD, security scanning, and Container Registry
stages:
- build
- scan
- sign
- deploy
variables:
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: "/certs"
# Use project's container registry
IMAGE_NAME: $CI_REGISTRY_IMAGE
IMAGE_TAG: $CI_COMMIT_SHORT_SHA
# BuildKit for better caching
DOCKER_BUILDKIT: 1
# Build multi-platform Docker image
build:
stage: build
image: docker:24-cli
services:
- docker:24-dind
before_script:
# Login to GitLab Container Registry
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
# Set up buildx for multi-platform builds
- docker buildx create --use --name builder || docker buildx use builder
- docker buildx inspect --bootstrap
script:
# Build and push multi-platform image
- |
docker buildx build \
--platform linux/amd64,linux/arm64 \
--cache-from type=registry,ref=$IMAGE_NAME:buildcache \
--cache-to type=registry,ref=$IMAGE_NAME:buildcache,mode=max \
--tag $IMAGE_NAME:$IMAGE_TAG \
--tag $IMAGE_NAME:latest \
--push \
--build-arg CI_COMMIT_SHA=$CI_COMMIT_SHA \
--build-arg CI_COMMIT_REF_NAME=$CI_COMMIT_REF_NAME \
.
- echo "IMAGE_FULL_NAME=$IMAGE_NAME:$IMAGE_TAG" >> build.env
artifacts:
reports:
dotenv: build.env
only:
- branches
- tags
tags:
- docker
# Alternative: Simple build without multi-arch
build:simple:
stage: build
image: docker:24-cli
services:
- docker:24-dind
before_script:
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
script:
# Pull previous image for layer caching
- docker pull $IMAGE_NAME:latest || true
# Build with cache
- |
docker build \
--cache-from $IMAGE_NAME:latest \
--tag $IMAGE_NAME:$IMAGE_TAG \
--tag $IMAGE_NAME:latest \
--build-arg BUILDKIT_INLINE_CACHE=1 \
.
# Push images
- docker push $IMAGE_NAME:$IMAGE_TAG
- docker push $IMAGE_NAME:latest
- echo "IMAGE_FULL_NAME=$IMAGE_NAME:$IMAGE_TAG" >> build.env
artifacts:
reports:
dotenv: build.env
only:
- branches
- tags
when: manual # Use this OR the multi-arch build above
tags:
- docker
# Trivy vulnerability scanning
trivy:scan:
stage: scan
image: aquasec/trivy:latest
needs: [build]
variables:
GIT_STRATEGY: none
script:
# Scan for HIGH and CRITICAL vulnerabilities
- trivy image --severity HIGH,CRITICAL --exit-code 0 --format json --output trivy-report.json $IMAGE_FULL_NAME
- trivy image --severity HIGH,CRITICAL --exit-code 1 $IMAGE_FULL_NAME
artifacts:
when: always
paths:
- trivy-report.json
expire_in: 30 days
allow_failure: false
only:
- branches
- tags
# Grype scanning (alternative/additional)
grype:scan:
stage: scan
image: anchore/grype:latest
needs: [build]
variables:
GIT_STRATEGY: none
before_script:
- echo $CI_REGISTRY_PASSWORD | grype registry login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
script:
- grype $IMAGE_FULL_NAME --fail-on high --output json --file grype-report.json
artifacts:
when: always
paths:
- grype-report.json
expire_in: 30 days
allow_failure: true
only:
- branches
- tags
# GitLab Container Scanning (uses Trivy)
container_scanning:
stage: scan
needs: [build]
variables:
CS_IMAGE: $IMAGE_FULL_NAME
GIT_STRATEGY: none
allow_failure: true
include:
- template: Security/Container-Scanning.gitlab-ci.yml
# Generate SBOM (Software Bill of Materials)
sbom:
stage: scan
image: anchore/syft:latest
needs: [build]
variables:
GIT_STRATEGY: none
before_script:
- echo $CI_REGISTRY_PASSWORD | syft registry login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
script:
- syft $IMAGE_FULL_NAME -o spdx-json > sbom.spdx.json
- syft $IMAGE_FULL_NAME -o cyclonedx-json > sbom.cyclonedx.json
artifacts:
paths:
- sbom.spdx.json
- sbom.cyclonedx.json
expire_in: 90 days
only:
- main
- tags
# Sign container image (requires cosign setup)
sign:image:
stage: sign
image: gcr.io/projectsigstore/cosign:latest
needs: [build, trivy:scan]
variables:
GIT_STRATEGY: none
before_script:
- echo $CI_REGISTRY_PASSWORD | cosign login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
script:
# Sign using keyless mode with OIDC
- cosign sign --yes $IMAGE_FULL_NAME
only:
- main
- tags
when: manual # Require manual approval for signing
# Deploy to Kubernetes
deploy:staging:
stage: deploy
image: bitnami/kubectl:latest
needs: [build, trivy:scan]
environment:
name: staging
url: https://staging.example.com
on_stop: stop:staging
before_script:
- kubectl config use-context staging-cluster
script:
- kubectl set image deployment/myapp myapp=$IMAGE_FULL_NAME --namespace=staging --record
- kubectl rollout status deployment/myapp --namespace=staging --timeout=5m
# Verify deployment
- |
POD=$(kubectl get pod -n staging -l app=myapp -o jsonpath="{.items[0].metadata.name}")
kubectl exec -n staging $POD -- curl -f http://localhost:8080/health || exit 1
only:
- develop
when: manual
stop:staging:
stage: deploy
image: bitnami/kubectl:latest
environment:
name: staging
action: stop
script:
- kubectl scale deployment/myapp --replicas=0 --namespace=staging
when: manual
only:
- develop
deploy:production:
stage: deploy
image: bitnami/kubectl:latest
needs: [build, trivy:scan, sign:image]
environment:
name: production
url: https://example.com
before_script:
- kubectl config use-context production-cluster
script:
- |
echo "Deploying version $IMAGE_TAG to production"
- kubectl set image deployment/myapp myapp=$IMAGE_FULL_NAME --namespace=production --record
- kubectl rollout status deployment/myapp --namespace=production --timeout=5m
# Health check
- sleep 10
- |
for i in {1..10}; do
POD=$(kubectl get pod -n production -l app=myapp -o jsonpath="{.items[0].metadata.name}")
if kubectl exec -n production $POD -- curl -f http://localhost:8080/health; then
echo "Health check passed"
exit 0
fi
echo "Attempt $i failed, retrying..."
sleep 10
done
echo "Health check failed"
exit 1
only:
- main
when: manual # Require manual approval for production
# Create GitLab release
release:
stage: deploy
image: registry.gitlab.com/gitlab-org/release-cli:latest
needs: [deploy:production]
script:
- echo "Creating release for $CI_COMMIT_TAG"
release:
tag_name: $CI_COMMIT_TAG
description: |
Docker Image: $IMAGE_FULL_NAME
Changes in this release:
$CI_COMMIT_MESSAGE
only:
- tags
# Cleanup old images from Container Registry
cleanup:registry:
stage: deploy
image: alpine:latest
before_script:
- apk add --no-cache curl jq
script:
- |
# Keep last 10 images, delete older ones
echo "Cleaning up old container images..."
# Use GitLab API to manage container registry
# This is a placeholder - implement based on your retention policy
only:
- schedules
when: manual
# Workflow rules
workflow:
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: '$CI_COMMIT_BRANCH == "main"'
- if: '$CI_COMMIT_BRANCH == "develop"'
- if: '$CI_COMMIT_TAG'
- if: '$CI_PIPELINE_SOURCE == "schedule"'
# Additional optimizations
.interruptible_jobs:
interruptible: true
build:
extends: .interruptible_jobs
trivy:scan:
extends: .interruptible_jobs

548
assets/templates/gitlab-ci/go-ci.yml Normal file
View File

@@ -0,0 +1,548 @@
# GitLab CI/CD Pipeline for Go
# Optimized with caching, parallel execution, and deployment
stages:
- security
- validate
- test
- build
- deploy
variables:
GO_VERSION: "1.22"
GOPATH: "$CI_PROJECT_DIR/.go"
GOCACHE: "$CI_PROJECT_DIR/.cache/go-build"
# Global cache configuration
cache:
key:
files:
- go.mod
- go.sum
paths:
- .go/pkg/mod/
- .cache/go-build/
policy: pull
# Reusable configuration
.go_template:
image: golang:${GO_VERSION}
before_script:
- go version
- go env
- mkdir -p .go
- export PATH=$GOPATH/bin:$PATH
# Validation stage
lint:
extends: .go_template
stage: validate
cache:
key:
files:
- go.mod
- go.sum
paths:
- .go/pkg/mod/
- .cache/go-build/
policy: pull-push
script:
# Install golangci-lint
- curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin
- golangci-lint run --timeout=5m
format-check:
extends: .go_template
stage: validate
script:
- test -z "$(gofmt -s -l .)"
- go mod tidy
- git diff --exit-code go.mod go.sum
only:
- merge_requests
- main
- develop
# Security: Secret Scanning
secret-scan:trufflehog:
stage: security
image: trufflesecurity/trufflehog:latest
script:
- trufflehog filesystem . --json --fail > trufflehog-report.json || true
- |
if [ -s trufflehog-report.json ]; then
echo "❌ Secrets detected!"
cat trufflehog-report.json
exit 1
fi
artifacts:
when: always
paths:
- trufflehog-report.json
expire_in: 30 days
allow_failure: false
only:
- merge_requests
- main
- develop
secret-scan:gitleaks:
stage: security
image: zricethezav/gitleaks:latest
script:
- gitleaks detect --source . --report-format json --report-path gitleaks-report.json
artifacts:
when: always
paths:
- gitleaks-report.json
expire_in: 30 days
allow_failure: true
only:
- merge_requests
- main
- develop
# Security: SAST
sast:semgrep:
stage: security
image: returntocorp/semgrep
script:
- semgrep scan --config=auto --sarif --output=semgrep.sarif .
- semgrep scan --config=p/owasp-top-ten --json --output=semgrep-owasp.json .
artifacts:
reports:
sast: semgrep.sarif
paths:
- semgrep.sarif
- semgrep-owasp.json
expire_in: 30 days
allow_failure: false
only:
- merge_requests
- main
- develop
security:gosec:
extends: .go_template
stage: security
script:
- go install github.com/securego/gosec/v2/cmd/gosec@latest
- gosec -fmt json -out gosec-report.json ./... || true
- gosec ./... # Fail on findings
artifacts:
when: always
paths:
- gosec-report.json
expire_in: 30 days
allow_failure: false
only:
- merge_requests
- main
- develop
security:govulncheck:
extends: .go_template
stage: security
script:
- go install golang.org/x/vuln/cmd/govulncheck@latest
- govulncheck ./...
allow_failure: false
only:
- merge_requests
- main
- develop
# Test stage with matrix
test:
extends: .go_template
stage: test
parallel:
matrix:
- GO_VERSION: ["1.21", "1.22"]
services:
- postgres:15
- redis:7-alpine
variables:
POSTGRES_DB: testdb
POSTGRES_USER: testuser
POSTGRES_PASSWORD: testpass
DATABASE_URL: "postgresql://testuser:testpass@postgres:5432/testdb?sslmode=disable"
REDIS_URL: "redis://redis:6379"
script:
- go mod download
- go test -v -race -coverprofile=coverage.out -covermode=atomic ./...
- go tool cover -func=coverage.out
coverage: '/total:.*?(\d+\.\d+)%/'
artifacts:
when: always
reports:
coverage_report:
coverage_format: cobertura
path: coverage.out
paths:
- coverage.out
expire_in: 30 days
benchmark:
extends: .go_template
stage: test
script:
- go test -bench=. -benchmem ./... | tee benchmark.txt
artifacts:
paths:
- benchmark.txt
expire_in: 7 days
only:
- main
- merge_requests
# Build stage - multi-platform
build:linux:amd64:
extends: .go_template
stage: build
variables:
GOOS: linux
GOARCH: amd64
CGO_ENABLED: "0"
script:
- |
go build \
-ldflags="-s -w -X main.version=$CI_COMMIT_SHORT_SHA -X main.buildTime=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
-o myapp-linux-amd64 \
./cmd/myapp
- ls -lh myapp-linux-amd64
artifacts:
paths:
- myapp-linux-amd64
expire_in: 7 days
only:
- main
- develop
- tags
build:linux:arm64:
extends: .go_template
stage: build
variables:
GOOS: linux
GOARCH: arm64
CGO_ENABLED: "0"
script:
- |
go build \
-ldflags="-s -w -X main.version=$CI_COMMIT_SHORT_SHA -X main.buildTime=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
-o myapp-linux-arm64 \
./cmd/myapp
- ls -lh myapp-linux-arm64
artifacts:
paths:
- myapp-linux-arm64
expire_in: 7 days
only:
- main
- develop
- tags
build:darwin:amd64:
extends: .go_template
stage: build
variables:
GOOS: darwin
GOARCH: amd64
CGO_ENABLED: "0"
script:
- |
go build \
-ldflags="-s -w -X main.version=$CI_COMMIT_SHORT_SHA -X main.buildTime=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
-o myapp-darwin-amd64 \
./cmd/myapp
- ls -lh myapp-darwin-amd64
artifacts:
paths:
- myapp-darwin-amd64
expire_in: 7 days
only:
- tags
build:darwin:arm64:
extends: .go_template
stage: build
variables:
GOOS: darwin
GOARCH: arm64
CGO_ENABLED: "0"
script:
- |
go build \
-ldflags="-s -w -X main.version=$CI_COMMIT_SHORT_SHA -X main.buildTime=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
-o myapp-darwin-arm64 \
./cmd/myapp
- ls -lh myapp-darwin-arm64
artifacts:
paths:
- myapp-darwin-arm64
expire_in: 7 days
only:
- tags
build:windows:amd64:
extends: .go_template
stage: build
variables:
GOOS: windows
GOARCH: amd64
CGO_ENABLED: "0"
script:
- |
go build \
-ldflags="-s -w -X main.version=$CI_COMMIT_SHORT_SHA -X main.buildTime=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
-o myapp-windows-amd64.exe \
./cmd/myapp
- ls -lh myapp-windows-amd64.exe
artifacts:
paths:
- myapp-windows-amd64.exe
expire_in: 7 days
only:
- tags
# Build Docker image
build:docker:
stage: build
image: docker:24-cli
services:
- docker:24-dind
variables:
IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
before_script:
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
script:
# Multi-stage build with Go
- docker pull $CI_REGISTRY_IMAGE:latest || true
- |
docker build \
--cache-from $CI_REGISTRY_IMAGE:latest \
--tag $IMAGE_TAG \
--tag $CI_REGISTRY_IMAGE:latest \
--build-arg GO_VERSION=$GO_VERSION \
--build-arg VERSION=$CI_COMMIT_SHORT_SHA \
--build-arg BUILD_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ) \
.
- docker push $IMAGE_TAG
- docker push $CI_REGISTRY_IMAGE:latest
- echo "IMAGE_FULL_NAME=$IMAGE_TAG" >> build.env
artifacts:
reports:
dotenv: build.env
only:
- main
- develop
- tags
tags:
- docker
# Integration tests
integration-test:
extends: .go_template
stage: test
needs: [build:linux:amd64]
dependencies:
- build:linux:amd64
services:
- postgres:15
variables:
POSTGRES_DB: testdb
POSTGRES_USER: testuser
POSTGRES_PASSWORD: testpass
DATABASE_URL: "postgresql://testuser:testpass@postgres:5432/testdb?sslmode=disable"
BINARY_PATH: "./myapp-linux-amd64"
script:
- chmod +x myapp-linux-amd64
- go test -v -tags=integration ./tests/integration/...
only:
- main
- merge_requests
# Deploy to staging
deploy:staging:
stage: deploy
image: bitnami/kubectl:latest
needs: [build:docker]
environment:
name: staging
url: https://staging.example.com
on_stop: stop:staging
before_script:
- kubectl config use-context staging-cluster
script:
- kubectl set image deployment/myapp myapp=$IMAGE_FULL_NAME --namespace=staging --record
- kubectl rollout status deployment/myapp --namespace=staging --timeout=5m
# Health check
- |
POD=$(kubectl get pod -n staging -l app=myapp -o jsonpath="{.items[0].metadata.name}")
kubectl exec -n staging $POD -- /myapp version
kubectl exec -n staging $POD -- curl -f http://localhost:8080/health || exit 1
only:
- develop
when: manual
stop:staging:
stage: deploy
image: bitnami/kubectl:latest
environment:
name: staging
action: stop
script:
- kubectl scale deployment/myapp --replicas=0 --namespace=staging
when: manual
only:
- develop
# Deploy to production
deploy:production:
stage: deploy
image: bitnami/kubectl:latest
needs: [build:docker, integration-test]
environment:
name: production
url: https://api.example.com
before_script:
- kubectl config use-context production-cluster
script:
- echo "Deploying to production..."
- kubectl set image deployment/myapp myapp=$IMAGE_FULL_NAME --namespace=production --record
- kubectl rollout status deployment/myapp --namespace=production --timeout=5m
# Health check
- sleep 10
- |
for i in {1..10}; do
POD=$(kubectl get pod -n production -l app=myapp -o jsonpath="{.items[0].metadata.name}")
if kubectl exec -n production $POD -- curl -f http://localhost:8080/health; then
echo "Health check passed"
exit 0
fi
echo "Attempt $i failed, retrying..."
sleep 10
done
echo "Health check failed"
exit 1
only:
- main
when: manual
# Deploy to Cloud Run
deploy:cloudrun:
stage: deploy
image: google/cloud-sdk:alpine
needs: [build:docker]
environment:
name: production
url: https://myapp.run.app
before_script:
- echo $GCP_SERVICE_KEY | base64 -d > ${HOME}/gcp-key.json
- gcloud auth activate-service-account --key-file ${HOME}/gcp-key.json
- gcloud config set project $GCP_PROJECT_ID
script:
- |
gcloud run deploy myapp \
--image $IMAGE_FULL_NAME \
--region us-central1 \
--platform managed \
--allow-unauthenticated \
--memory 512Mi \
--cpu 1 \
--max-instances 10
# Health check
- |
URL=$(gcloud run services describe myapp --region us-central1 --format 'value(status.url)')
curl -f $URL/health || exit 1
only:
- main
when: manual
# Create release
release:
stage: deploy
image: registry.gitlab.com/gitlab-org/release-cli:latest
needs:
- build:linux:amd64
- build:linux:arm64
- build:darwin:amd64
- build:darwin:arm64
- build:windows:amd64
dependencies:
- build:linux:amd64
- build:linux:arm64
- build:darwin:amd64
- build:darwin:arm64
- build:windows:amd64
before_script:
- apk add --no-cache coreutils
script:
# Create checksums
- sha256sum myapp-* > checksums.txt
- cat checksums.txt
release:
tag_name: $CI_COMMIT_TAG
description: |
Go Binary Release
Binaries for multiple platforms:
- Linux (amd64, arm64)
- macOS (amd64, arm64/Apple Silicon)
- Windows (amd64)
Docker Image: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
Checksums: See attached checksums.txt
assets:
links:
- name: 'Linux AMD64'
url: '${CI_PROJECT_URL}/-/jobs/artifacts/${CI_COMMIT_TAG}/raw/myapp-linux-amd64?job=build:linux:amd64'
- name: 'Linux ARM64'
url: '${CI_PROJECT_URL}/-/jobs/artifacts/${CI_COMMIT_TAG}/raw/myapp-linux-arm64?job=build:linux:arm64'
- name: 'macOS AMD64'
url: '${CI_PROJECT_URL}/-/jobs/artifacts/${CI_COMMIT_TAG}/raw/myapp-darwin-amd64?job=build:darwin:amd64'
- name: 'macOS ARM64'
url: '${CI_PROJECT_URL}/-/jobs/artifacts/${CI_COMMIT_TAG}/raw/myapp-darwin-arm64?job=build:darwin:arm64'
- name: 'Windows AMD64'
url: '${CI_PROJECT_URL}/-/jobs/artifacts/${CI_COMMIT_TAG}/raw/myapp-windows-amd64.exe?job=build:windows:amd64'
- name: 'Checksums'
url: '${CI_PROJECT_URL}/-/jobs/artifacts/${CI_COMMIT_TAG}/raw/checksums.txt?job=release'
only:
- tags
# GitLab built-in security templates
include:
- template: Security/Dependency-Scanning.gitlab-ci.yml
- template: Security/SAST.gitlab-ci.yml
# Override GitLab template stages
dependency_scanning:
stage: security
sast:
stage: security
# Workflow rules
workflow:
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: '$CI_COMMIT_BRANCH == "main"'
- if: '$CI_COMMIT_BRANCH == "develop"'
- if: '$CI_COMMIT_TAG'
# Interruptible jobs
.interruptible_template:
interruptible: true
lint:
extends: [.go_template, .interruptible_template]
test:
extends: [.go_template, .interruptible_template]
integration-test:
extends: [.go_template, .interruptible_template]

334
assets/templates/gitlab-ci/node-ci.yml Normal file
View File

@@ -0,0 +1,334 @@
# GitLab CI/CD Pipeline for Node.js
# Optimized with caching, parallel execution, and deployment
stages:
- security
- validate
- test
- build
- deploy
# Global variables
variables:
NODE_VERSION: "20"
npm_config_cache: "$CI_PROJECT_DIR/.npm"
CYPRESS_CACHE_FOLDER: "$CI_PROJECT_DIR/.cache/Cypress"
# Global cache configuration
cache:
key:
files:
- package-lock.json
paths:
- node_modules/
- .npm/
- .cache/Cypress/
policy: pull
# Reusable configuration
.node_template:
image: node:${NODE_VERSION}
before_script:
- node --version
- npm --version
- npm ci
# Validation stage
lint:
extends: .node_template
stage: validate
cache:
key:
files:
- package-lock.json
paths:
- node_modules/
- .npm/
policy: pull-push
script:
- npm run lint
- npm run format:check
only:
- merge_requests
- main
- develop
# Test stage
unit-test:
extends: .node_template
stage: test
parallel:
matrix:
- NODE_VERSION: ["18", "20", "22"]
script:
- npm run test:unit -- --coverage
coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
artifacts:
reports:
coverage_report:
coverage_format: cobertura
path: coverage/cobertura-coverage.xml
paths:
- coverage/
expire_in: 30 days
integration-test:
extends: .node_template
stage: test
services:
- postgres:15
- redis:7-alpine
variables:
POSTGRES_DB: testdb
POSTGRES_USER: testuser
POSTGRES_PASSWORD: testpass
DATABASE_URL: "postgresql://testuser:testpass@postgres:5432/testdb"
REDIS_URL: "redis://redis:6379"
script:
- npm run test:integration
artifacts:
when: always
reports:
junit: test-results/junit.xml
paths:
- test-results/
expire_in: 7 days
# Build stage
build:
extends: .node_template
stage: build
cache:
key:
files:
- package-lock.json
paths:
- node_modules/
- .npm/
policy: pull
script:
- npm run build
- echo "BUILD_VERSION=$(node -p "require('./package.json').version")" >> build.env
artifacts:
paths:
- dist/
reports:
dotenv: build.env
expire_in: 7 days
only:
- main
- develop
- tags
# Security: Secret Scanning
secret-scan:trufflehog:
stage: security
image: trufflesecurity/trufflehog:latest
script:
- trufflehog filesystem . --json --fail > trufflehog-report.json || true
- |
if [ -s trufflehog-report.json ]; then
echo "❌ Secrets detected!"
cat trufflehog-report.json
exit 1
fi
artifacts:
when: always
paths:
- trufflehog-report.json
expire_in: 30 days
allow_failure: false
only:
- merge_requests
- main
- develop
secret-scan:gitleaks:
stage: security
image: zricethezav/gitleaks:latest
script:
- gitleaks detect --source . --report-format json --report-path gitleaks-report.json
artifacts:
when: always
paths:
- gitleaks-report.json
expire_in: 30 days
allow_failure: true
only:
- merge_requests
- main
- develop
# Security: SAST
sast:semgrep:
stage: security
image: returntocorp/semgrep
script:
- semgrep scan --config=auto --sarif --output=semgrep.sarif .
- semgrep scan --config=p/owasp-top-ten --json --output=semgrep-owasp.json .
artifacts:
reports:
sast: semgrep.sarif
paths:
- semgrep.sarif
- semgrep-owasp.json
expire_in: 30 days
allow_failure: false
only:
- merge_requests
- main
- develop
sast:nodejs:
stage: security
image: node:20-alpine
script:
- npm install -g eslint eslint-plugin-security
- eslint . --plugin=security --format=json --output-file=eslint-security.json || true
artifacts:
paths:
- eslint-security.json
expire_in: 30 days
only:
exists:
- package.json
allow_failure: true
# Security: Dependency Scanning
dependency-scan:npm:
extends: .node_template
stage: security
cache:
key:
files:
- package-lock.json
paths:
- node_modules/
- .npm/
policy: pull-push
script:
- npm audit --audit-level=moderate --json > npm-audit.json || true
- npm audit --audit-level=high # Fail on high severity
artifacts:
paths:
- npm-audit.json
expire_in: 30 days
allow_failure: false
only:
- merge_requests
- main
- develop
# GitLab built-in security templates
include:
- template: Security/SAST.gitlab-ci.yml
- template: Security/Dependency-Scanning.gitlab-ci.yml
# E2E tests (only on main)
e2e-test:
extends: .node_template
stage: test
needs: [build]
dependencies:
- build
script:
- npm run test:e2e
artifacts:
when: always
paths:
- cypress/videos/
- cypress/screenshots/
expire_in: 7 days
only:
- main
# Deploy to staging
deploy:staging:
stage: deploy
image: node:${NODE_VERSION}
needs: [build]
dependencies:
- build
environment:
name: staging
url: https://staging.example.com
on_stop: stop:staging
script:
- echo "Deploying to staging..."
- npm install -g aws-cli
- aws s3 sync dist/ s3://${STAGING_BUCKET}
- aws cloudfront create-invalidation --distribution-id ${STAGING_CF_DIST} --paths "/*"
only:
- develop
when: manual
stop:staging:
stage: deploy
image: node:${NODE_VERSION}
environment:
name: staging
action: stop
script:
- echo "Stopping staging environment..."
when: manual
only:
- develop
# Deploy to production
deploy:production:
stage: deploy
image: node:${NODE_VERSION}
needs: [build, e2e-test]
dependencies:
- build
environment:
name: production
url: https://example.com
before_script:
- echo "Deploying version ${BUILD_VERSION} to production"
script:
- npm install -g aws-cli
- aws s3 sync dist/ s3://${PRODUCTION_BUCKET}
- aws cloudfront create-invalidation --distribution-id ${PRODUCTION_CF_DIST} --paths "/*"
# Health check
- sleep 10
- curl -f https://example.com/health || exit 1
after_script:
- echo "Deployed successfully"
only:
- main
when: manual
# Create release
release:
stage: deploy
image: registry.gitlab.com/gitlab-org/release-cli:latest
needs: [deploy:production]
script:
- echo "Creating release for version ${BUILD_VERSION}"
release:
tag_name: 'v${BUILD_VERSION}'
description: 'Release v${BUILD_VERSION}'
only:
- main
# Workflow rules
workflow:
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: '$CI_COMMIT_BRANCH == "main"'
- if: '$CI_COMMIT_BRANCH == "develop"'
- if: '$CI_COMMIT_TAG'
# Additional optimizations
.interruptible_template:
interruptible: true
lint:
extends: [.node_template, .interruptible_template]
unit-test:
extends: [.node_template, .interruptible_template]
integration-test:
extends: [.node_template, .interruptible_template]

472
assets/templates/gitlab-ci/python-ci.yml Normal file
View File

@@ -0,0 +1,472 @@
# GitLab CI/CD Pipeline for Python
# Optimized with caching, parallel execution, and deployment
stages:
- security
- validate
- test
- build
- deploy
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
PYTHON_VERSION: "3.11"
# Global cache configuration
cache:
key:
files:
- requirements.txt
- requirements-dev.txt
paths:
- .cache/pip
- .venv/
policy: pull
# Reusable configuration
.python_template:
image: python:${PYTHON_VERSION}
before_script:
- python --version
- pip install --upgrade pip
- python -m venv .venv
- source .venv/bin/activate
- pip install -r requirements.txt
- pip install -r requirements-dev.txt
# Validation stage
lint:
extends: .python_template
stage: validate
cache:
key:
files:
- requirements.txt
- requirements-dev.txt
paths:
- .cache/pip
- .venv/
policy: pull-push
script:
- ruff check .
- black --check .
- isort --check-only .
- mypy . || true # Don't fail on type errors initially
only:
- merge_requests
- main
- develop
# Security: Secret Scanning
secret-scan:trufflehog:
stage: security
image: trufflesecurity/trufflehog:latest
script:
- trufflehog filesystem . --json --fail > trufflehog-report.json || true
- |
if [ -s trufflehog-report.json ]; then
echo "❌ Secrets detected!"
cat trufflehog-report.json
exit 1
fi
artifacts:
when: always
paths:
- trufflehog-report.json
expire_in: 30 days
allow_failure: false
only:
- merge_requests
- main
- develop
secret-scan:gitleaks:
stage: security
image: zricethezav/gitleaks:latest
script:
- gitleaks detect --source . --report-format json --report-path gitleaks-report.json
artifacts:
when: always
paths:
- gitleaks-report.json
expire_in: 30 days
allow_failure: true
only:
- merge_requests
- main
- develop
# Security: SAST
sast:semgrep:
stage: security
image: returntocorp/semgrep
script:
- semgrep scan --config=auto --sarif --output=semgrep.sarif .
- semgrep scan --config=p/owasp-top-ten --json --output=semgrep-owasp.json .
artifacts:
reports:
sast: semgrep.sarif
paths:
- semgrep.sarif
- semgrep-owasp.json
expire_in: 30 days
allow_failure: false
only:
- merge_requests
- main
- develop
security:bandit:
extends: .python_template
stage: security
script:
- pip install bandit
- bandit -r src/ -f json -o bandit-report.json -ll || true
- bandit -r src/ -ll # Fail on high severity
artifacts:
when: always
paths:
- bandit-report.json
expire_in: 30 days
allow_failure: false
only:
- merge_requests
- main
- develop
security:safety:
extends: .python_template
stage: security
script:
- pip install safety
- safety check --json --output safety-report.json || true
- safety check
artifacts:
when: always
paths:
- safety-report.json
expire_in: 30 days
allow_failure: true
only:
- merge_requests
- main
- develop
# Security: Dependency Scanning
security:pip-audit:
extends: .python_template
stage: security
script:
- pip install pip-audit
- pip-audit --requirement requirements.txt --format json --output pip-audit.json || true
- pip-audit --requirement requirements.txt # Fail on vulnerabilities
artifacts:
when: always
paths:
- pip-audit.json
expire_in: 30 days
allow_failure: false
only:
- merge_requests
- main
- develop
# Test stage with matrix
test:
extends: .python_template
stage: test
parallel:
matrix:
- PYTHON_VERSION: ["3.9", "3.10", "3.11", "3.12"]
services:
- postgres:15
- redis:7-alpine
variables:
POSTGRES_DB: testdb
POSTGRES_USER: testuser
POSTGRES_PASSWORD: testpass
DATABASE_URL: "postgresql://testuser:testpass@postgres:5432/testdb"
REDIS_URL: "redis://redis:6379"
script:
- |
pytest tests/unit \
--cov=src \
--cov-report=xml \
--cov-report=term \
--cov-report=html \
--junitxml=junit.xml \
-v
coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
artifacts:
when: always
reports:
junit: junit.xml
coverage_report:
coverage_format: cobertura
path: coverage.xml
paths:
- coverage.xml
- htmlcov/
expire_in: 30 days
integration-test:
extends: .python_template
stage: test
services:
- postgres:15
- redis:7-alpine
variables:
POSTGRES_DB: testdb
POSTGRES_USER: testuser
POSTGRES_PASSWORD: testpass
DATABASE_URL: "postgresql://testuser:testpass@postgres:5432/testdb"
REDIS_URL: "redis://redis:6379"
script:
- pytest tests/integration -v --junitxml=junit-integration.xml
artifacts:
when: always
reports:
junit: junit-integration.xml
expire_in: 7 days
only:
- main
- develop
- merge_requests
# Build stage
build:package:
extends: .python_template
stage: build
script:
- pip install build wheel setuptools
- python -m build
- ls -lh dist/
artifacts:
paths:
- dist/
expire_in: 7 days
only:
- main
- develop
- tags
build:docker:
stage: build
image: docker:24-cli
services:
- docker:24-dind
variables:
IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
before_script:
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
script:
# Pull previous image for caching
- docker pull $CI_REGISTRY_IMAGE:latest || true
# Build with cache
- |
docker build \
--cache-from $CI_REGISTRY_IMAGE:latest \
--tag $IMAGE_TAG \
--tag $CI_REGISTRY_IMAGE:latest \
--build-arg BUILDKIT_INLINE_CACHE=1 \
.
# Push images
- docker push $IMAGE_TAG
- docker push $CI_REGISTRY_IMAGE:latest
- echo "IMAGE_FULL_NAME=$IMAGE_TAG" >> build.env
artifacts:
reports:
dotenv: build.env
only:
- main
- develop
- tags
tags:
- docker
# E2E tests (only on main)
e2e-test:
extends: .python_template
stage: test
needs: [build:package]
dependencies:
- build:package
script:
- pip install dist/*.whl
- pytest tests/e2e -v
artifacts:
when: always
paths:
- test-results/
expire_in: 7 days
only:
- main
# Deploy to PyPI
deploy:pypi:
stage: deploy
image: python:3.11
needs: [build:package, test]
dependencies:
- build:package
environment:
name: pypi
url: https://pypi.org/project/your-package
before_script:
- pip install twine
script:
- twine check dist/*
- twine upload dist/* --username __token__ --password $PYPI_TOKEN
only:
- tags
when: manual
# Deploy Docker to staging
deploy:staging:
stage: deploy
image: bitnami/kubectl:latest
needs: [build:docker]
environment:
name: staging
url: https://staging.example.com
on_stop: stop:staging
before_script:
- kubectl config use-context staging-cluster
script:
- kubectl set image deployment/myapp myapp=$IMAGE_FULL_NAME --namespace=staging --record
- kubectl rollout status deployment/myapp --namespace=staging --timeout=5m
# Smoke test
- |
POD=$(kubectl get pod -n staging -l app=myapp -o jsonpath="{.items[0].metadata.name}")
kubectl exec -n staging $POD -- python -c "import sys; print(sys.version)"
kubectl exec -n staging $POD -- curl -f http://localhost:8000/health || exit 1
only:
- develop
when: manual
stop:staging:
stage: deploy
image: bitnami/kubectl:latest
environment:
name: staging
action: stop
script:
- kubectl scale deployment/myapp --replicas=0 --namespace=staging
when: manual
only:
- develop
# Deploy to production
deploy:production:
stage: deploy
image: bitnami/kubectl:latest
needs: [build:docker, e2e-test]
environment:
name: production
url: https://example.com
before_script:
- kubectl config use-context production-cluster
script:
- echo "Deploying to production..."
- kubectl set image deployment/myapp myapp=$IMAGE_FULL_NAME --namespace=production --record
- kubectl rollout status deployment/myapp --namespace=production --timeout=5m
# Health check
- sleep 10
- |
for i in {1..10}; do
POD=$(kubectl get pod -n production -l app=myapp -o jsonpath="{.items[0].metadata.name}")
if kubectl exec -n production $POD -- curl -f http://localhost:8000/health; then
echo "Health check passed"
exit 0
fi
echo "Attempt $i failed, retrying..."
sleep 10
done
echo "Health check failed"
exit 1
only:
- main
when: manual
# Deploy to Cloud Run (Google Cloud)
deploy:cloudrun:
stage: deploy
image: google/cloud-sdk:alpine
needs: [build:docker]
environment:
name: production
url: https://your-app.run.app
before_script:
- echo $GCP_SERVICE_KEY | base64 -d > ${HOME}/gcp-key.json
- gcloud auth activate-service-account --key-file ${HOME}/gcp-key.json
- gcloud config set project $GCP_PROJECT_ID
script:
- |
gcloud run deploy your-app \
--image $IMAGE_FULL_NAME \
--region us-central1 \
--platform managed \
--allow-unauthenticated
# Health check
- |
URL=$(gcloud run services describe your-app --region us-central1 --format 'value(status.url)')
curl -f $URL/health || exit 1
only:
- main
when: manual
# Create release
release:
stage: deploy
image: registry.gitlab.com/gitlab-org/release-cli:latest
needs: [deploy:production]
script:
- echo "Creating release"
release:
tag_name: $CI_COMMIT_TAG
description: |
Python Package: https://pypi.org/project/your-package/$CI_COMMIT_TAG
Docker Image: $IMAGE_FULL_NAME
Changes in this release:
$CI_COMMIT_MESSAGE
only:
- tags
# GitLab built-in security templates
include:
- template: Security/Dependency-Scanning.gitlab-ci.yml
- template: Security/SAST.gitlab-ci.yml
# Override GitLab template stages
dependency_scanning:
stage: security
sast:
stage: security
# Workflow rules
workflow:
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: '$CI_COMMIT_BRANCH == "main"'
- if: '$CI_COMMIT_BRANCH == "develop"'
- if: '$CI_COMMIT_TAG'
# Interruptible jobs
.interruptible_template:
interruptible: true
lint:
extends: [.python_template, .interruptible_template]
test:
extends: [.python_template, .interruptible_template]
integration-test:
extends: [.python_template, .interruptible_template]

479
assets/templates/gitlab-ci/security-scan.yml Normal file
View File

@@ -0,0 +1,479 @@
# Complete DevSecOps Security Scanning Pipeline for GitLab CI
# SAST, DAST, SCA, Container Scanning, Secret Scanning
stages:
- secret-scan
- sast
- sca
- build
- container-scan
- dast
- compliance
- report
variables:
SECURE_LOG_LEVEL: "info"
# Enable Auto DevOps security scanners
SAST_EXCLUDED_PATHS: "spec, test, tests, tmp"
SCAN_KUBERNETES_MANIFESTS: "false"
# Stage 1: Secret Scanning
secret-scan:trufflehog:
stage: secret-scan
image: trufflesecurity/trufflehog:latest
script:
- trufflehog filesystem . --json --fail > trufflehog-report.json || true
- |
if [ -s trufflehog-report.json ]; then
echo "❌ Secrets detected!"
cat trufflehog-report.json
exit 1
fi
artifacts:
when: always
paths:
- trufflehog-report.json
expire_in: 30 days
allow_failure: false
secret-scan:gitleaks:
stage: secret-scan
image: zricethezav/gitleaks:latest
script:
- gitleaks detect --source . --report-format json --report-path gitleaks-report.json
artifacts:
when: always
paths:
- gitleaks-report.json
expire_in: 30 days
allow_failure: true
# Stage 2: SAST (Static Application Security Testing)
sast:semgrep:
stage: sast
image: returntocorp/semgrep
script:
- semgrep scan --config=auto --sarif --output=semgrep.sarif .
- semgrep scan --config=p/owasp-top-ten --json --output=semgrep-owasp.json .
artifacts:
reports:
sast: semgrep.sarif
paths:
- semgrep.sarif
- semgrep-owasp.json
expire_in: 30 days
allow_failure: false
sast:nodejs:
stage: sast
image: node:20-alpine
script:
- npm install -g eslint eslint-plugin-security
- eslint . --plugin=security --format=json --output-file=eslint-security.json || true
artifacts:
paths:
- eslint-security.json
expire_in: 30 days
only:
exists:
- package.json
allow_failure: true
sast:python:
stage: sast
image: python:3.11-alpine
script:
- pip install bandit
- bandit -r . -f json -o bandit-report.json -ll || true
- bandit -r . -ll # Fail on high severity
artifacts:
reports:
sast: bandit-report.json
paths:
- bandit-report.json
expire_in: 30 days
only:
exists:
- requirements.txt
allow_failure: false
sast:go:
stage: sast
image: securego/gosec:latest
script:
- gosec -fmt json -out gosec-report.json ./... || true
- gosec ./... # Fail on findings
artifacts:
reports:
sast: gosec-report.json
paths:
- gosec-report.json
expire_in: 30 days
only:
exists:
- go.mod
allow_failure: false
# GitLab built-in SAST
include:
- template: Security/SAST.gitlab-ci.yml
sast:
variables:
SAST_EXCLUDED_ANALYZERS: ""
# Stage 3: SCA (Software Composition Analysis)
sca:npm-audit:
stage: sca
image: node:20-alpine
before_script:
- npm ci
script:
- npm audit --audit-level=moderate --json > npm-audit.json || true
- npm audit --audit-level=high # Fail on high severity
artifacts:
paths:
- npm-audit.json
expire_in: 30 days
only:
exists:
- package.json
allow_failure: false
sca:python:
stage: sca
image: python:3.11-alpine
script:
- pip install pip-audit
- pip-audit --requirement requirements.txt --format json --output pip-audit.json || true
- pip-audit --requirement requirements.txt --vulnerability-service osv # Fail on vulns
artifacts:
paths:
- pip-audit.json
expire_in: 30 days
only:
exists:
- requirements.txt
allow_failure: false
# GitLab built-in Dependency Scanning
include:
- template: Security/Dependency-Scanning.gitlab-ci.yml
dependency_scanning:
variables:
DS_EXCLUDED_PATHS: "test/,tests/,spec/,vendor/"
# Stage 4: Build
build:app:
stage: build
image: node:20-alpine
script:
- npm ci
- npm run build
artifacts:
paths:
- dist/
expire_in: 7 days
only:
- branches
- merge_requests
build:docker:
stage: build
image: docker:24-cli
services:
- docker:24-dind
variables:
IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
before_script:
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
script:
- docker build --tag $IMAGE_TAG .
- docker push $IMAGE_TAG
- echo "IMAGE_TAG=$IMAGE_TAG" > build.env
artifacts:
reports:
dotenv: build.env
only:
- branches
- merge_requests
# Stage 5: Container Security Scanning
container:trivy:
stage: container-scan
image: aquasec/trivy:latest
needs: [build:docker]
dependencies:
- build:docker
variables:
GIT_STRATEGY: none
script:
# Scan for vulnerabilities
- trivy image --severity HIGH,CRITICAL --format json --output trivy-report.json $IMAGE_TAG
- trivy image --severity HIGH,CRITICAL --exit-code 1 $IMAGE_TAG
artifacts:
reports:
container_scanning: trivy-report.json
paths:
- trivy-report.json
expire_in: 30 days
allow_failure: false
container:grype:
stage: container-scan
image: anchore/grype:latest
needs: [build:docker]
dependencies:
- build:docker
variables:
GIT_STRATEGY: none
before_script:
- echo $CI_REGISTRY_PASSWORD | grype registry login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
script:
- grype $IMAGE_TAG --fail-on high --output json --file grype-report.json
artifacts:
paths:
- grype-report.json
expire_in: 30 days
allow_failure: true
container:sbom:
stage: container-scan
image: anchore/syft:latest
needs: [build:docker]
dependencies:
- build:docker
variables:
GIT_STRATEGY: none
before_script:
- echo $CI_REGISTRY_PASSWORD | syft registry login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
script:
- syft $IMAGE_TAG -o spdx-json > sbom.spdx.json
- syft $IMAGE_TAG -o cyclonedx-json > sbom.cyclonedx.json
artifacts:
paths:
- sbom.spdx.json
- sbom.cyclonedx.json
expire_in: 90 days
only:
- main
- tags
# GitLab built-in Container Scanning
include:
- template: Security/Container-Scanning.gitlab-ci.yml
container_scanning:
needs: [build:docker]
dependencies:
- build:docker
variables:
CS_IMAGE: $IMAGE_TAG
GIT_STRATEGY: none
# Stage 6: DAST (Dynamic Application Security Testing)
dast:zap-baseline:
stage: dast
image: owasp/zap2docker-stable
needs: [build:docker]
services:
- name: $IMAGE_TAG
alias: testapp
script:
# Wait for app to be ready
- sleep 10
# Run baseline scan
- zap-baseline.py -t http://testapp:8080 -r zap-baseline-report.html -J zap-baseline.json
artifacts:
when: always
paths:
- zap-baseline-report.html
- zap-baseline.json
expire_in: 30 days
only:
- main
- schedules
allow_failure: true
dast:zap-full:
stage: dast
image: owasp/zap2docker-stable
script:
# Full scan on staging environment
- zap-full-scan.py -t https://staging.example.com -r zap-full-report.html -J zap-full.json
artifacts:
when: always
paths:
- zap-full-report.html
- zap-full.json
reports:
dast: zap-full.json
expire_in: 30 days
only:
- schedules # Run on schedule only (slow)
allow_failure: true
# GitLab built-in DAST
include:
- template: DAST.gitlab-ci.yml
dast:
variables:
DAST_WEBSITE: https://staging.example.com
DAST_FULL_SCAN_ENABLED: "false"
only:
- schedules
- main
# Stage 7: License Compliance
license:check:
stage: compliance
image: node:20-alpine
needs: [build:app]
script:
- npm ci
- npm install -g license-checker
- license-checker --production --onlyAllow "MIT;Apache-2.0;BSD-2-Clause;BSD-3-Clause;ISC;0BSD" --json --out license-report.json
artifacts:
paths:
- license-report.json
expire_in: 30 days
only:
exists:
- package.json
allow_failure: false
# GitLab built-in License Scanning
include:
- template: Security/License-Scanning.gitlab-ci.yml
license_scanning:
only:
- main
- merge_requests
# Stage 8: Security Report & Gate
security:gate:
stage: report
image: alpine:latest
needs:
- secret-scan:trufflehog
- sast:semgrep
- sca:npm-audit
- container:trivy
- license:check
before_script:
- apk add --no-cache jq curl
script:
- |
echo "==================================="
echo "🔒 Security Gate Evaluation"
echo "==================================="
GATE_PASSED=true
# Check Trivy results
if [ -f trivy-report.json ]; then
CRITICAL=$(jq '[.Results[]?.Vulnerabilities[]? | select(.Severity=="CRITICAL")] | length' trivy-report.json)
HIGH=$(jq '[.Results[]?.Vulnerabilities[]? | select(.Severity=="HIGH")] | length' trivy-report.json)
echo "Container Vulnerabilities:"
echo " - Critical: $CRITICAL"
echo " - High: $HIGH"
if [ "$CRITICAL" -gt 0 ]; then
echo "❌ CRITICAL vulnerabilities found in container"
GATE_PASSED=false
fi
if [ "$HIGH" -gt 10 ]; then
echo "⚠️ Too many HIGH vulnerabilities: $HIGH"
GATE_PASSED=false
fi
fi
# Final gate decision
if [ "$GATE_PASSED" = true ]; then
echo ""
echo "✅ Security gate PASSED"
echo "All security checks completed successfully"
exit 0
else
echo ""
echo "❌ Security gate FAILED"
echo "Critical security issues detected"
exit 1
fi
allow_failure: false
security:report:
stage: report
image: alpine:latest
needs:
- security:gate
when: always
script:
- |
cat << EOF > security-report.md
# Security Scan Report - $(date +%Y-%m-%d)
## Summary
- **Project:** $CI_PROJECT_NAME
- **Branch:** $CI_COMMIT_BRANCH
- **Commit:** $CI_COMMIT_SHORT_SHA
- **Pipeline:** $CI_PIPELINE_URL
- **Scan Date:** $(date -u +"%Y-%m-%d %H:%M:%S UTC")
## Scans Performed
1. ✅ Secret Scanning (TruffleHog, Gitleaks)
2. ✅ SAST (Semgrep, Language-specific)
3. ✅ SCA (npm audit, pip-audit)
4. ✅ Container Scanning (Trivy, Grype)
5. ✅ SBOM Generation (Syft)
6. ✅ License Compliance
## Security Gate
Status: See job logs for details
## Artifacts
- Trivy container scan report
- Semgrep SAST report
- SBOM (SPDX & CycloneDX)
- License compliance report
- ZAP DAST report (if applicable)
## Next Steps
1. Review all security findings in artifacts
2. Address critical and high severity vulnerabilities
3. Update dependencies with known vulnerabilities
4. Re-run pipeline to verify fixes
EOF
cat security-report.md
artifacts:
paths:
- security-report.md
expire_in: 90 days
# Workflow rules
workflow:
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: '$CI_COMMIT_BRANCH == "main"'
- if: '$CI_COMMIT_BRANCH == "develop"'
- if: '$CI_PIPELINE_SOURCE == "schedule"'
- if: '$CI_COMMIT_TAG'
# Interruptible jobs for MR pipelines
.interruptible:
interruptible: true
secret-scan:trufflehog:
extends: .interruptible
sast:semgrep:
extends: .interruptible
sca:npm-audit:
extends: .interruptible