Initial commit

This commit is contained in:
Zhongwei Li
2025-11-29 17:51:12 +08:00
commit 1878d01517
21 changed files with 8728 additions and 0 deletions

301
scripts/ci_health.py Normal file
View File

@@ -0,0 +1,301 @@
#!/usr/bin/env python3
"""
CI/CD Pipeline Health Checker
Checks pipeline status, recent failures, and provides insights for GitHub Actions,
GitLab CI, and other platforms. Identifies failing workflows, slow pipelines,
and provides actionable recommendations.
Usage:
# GitHub Actions
python3 ci_health.py --platform github --repo owner/repo
# GitLab CI
python3 ci_health.py --platform gitlab --project-id 12345 --token <token>
# Check specific workflow/pipeline
python3 ci_health.py --platform github --repo owner/repo --workflow ci.yml
"""
import argparse
import json
import subprocess
import sys
import urllib.request
import urllib.error
from datetime import datetime, timedelta
from typing import Dict, List, Optional
class CIHealthChecker:
def __init__(self, platform: str, **kwargs):
self.platform = platform.lower()
self.config = kwargs
self.issues = []
self.warnings = []
self.insights = []
self.metrics = {}
def check_github_workflows(self) -> Dict:
"""Check GitHub Actions workflow health"""
print(f"🔍 Checking GitHub Actions workflows...")
if not self._check_command("gh"):
self.issues.append("GitHub CLI (gh) is not installed")
self.insights.append("Install gh CLI: https://cli.github.com/")
return self._generate_report()
repo = self.config.get('repo')
if not repo:
self.issues.append("Repository not specified")
self.insights.append("Use --repo owner/repo")
return self._generate_report()
try:
# Get recent workflow runs
limit = self.config.get('limit', 20)
cmd = ['gh', 'run', 'list', '--repo', repo, '--limit', str(limit), '--json',
'status,conclusion,name,workflowName,createdAt,displayTitle']
result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
if result.returncode != 0:
self.issues.append(f"Failed to fetch workflows: {result.stderr}")
self.insights.append("Verify gh CLI authentication: gh auth status")
return self._generate_report()
runs = json.loads(result.stdout)
if not runs:
self.warnings.append("No recent workflow runs found")
return self._generate_report()
# Analyze runs
total_runs = len(runs)
failed_runs = [r for r in runs if r.get('conclusion') == 'failure']
cancelled_runs = [r for r in runs if r.get('conclusion') == 'cancelled']
success_runs = [r for r in runs if r.get('conclusion') == 'success']
self.metrics['total_runs'] = total_runs
self.metrics['failed_runs'] = len(failed_runs)
self.metrics['cancelled_runs'] = len(cancelled_runs)
self.metrics['success_runs'] = len(success_runs)
self.metrics['failure_rate'] = (len(failed_runs) / total_runs * 100) if total_runs > 0 else 0
# Group failures by workflow
failure_by_workflow = {}
for run in failed_runs:
workflow = run.get('workflowName', 'unknown')
failure_by_workflow[workflow] = failure_by_workflow.get(workflow, 0) + 1
print(f"✅ Analyzed {total_runs} recent runs:")
print(f" - Success: {len(success_runs)} ({len(success_runs)/total_runs*100:.1f}%)")
print(f" - Failed: {len(failed_runs)} ({len(failed_runs)/total_runs*100:.1f}%)")
print(f" - Cancelled: {len(cancelled_runs)} ({len(cancelled_runs)/total_runs*100:.1f}%)")
# Identify issues
if self.metrics['failure_rate'] > 20:
self.issues.append(f"High failure rate: {self.metrics['failure_rate']:.1f}%")
self.insights.append("Investigate failing workflows and address root causes")
if failure_by_workflow:
self.warnings.append("Workflows with recent failures:")
for workflow, count in sorted(failure_by_workflow.items(), key=lambda x: x[1], reverse=True):
self.warnings.append(f" - {workflow}: {count} failure(s)")
self.insights.append(f"Review logs for '{workflow}': gh run view --repo {repo}")
if len(cancelled_runs) > total_runs * 0.3:
self.warnings.append(f"High cancellation rate: {len(cancelled_runs)/total_runs*100:.1f}%")
self.insights.append("Excessive cancellations may indicate workflow timeout issues or manual interventions")
except subprocess.TimeoutExpired:
self.issues.append("Request timed out - check network connectivity")
except json.JSONDecodeError as e:
self.issues.append(f"Failed to parse workflow data: {e}")
except Exception as e:
self.issues.append(f"Unexpected error: {e}")
return self._generate_report()
def check_gitlab_pipelines(self) -> Dict:
"""Check GitLab CI pipeline health"""
print(f"🔍 Checking GitLab CI pipelines...")
url = self.config.get('url', 'https://gitlab.com')
token = self.config.get('token')
project_id = self.config.get('project_id')
if not token:
self.issues.append("GitLab token not provided")
self.insights.append("Provide token with --token or GITLAB_TOKEN env var")
return self._generate_report()
if not project_id:
self.issues.append("Project ID not specified")
self.insights.append("Use --project-id <id>")
return self._generate_report()
try:
# Get recent pipelines
per_page = self.config.get('limit', 20)
api_url = f"{url}/api/v4/projects/{project_id}/pipelines?per_page={per_page}"
req = urllib.request.Request(api_url, headers={'PRIVATE-TOKEN': token})
with urllib.request.urlopen(req, timeout=30) as response:
pipelines = json.loads(response.read())
if not pipelines:
self.warnings.append("No recent pipelines found")
return self._generate_report()
# Analyze pipelines
total_pipelines = len(pipelines)
failed = [p for p in pipelines if p.get('status') == 'failed']
success = [p for p in pipelines if p.get('status') == 'success']
running = [p for p in pipelines if p.get('status') == 'running']
cancelled = [p for p in pipelines if p.get('status') == 'canceled']
self.metrics['total_pipelines'] = total_pipelines
self.metrics['failed'] = len(failed)
self.metrics['success'] = len(success)
self.metrics['running'] = len(running)
self.metrics['failure_rate'] = (len(failed) / total_pipelines * 100) if total_pipelines > 0 else 0
print(f"✅ Analyzed {total_pipelines} recent pipelines:")
print(f" - Success: {len(success)} ({len(success)/total_pipelines*100:.1f}%)")
print(f" - Failed: {len(failed)} ({len(failed)/total_pipelines*100:.1f}%)")
print(f" - Running: {len(running)}")
print(f" - Cancelled: {len(cancelled)}")
# Identify issues
if self.metrics['failure_rate'] > 20:
self.issues.append(f"High failure rate: {self.metrics['failure_rate']:.1f}%")
self.insights.append("Review failing pipelines and fix recurring issues")
# Get details of recent failures
if failed:
self.warnings.append(f"Recent pipeline failures:")
for pipeline in failed[:5]: # Show up to 5 recent failures
ref = pipeline.get('ref', 'unknown')
pipeline_id = pipeline.get('id')
self.warnings.append(f" - Pipeline #{pipeline_id} on {ref}")
self.insights.append(f"View pipeline details: {url}/{project_id}/-/pipelines")
except urllib.error.HTTPError as e:
self.issues.append(f"API error: {e.code} - {e.reason}")
if e.code == 401:
self.insights.append("Check GitLab token permissions")
except urllib.error.URLError as e:
self.issues.append(f"Network error: {e.reason}")
self.insights.append("Check GitLab URL and network connectivity")
except Exception as e:
self.issues.append(f"Unexpected error: {e}")
return self._generate_report()
def _check_command(self, command: str) -> bool:
"""Check if command is available"""
try:
subprocess.run([command, '--version'], capture_output=True, timeout=5)
return True
except (FileNotFoundError, subprocess.TimeoutExpired):
return False
def _generate_report(self) -> Dict:
"""Generate health check report"""
# Determine overall health status
if self.issues:
status = 'unhealthy'
elif self.warnings:
status = 'degraded'
else:
status = 'healthy'
return {
'platform': self.platform,
'status': status,
'issues': self.issues,
'warnings': self.warnings,
'insights': self.insights,
'metrics': self.metrics
}
def print_report(report: Dict):
"""Print formatted health check report"""
print("\n" + "="*60)
print(f"🏥 CI/CD Health Report - {report['platform'].upper()}")
print("="*60)
status_emoji = {"healthy": "", "degraded": "⚠️", "unhealthy": ""}.get(report['status'], "")
print(f"\nStatus: {status_emoji} {report['status'].upper()}")
if report['metrics']:
print(f"\n📊 Metrics:")
for key, value in report['metrics'].items():
formatted_key = key.replace('_', ' ').title()
if 'rate' in key:
print(f" - {formatted_key}: {value:.1f}%")
else:
print(f" - {formatted_key}: {value}")
if report['issues']:
print(f"\n🚨 Issues ({len(report['issues'])}):")
for i, issue in enumerate(report['issues'], 1):
print(f" {i}. {issue}")
if report['warnings']:
print(f"\n⚠️ Warnings:")
for warning in report['warnings']:
if warning.startswith(' -'):
print(f" {warning}")
else:
print(f"{warning}")
if report['insights']:
print(f"\n💡 Insights & Recommendations:")
for i, insight in enumerate(report['insights'], 1):
print(f" {i}. {insight}")
print("\n" + "="*60 + "\n")
def main():
parser = argparse.ArgumentParser(
description='CI/CD Pipeline Health Checker',
formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument('--platform', required=True, choices=['github', 'gitlab'],
help='CI/CD platform')
parser.add_argument('--repo', help='GitHub repository (owner/repo)')
parser.add_argument('--workflow', help='Specific workflow name to check')
parser.add_argument('--project-id', help='GitLab project ID')
parser.add_argument('--url', default='https://gitlab.com', help='GitLab URL')
parser.add_argument('--token', help='GitLab token (or use GITLAB_TOKEN env var)')
parser.add_argument('--limit', type=int, default=20, help='Number of recent runs/pipelines to analyze')
args = parser.parse_args()
# Create checker
checker = CIHealthChecker(
platform=args.platform,
repo=args.repo,
workflow=args.workflow,
project_id=args.project_id,
url=args.url,
token=args.token,
limit=args.limit
)
# Run checks
if args.platform == 'github':
report = checker.check_github_workflows()
elif args.platform == 'gitlab':
report = checker.check_gitlab_pipelines()
# Print report
print_report(report)
# Exit with error code if unhealthy
sys.exit(0 if report['status'] == 'healthy' else 1)
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,440 @@
#!/usr/bin/env python3
"""
CI/CD Pipeline Performance Analyzer
Analyzes CI/CD pipeline configuration and execution to identify performance
bottlenecks, caching opportunities, and optimization recommendations.
Usage:
# Analyze GitHub Actions workflow
python3 pipeline_analyzer.py --platform github --workflow .github/workflows/ci.yml
# Analyze GitLab CI pipeline
python3 pipeline_analyzer.py --platform gitlab --config .gitlab-ci.yml
# Analyze recent workflow runs
python3 pipeline_analyzer.py --platform github --repo owner/repo --analyze-runs 10
"""
import argparse
import json
import os
import re
import subprocess
import sys
from pathlib import Path
from typing import Dict, List, Optional, Tuple
import yaml
class PipelineAnalyzer:
def __init__(self, platform: str, **kwargs):
self.platform = platform.lower()
self.config = kwargs
self.findings = []
self.optimizations = []
self.metrics = {}
def analyze_github_workflow(self, workflow_file: str) -> Dict:
"""Analyze GitHub Actions workflow file"""
print(f"🔍 Analyzing GitHub Actions workflow: {workflow_file}")
if not os.path.exists(workflow_file):
return self._error(f"Workflow file not found: {workflow_file}")
try:
with open(workflow_file, 'r') as f:
workflow = yaml.safe_load(f)
# Analyze workflow structure
self._check_workflow_triggers(workflow)
self._check_caching_strategy(workflow, 'github')
self._check_job_parallelization(workflow, 'github')
self._check_dependency_management(workflow, 'github')
self._check_matrix_strategy(workflow)
self._check_artifact_usage(workflow)
self._analyze_action_versions(workflow)
return self._generate_report()
except yaml.YAMLError as e:
return self._error(f"Invalid YAML: {e}")
except Exception as e:
return self._error(f"Analysis failed: {e}")
def analyze_gitlab_pipeline(self, config_file: str) -> Dict:
"""Analyze GitLab CI pipeline configuration"""
print(f"🔍 Analyzing GitLab CI pipeline: {config_file}")
if not os.path.exists(config_file):
return self._error(f"Config file not found: {config_file}")
try:
with open(config_file, 'r') as f:
config = yaml.safe_load(f)
# Analyze pipeline structure
self._check_caching_strategy(config, 'gitlab')
self._check_job_parallelization(config, 'gitlab')
self._check_dependency_management(config, 'gitlab')
self._check_gitlab_specific_features(config)
return self._generate_report()
except yaml.YAMLError as e:
return self._error(f"Invalid YAML: {e}")
except Exception as e:
return self._error(f"Analysis failed: {e}")
def _check_workflow_triggers(self, workflow: Dict):
"""Check workflow trigger configuration"""
triggers = workflow.get('on', {})
if isinstance(triggers, list):
trigger_types = triggers
elif isinstance(triggers, dict):
trigger_types = list(triggers.keys())
else:
trigger_types = [triggers] if triggers else []
# Check for overly broad triggers
if 'push' in trigger_types:
push_config = triggers.get('push', {}) if isinstance(triggers, dict) else {}
if not push_config or not push_config.get('branches'):
self.findings.append("Workflow triggers on all push events (no branch filter)")
self.optimizations.append(
"Add branch filters to 'push' trigger to reduce unnecessary runs:\n"
" on:\n"
" push:\n"
" branches: [main, develop]"
)
# Check for path filters
if 'pull_request' in trigger_types:
pr_config = triggers.get('pull_request', {}) if isinstance(triggers, dict) else {}
if not pr_config.get('paths') and not pr_config.get('paths-ignore'):
self.optimizations.append(
"Consider adding path filters to skip unnecessary PR runs:\n"
" pull_request:\n"
" paths-ignore:\n"
" - 'docs/**'\n"
" - '**.md'"
)
def _check_caching_strategy(self, config: Dict, platform: str):
"""Check for dependency caching"""
has_cache = False
if platform == 'github':
jobs = config.get('jobs', {})
for job_name, job in jobs.items():
steps = job.get('steps', [])
for step in steps:
if isinstance(step, dict) and step.get('uses', '').startswith('actions/cache'):
has_cache = True
break
if not has_cache:
self.findings.append("No dependency caching detected")
self.optimizations.append(
"Add dependency caching to speed up builds:\n"
" - uses: actions/cache@v4\n"
" with:\n"
" path: |\n"
" ~/.cargo\n"
" ~/.npm\n"
" ~/.cache/pip\n"
" key: ${{ runner.os }}-deps-${{ hashFiles('**/package-lock.json') }}"
)
elif platform == 'gitlab':
cache_config = config.get('cache', {})
job_has_cache = False
# Check global cache
if cache_config:
has_cache = True
# Check job-level cache
for key, value in config.items():
if isinstance(value, dict) and 'script' in value:
if value.get('cache'):
job_has_cache = True
if not has_cache and not job_has_cache:
self.findings.append("No caching configuration detected")
self.optimizations.append(
"Add caching to speed up builds:\n"
"cache:\n"
" key: ${CI_COMMIT_REF_SLUG}\n"
" paths:\n"
" - node_modules/\n"
" - .npm/\n"
" - vendor/"
)
def _check_job_parallelization(self, config: Dict, platform: str):
"""Check for job parallelization opportunities"""
if platform == 'github':
jobs = config.get('jobs', {})
# Count jobs with dependencies
jobs_with_needs = sum(1 for job in jobs.values()
if isinstance(job, dict) and 'needs' in job)
if len(jobs) > 1 and jobs_with_needs == 0:
self.optimizations.append(
f"Found {len(jobs)} jobs with no dependencies - they will run in parallel (good!)"
)
elif len(jobs) > 3 and jobs_with_needs == len(jobs):
self.findings.append("All jobs have 'needs' dependencies - may be unnecessarily sequential")
self.optimizations.append(
"Review job dependencies - remove 'needs' where jobs can run in parallel"
)
elif platform == 'gitlab':
stages = config.get('stages', [])
if len(stages) > 5:
self.findings.append(f"Pipeline has {len(stages)} stages - may be overly sequential")
self.optimizations.append(
"Consider reducing stages to allow more parallel execution"
)
def _check_dependency_management(self, config: Dict, platform: str):
"""Check dependency installation patterns"""
if platform == 'github':
jobs = config.get('jobs', {})
for job_name, job in jobs.items():
steps = job.get('steps', [])
for step in steps:
if isinstance(step, dict):
run_cmd = step.get('run', '')
# Check for npm ci vs npm install
if 'npm install' in run_cmd and 'npm ci' not in run_cmd:
self.findings.append(f"Job '{job_name}' uses 'npm install' instead of 'npm ci'")
self.optimizations.append(
f"Use 'npm ci' instead of 'npm install' for faster, reproducible installs"
)
# Check for pip install without cache
if 'pip install' in run_cmd:
has_pip_cache = any(
s.get('uses', '').startswith('actions/cache') and
'pip' in str(s.get('with', {}).get('path', ''))
for s in steps if isinstance(s, dict)
)
if not has_pip_cache:
self.optimizations.append(
f"Add pip cache for job '{job_name}' to speed up Python dependency installation"
)
def _check_matrix_strategy(self, workflow: Dict):
"""Check for matrix strategy usage"""
jobs = workflow.get('jobs', {})
for job_name, job in jobs.items():
if isinstance(job, dict):
strategy = job.get('strategy', {})
matrix = strategy.get('matrix', {})
if matrix:
# Check fail-fast
fail_fast = strategy.get('fail-fast', True)
if fail_fast:
self.optimizations.append(
f"Job '{job_name}' has fail-fast=true (default). "
f"Consider fail-fast=false to see all matrix results"
)
# Check for large matrices
matrix_size = 1
for key, values in matrix.items():
if isinstance(values, list):
matrix_size *= len(values)
if matrix_size > 20:
self.findings.append(
f"Job '{job_name}' has large matrix ({matrix_size} combinations)"
)
self.optimizations.append(
f"Consider reducing matrix size or using 'exclude' to skip unnecessary combinations"
)
def _check_artifact_usage(self, workflow: Dict):
"""Check artifact upload/download patterns"""
jobs = workflow.get('jobs', {})
uploads = {}
downloads = {}
for job_name, job in jobs.items():
if not isinstance(job, dict):
continue
steps = job.get('steps', [])
for step in steps:
if isinstance(step, dict):
uses = step.get('uses', '')
if 'actions/upload-artifact' in uses:
artifact_name = step.get('with', {}).get('name', 'unknown')
uploads[artifact_name] = job_name
if 'actions/download-artifact' in uses:
artifact_name = step.get('with', {}).get('name', 'unknown')
downloads.setdefault(artifact_name, []).append(job_name)
# Check for unused artifacts
for artifact, uploader in uploads.items():
if artifact not in downloads:
self.findings.append(f"Artifact '{artifact}' uploaded but never downloaded")
self.optimizations.append(f"Remove unused artifact upload or add download step")
def _analyze_action_versions(self, workflow: Dict):
"""Check for outdated action versions"""
jobs = workflow.get('jobs', {})
outdated_actions = []
for job_name, job in jobs.items():
if not isinstance(job, dict):
continue
steps = job.get('steps', [])
for step in steps:
if isinstance(step, dict):
uses = step.get('uses', '')
# Check for @v1, @v2 versions (likely outdated)
if '@v1' in uses or '@v2' in uses:
outdated_actions.append(uses)
if outdated_actions:
self.findings.append(f"Found {len(outdated_actions)} potentially outdated actions")
self.optimizations.append(
f"Update to latest action versions:\n" +
"\n".join(f" - {action}" for action in set(outdated_actions))
)
def _check_gitlab_specific_features(self, config: Dict):
"""Check GitLab-specific optimization opportunities"""
# Check for interruptible jobs
has_interruptible = any(
isinstance(v, dict) and v.get('interruptible')
for v in config.values()
)
if not has_interruptible:
self.optimizations.append(
"Consider marking jobs as 'interruptible: true' to cancel redundant pipeline runs:\n"
"job_name:\n"
" interruptible: true"
)
# Check for DAG usage (needs keyword)
has_needs = any(
isinstance(v, dict) and 'needs' in v
for v in config.values()
)
if not has_needs and config.get('stages') and len(config.get('stages', [])) > 2:
self.optimizations.append(
"Consider using 'needs' keyword for DAG pipelines to improve parallelization:\n"
"test:\n"
" needs: [build]"
)
def _error(self, message: str) -> Dict:
"""Return error report"""
return {
'status': 'error',
'error': message,
'findings': [],
'optimizations': []
}
def _generate_report(self) -> Dict:
"""Generate analysis report"""
return {
'status': 'success',
'platform': self.platform,
'findings': self.findings,
'optimizations': self.optimizations,
'metrics': self.metrics
}
def print_report(report: Dict):
"""Print formatted analysis report"""
if report['status'] == 'error':
print(f"\n❌ Error: {report['error']}\n")
return
print("\n" + "="*60)
print(f"📊 Pipeline Analysis Report - {report['platform'].upper()}")
print("="*60)
if report['findings']:
print(f"\n🔍 Findings ({len(report['findings'])}):")
for i, finding in enumerate(report['findings'], 1):
print(f"\n {i}. {finding}")
if report['optimizations']:
print(f"\n💡 Optimization Recommendations ({len(report['optimizations'])}):")
for i, opt in enumerate(report['optimizations'], 1):
print(f"\n {i}. {opt}")
if not report['findings'] and not report['optimizations']:
print("\n✅ No issues found - pipeline looks well optimized!")
print("\n" + "="*60 + "\n")
def main():
parser = argparse.ArgumentParser(
description='CI/CD Pipeline Performance Analyzer',
formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument('--platform', required=True, choices=['github', 'gitlab'],
help='CI/CD platform')
parser.add_argument('--workflow', help='Path to GitHub Actions workflow file')
parser.add_argument('--config', help='Path to GitLab CI config file')
parser.add_argument('--repo', help='Repository (owner/repo) for run analysis')
parser.add_argument('--analyze-runs', type=int, help='Number of recent runs to analyze')
args = parser.parse_args()
# Create analyzer
analyzer = PipelineAnalyzer(
platform=args.platform,
repo=args.repo
)
# Run analysis
if args.platform == 'github':
if args.workflow:
report = analyzer.analyze_github_workflow(args.workflow)
else:
# Try to find workflow files
workflow_dir = Path('.github/workflows')
if workflow_dir.exists():
workflows = list(workflow_dir.glob('*.yml')) + list(workflow_dir.glob('*.yaml'))
if workflows:
print(f"Found {len(workflows)} workflow(s), analyzing first one...")
report = analyzer.analyze_github_workflow(str(workflows[0]))
else:
print("❌ No workflow files found in .github/workflows/")
sys.exit(1)
else:
print("❌ No .github/workflows/ directory found")
sys.exit(1)
elif args.platform == 'gitlab':
config_file = args.config or '.gitlab-ci.yml'
report = analyzer.analyze_gitlab_pipeline(config_file)
# Print report
print_report(report)
# Exit with appropriate code
sys.exit(0 if report['status'] == 'success' else 1)
if __name__ == '__main__':
main()