Initial commit

skills/testing-strategy/scripts/coverage_check.py

@@ -0,0 +1,260 @@
+#!/usr/bin/env python3
+"""
+Check test coverage and enforce 80% minimum threshold for Grey Haven projects.
+
+Analyzes coverage reports from Vitest or pytest and provides detailed
+breakdown of coverage by file, function, and line.
+
+Usage:
+# Check Vitest coverage
+python scripts/coverage_check.py
+
+# Check pytest coverage
+python scripts/coverage_check.py --backend pytest
+
+# Show detailed file-by-file breakdown
+python scripts/coverage_check.py --detailed
+
+# Check coverage and fail if below threshold
+python scripts/coverage_check.py --strict
+
+# Generate coverage report if missing
+python scripts/coverage_check.py --generate
+
+Always run with --help first to see all options.
+"""
+
+import argparse
+import subprocess
+import sys
+import json
+import os
+from pathlib import Path
+
+
+def run_command(cmd: str, capture: bool = True) -> tuple[bool, str]:
+"""Run a shell command and return success status and output."""
+result = subprocess.run(
+cmd,
+shell=True,
+capture_output=capture,
+text=True
+)
+return result.returncode == 0, result.stdout if capture else ""
+
+
+def check_vitest_coverage(detailed: bool = False) -> dict:
+"""Check Vitest coverage from coverage/coverage-summary.json."""
+coverage_file = Path("coverage/coverage-summary.json")
+
+if not coverage_file.exists():
+print("ERROR: Coverage report not found: coverage/coverage-summary.json")
+print(" Run tests with coverage first:")
+print(" doppler run --config test -- vitest run --coverage")
+sys.exit(1)
+
+with open(coverage_file) as f:
+coverage_data = json.load(f)
+
+# Total coverage
+total = coverage_data["total"]
+
+results = {
+"lines": total["lines"]["pct"],
+"statements": total["statements"]["pct"],
+"functions": total["functions"]["pct"],
+"branches": total["branches"]["pct"],
+}
+
+# Detailed breakdown by file
+if detailed:
+print("\n Coverage by File:")
+print(f"{'File':<50} {'Lines':<10} {'Funcs':<10} {'Branches':<10}")
+print("=" * 80)
+
+for file_path, file_data in coverage_data.items():
+if file_path == "total":
+continue
+
+# Shorten file path for display
+short_path = file_path.replace(os.getcwd(), ".")
+if len(short_path) > 47:
+short_path = "..." + short_path[-44:]
+
+lines_pct = file_data["lines"]["pct"]
+funcs_pct = file_data["functions"]["pct"]
+branches_pct = file_data["branches"]["pct"]
+
+# Color code based on coverage
+if lines_pct < 80:
+status = "[BELOW 80%]"
+elif lines_pct < 90:
+status = "[80-90%]"
+else:
+status = "[ABOVE 90%]"
+
+print(f"{status} {short_path:<47} {lines_pct:<9.1f}% {funcs_pct:<9.1f}% {branches_pct:<9.1f}%")
+
+return results
+
+
+def check_pytest_coverage(detailed: bool = False) -> dict:
+"""Check pytest coverage from .coverage file."""
+coverage_file = Path(".coverage")
+
+if not coverage_file.exists():
+print("ERROR: Coverage report not found: .coverage")
+print(" Run tests with coverage first:")
+print(" doppler run --config test -- pytest --cov=app")
+sys.exit(1)
+
+# Use coverage.py to get report
+success, output = run_command("coverage report --format=total")
+
+if not success:
+print("ERROR: Failed to generate coverage report")
+sys.exit(1)
+
+# Parse total coverage percentage
+total_coverage = float(output.strip().rstrip("%"))
+
+# Get detailed report if requested
+if detailed:
+print("\n Coverage by File:")
+success, detailed_output = run_command("coverage report")
+print(detailed_output)
+
+# pytest coverage doesn't separate by type, so we use total for all
+results = {
+"lines": total_coverage,
+"statements": total_coverage,
+"functions": total_coverage,
+"branches": total_coverage,
+}
+
+return results
+
+
+def main():
+parser = argparse.ArgumentParser(
+description="Check test coverage and enforce thresholds",
+formatter_class=argparse.RawDescriptionHelpFormatter,
+epilog="""
+Examples:
+# Check Vitest coverage
+python scripts/coverage_check.py
+
+# Check pytest coverage with detailed breakdown
+python scripts/coverage_check.py --backend pytest --detailed
+
+# Generate coverage and check (strict mode)
+python scripts/coverage_check.py --generate --strict
+
+Coverage Thresholds:
+Minimum 80% coverage required for:
+- Lines
+- Functions
+- Branches
+- Statements
+
+Backends:
+vitest - Vitest (TypeScript/React) - default
+pytest - pytest (Python/FastAPI)
+"""
+)
+
+parser.add_argument(
+"--backend",
+default="vitest",
+choices=["vitest", "pytest"],
+help="Test backend to check coverage for (default: vitest)"
+)
+parser.add_argument(
+"--detailed",
+action="store_true",
+help="Show detailed file-by-file breakdown"
+)
+parser.add_argument(
+"--strict",
+action="store_true",
+help="Exit with error if coverage below 80 percent"
+)
+parser.add_argument(
+"--generate",
+action="store_true",
+help="Generate coverage report before checking"
+)
+
+args = parser.parse_args()
+
+print(f"\n{'=' * 70}")
+print(f" Coverage Check - {args.backend.upper()}")
+print(f"{'=' * 70}")
+
+# Generate coverage if requested
+if args.generate:
+print("\n→ Generating coverage report...")
+
+if args.backend == "vitest":
+cmd = "doppler run --config test -- vitest run --coverage"
+else:
+cmd = "doppler run --config test -- pytest --cov=app --cov-report=term --cov-report=html"
+
+success, _ = run_command(cmd, capture=False)
+
+if not success:
+print("\nERROR: Failed to generate coverage")
+sys.exit(1)
+
+# Check coverage
+if args.backend == "vitest":
+coverage = check_vitest_coverage(args.detailed)
+else:
+coverage = check_pytest_coverage(args.detailed)
+
+# Display summary
+print(f"\n{'=' * 70}")
+print(" Coverage Summary")
+print(f"{'=' * 70}")
+
+threshold = 80.0
+all_pass = True
+
+for metric, value in coverage.items():
+if value >= threshold:
+status = "SUCCESS:"
+else:
+status = "ERROR:"
+all_pass = False
+
+print(f"{status} {metric.capitalize():<15} {value:>6.2f}% (threshold: {threshold}%)")
+
+# Overall result
+print(f"\n{'=' * 70}")
+if all_pass:
+print(" SUCCESS: All coverage thresholds met!")
+else:
+print(" ERROR: Coverage below 80% threshold")
+
+print(f"{'=' * 70}")
+
+# Additional info
+if not all_pass:
+print("\nTIP: Tips to improve coverage:")
+print(" • Add unit tests for uncovered functions")
+print(" • Add integration tests for API endpoints")
+print(" • Add edge case tests for conditionals")
+print(" • Test error handling paths")
+
+if args.backend == "vitest":
+print("\n View detailed report: coverage/index.html")
+else:
+print("\n View detailed report: htmlcov/index.html")
+
+# Exit with error in strict mode if coverage below threshold
+if args.strict and not all_pass:
+sys.exit(1)
+
+
+if __name__ == "__main__":
+main()

skills/testing-strategy/scripts/run_tests.py

@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+"""
+Run tests for Grey Haven applications with Doppler environment variables.
+
+Supports Vitest (TypeScript) and pytest (Python) with markers for different
+test types (unit, integration, e2e, benchmark).
+
+Usage:
+# Run all tests with test environment
+python scripts/run_tests.py
+
+# Run unit tests only
+python scripts/run_tests.py --type unit
+
+# Run integration and e2e tests
+python scripts/run_tests.py --type integration --type e2e
+
+# Run tests with coverage
+python scripts/run_tests.py --coverage
+
+# Run tests in watch mode (for development)
+python scripts/run_tests.py --watch
+
+# Run pytest instead of Vitest
+python scripts/run_tests.py --backend pytest
+
+# Run with specific Doppler environment
+python scripts/run_tests.py --env ci
+
+Always run with --help first to see all options.
+"""
+
+import argparse
+import subprocess
+import sys
+from typing import List
+
+
+def run_command(cmd: str, description: str) -> bool:
+"""Run a shell command and return success status."""
+print(f"\n→ {description}")
+print(f" Command: {cmd}\n")
+
+result = subprocess.run(cmd, shell=True)
+return result.returncode == 0
+
+
+def main():
+parser = argparse.ArgumentParser(
+description="Run tests with Doppler environment variables",
+formatter_class=argparse.RawDescriptionHelpFormatter,
+epilog="""
+Examples:
+# Run all Vitest tests
+python scripts/run_tests.py
+
+# Run unit tests only
+python scripts/run_tests.py --type unit
+
+# Run integration and e2e tests with coverage
+python scripts/run_tests.py --type integration --type e2e --coverage
+
+# Run pytest unit tests
+python scripts/run_tests.py --backend pytest --type unit
+
+# Run tests in watch mode
+python scripts/run_tests.py --watch
+
+Test Types (Markers):
+unit - Unit tests (fast, isolated)
+integration - Integration tests (database, external services)
+e2e - End-to-end tests (full application flow)
+benchmark - Performance benchmark tests
+
+Backends:
+vitest - Vitest (TypeScript/React) - default
+pytest - pytest (Python/FastAPI)
+
+Doppler Configuration:
+Uses 'test' config by default.
+Override with --env flag for CI environments.
+"""
+)
+
+parser.add_argument(
+"--backend",
+default="vitest",
+choices=["vitest", "pytest"],
+help="Test backend to use (default: vitest)"
+)
+parser.add_argument(
+"--type",
+action="append",
+choices=["unit", "integration", "e2e", "benchmark"],
+help="Test type(s) to run (can be repeated). If not specified, runs all tests."
+)
+parser.add_argument(
+"--coverage",
+action="store_true",
+help="Run with coverage reporting"
+)
+parser.add_argument(
+"--watch",
+action="store_true",
+help="Run in watch mode (for development)"
+)
+parser.add_argument(
+"--env",
+default="test",
+help="Doppler environment config to use (default: test)"
+)
+parser.add_argument(
+"--verbose",
+action="store_true",
+help="Verbose output"
+)
+
+args = parser.parse_args()
+
+backend = args.backend
+env = args.env
+test_types = args.type or []
+
+print(f"\n{'=' * 70}")
+print(f" Running {backend.upper()} Tests")
+print(f" Environment: {env}")
+if test_types:
+print(f" Types: {', '.join(test_types)}")
+print(f"{'=' * 70}")
+
+# Construct test command based on backend
+if backend == "vitest":
+# Base Vitest command
+cmd_parts = ["doppler", "run", "--config", env, "--", "vitest"]
+
+# Add test types as grep patterns
+if test_types:
+# Vitest uses file patterns or test name patterns
+# We'll use test name patterns matching our markers
+patterns = "|".join(test_types)
+cmd_parts.extend(["-t", f"({patterns})"])
+
+# Add coverage flag
+if args.coverage:
+cmd_parts.append("--coverage")
+
+# Add watch mode
+if args.watch:
+cmd_parts.append("--watch")
+
+# Add verbose flag
+if args.verbose:
+cmd_parts.append("--reporter=verbose")
+
+# Run mode (not watch)
+if not args.watch:
+cmd_parts.append("run")
+
+elif backend == "pytest":
+# Base pytest command
+cmd_parts = ["doppler", "run", "--config", env, "--", "pytest"]
+
+# Add test types as markers
+if test_types:
+markers = " or ".join(test_types)
+cmd_parts.extend(["-m", markers])
+
+# Add coverage flag
+if args.coverage:
+cmd_parts.extend([
+"--cov=app",
+"--cov-report=term-missing",
+"--cov-report=html"
+])
+
+# Add verbose flag
+if args.verbose:
+cmd_parts.append("-vv")
+
+# pytest doesn't have built-in watch mode
+if args.watch:
+print("\nWARNING: Warning: pytest doesn't support watch mode natively")
+print(" Consider using pytest-watch: pip install pytest-watch")
+
+cmd = " ".join(cmd_parts)
+success = run_command(cmd, f"Running {backend} tests")
+
+if not success:
+print(f"\nERROR: Tests failed")
+sys.exit(1)
+
+# Coverage threshold check (if coverage was run)
+if args.coverage and backend == "vitest":
+print("\n→ Checking coverage thresholds...")
+print(" Required: 80% (lines, functions, branches, statements)")
+
+# Vitest coverage is configured in vitest.config.ts
+# Thresholds are enforced automatically
+print(" ✓ Coverage thresholds enforced by Vitest config")
+
+elif args.coverage and backend == "pytest":
+print("\n→ Checking coverage thresholds...")
+print(" Required: 80% coverage")
+
+# Check coverage with pytest-cov
+coverage_cmd = f"doppler run --config {env} -- pytest --cov=app --cov-fail-under=80 -q"
+coverage_success = run_command(coverage_cmd, "Validating coverage threshold")
+
+if not coverage_success:
+print("\nERROR: Coverage below 80% threshold")
+print(" Add more tests to increase coverage")
+sys.exit(1)
+
+# Success!
+print(f"\n{'=' * 70}")
+print(f" SUCCESS: All tests passed!")
+print(f"{'=' * 70}")
+
+if args.coverage:
+if backend == "vitest":
+print("\n Coverage report: coverage/index.html")
+else:
+print("\n Coverage report: htmlcov/index.html")
+
+print("\nNext steps:")
+if not test_types:
+print(" • All tests passed - ready to commit")
+else:
+print(f" • {', '.join(test_types)} tests passed")
+if not args.coverage:
+print(" • Run with --coverage to check code coverage")
+print(" • Deploy with: python scripts/deploy.py --env staging")
+
+
+if __name__ == "__main__":
+main()