Initial commit

This commit is contained in:
Zhongwei Li
2025-11-30 08:48:52 +08:00
commit 6ec3196ecc
434 changed files with 125248 additions and 0 deletions

View File

@@ -0,0 +1,502 @@
#!/usr/bin/env python3
"""
Database backup and restore tool for MongoDB and PostgreSQL.
Supports compression, scheduling, and verification.
"""
import argparse
import gzip
import json
import os
import shutil
import subprocess
import sys
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Optional
@dataclass
class BackupInfo:
"""Backup metadata."""
filename: str
database_type: str
database_name: str
timestamp: datetime
size_bytes: int
compressed: bool
verified: bool = False
class BackupManager:
"""Manages database backups for MongoDB and PostgreSQL."""
def __init__(self, db_type: str, backup_dir: str = "./backups"):
"""
Initialize backup manager.
Args:
db_type: Database type ('mongodb' or 'postgres')
backup_dir: Directory to store backups
"""
self.db_type = db_type.lower()
self.backup_dir = Path(backup_dir)
self.backup_dir.mkdir(exist_ok=True)
def create_backup(
self,
uri: str,
database: Optional[str] = None,
compress: bool = True,
verify: bool = True
) -> Optional[BackupInfo]:
"""
Create database backup.
Args:
uri: Database connection string
database: Database name (optional for MongoDB)
compress: Compress backup file
verify: Verify backup after creation
Returns:
BackupInfo if successful, None otherwise
"""
timestamp = datetime.now()
date_str = timestamp.strftime("%Y%m%d_%H%M%S")
if self.db_type == "mongodb":
return self._backup_mongodb(uri, database, date_str, compress, verify)
elif self.db_type == "postgres":
return self._backup_postgres(uri, database, date_str, compress, verify)
else:
print(f"Error: Unsupported database type: {self.db_type}")
return None
def _backup_mongodb(
self,
uri: str,
database: Optional[str],
date_str: str,
compress: bool,
verify: bool
) -> Optional[BackupInfo]:
"""Create MongoDB backup using mongodump."""
db_name = database or "all"
filename = f"mongodb_{db_name}_{date_str}"
backup_path = self.backup_dir / filename
try:
cmd = ["mongodump", "--uri", uri, "--out", str(backup_path)]
if database:
cmd.extend(["--db", database])
print(f"Creating MongoDB backup: {filename}")
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
print(f"Error: {result.stderr}")
return None
# Compress if requested
if compress:
archive_path = backup_path.with_suffix(".tar.gz")
print(f"Compressing backup...")
shutil.make_archive(str(backup_path), "gztar", backup_path)
shutil.rmtree(backup_path)
backup_path = archive_path
filename = archive_path.name
size_bytes = self._get_size(backup_path)
backup_info = BackupInfo(
filename=filename,
database_type="mongodb",
database_name=db_name,
timestamp=datetime.now(),
size_bytes=size_bytes,
compressed=compress
)
if verify:
backup_info.verified = self._verify_backup(backup_info)
self._save_metadata(backup_info)
print(f"✓ Backup created: {filename} ({self._format_size(size_bytes)})")
return backup_info
except Exception as e:
print(f"Error creating MongoDB backup: {e}")
return None
def _backup_postgres(
self,
uri: str,
database: str,
date_str: str,
compress: bool,
verify: bool
) -> Optional[BackupInfo]:
"""Create PostgreSQL backup using pg_dump."""
if not database:
print("Error: Database name required for PostgreSQL backup")
return None
ext = ".sql.gz" if compress else ".sql"
filename = f"postgres_{database}_{date_str}{ext}"
backup_path = self.backup_dir / filename
try:
cmd = ["pg_dump", uri]
if compress:
# Use pg_dump with gzip
with open(backup_path, "wb") as f:
dump_proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
gzip_proc = subprocess.Popen(
["gzip"],
stdin=dump_proc.stdout,
stdout=f
)
dump_proc.stdout.close()
gzip_proc.communicate()
if dump_proc.returncode != 0:
print("Error: pg_dump failed")
return None
else:
with open(backup_path, "w") as f:
result = subprocess.run(cmd, stdout=f, stderr=subprocess.PIPE, text=True)
if result.returncode != 0:
print(f"Error: {result.stderr}")
return None
size_bytes = backup_path.stat().st_size
backup_info = BackupInfo(
filename=filename,
database_type="postgres",
database_name=database,
timestamp=datetime.now(),
size_bytes=size_bytes,
compressed=compress
)
if verify:
backup_info.verified = self._verify_backup(backup_info)
self._save_metadata(backup_info)
print(f"✓ Backup created: {filename} ({self._format_size(size_bytes)})")
return backup_info
except Exception as e:
print(f"Error creating PostgreSQL backup: {e}")
return None
def restore_backup(self, filename: str, uri: str, dry_run: bool = False) -> bool:
"""
Restore database from backup.
Args:
filename: Backup filename
uri: Database connection string
dry_run: If True, only show what would be done
Returns:
True if successful, False otherwise
"""
backup_path = self.backup_dir / filename
if not backup_path.exists():
print(f"Error: Backup not found: {filename}")
return False
# Load metadata
metadata_path = backup_path.with_suffix(".json")
if metadata_path.exists():
with open(metadata_path) as f:
metadata = json.load(f)
print(f"Restoring backup from {metadata['timestamp']}")
print(f"Database: {metadata['database_name']}")
if dry_run:
print(f"Would restore from: {backup_path}")
return True
print(f"Restoring backup: {filename}")
try:
if self.db_type == "mongodb":
return self._restore_mongodb(backup_path, uri)
elif self.db_type == "postgres":
return self._restore_postgres(backup_path, uri)
else:
print(f"Error: Unsupported database type: {self.db_type}")
return False
except Exception as e:
print(f"Error restoring backup: {e}")
return False
def _restore_mongodb(self, backup_path: Path, uri: str) -> bool:
"""Restore MongoDB backup using mongorestore."""
try:
# Extract if compressed
restore_path = backup_path
if backup_path.suffix == ".gz":
print("Extracting backup...")
extract_path = backup_path.with_suffix("")
shutil.unpack_archive(backup_path, extract_path)
restore_path = extract_path
cmd = ["mongorestore", "--uri", uri, str(restore_path)]
result = subprocess.run(cmd, capture_output=True, text=True)
# Cleanup extracted files
if restore_path != backup_path and restore_path.is_dir():
shutil.rmtree(restore_path)
if result.returncode != 0:
print(f"Error: {result.stderr}")
return False
print("✓ Restore completed")
return True
except Exception as e:
print(f"Error restoring MongoDB: {e}")
return False
def _restore_postgres(self, backup_path: Path, uri: str) -> bool:
"""Restore PostgreSQL backup using psql."""
try:
if backup_path.suffix == ".gz":
# Decompress and restore
with gzip.open(backup_path, "rb") as f:
cmd = ["psql", uri]
result = subprocess.run(
cmd,
stdin=f,
capture_output=True,
text=False
)
else:
with open(backup_path) as f:
cmd = ["psql", uri]
result = subprocess.run(
cmd,
stdin=f,
capture_output=True,
text=True
)
if result.returncode != 0:
print(f"Error: {result.stderr}")
return False
print("✓ Restore completed")
return True
except Exception as e:
print(f"Error restoring PostgreSQL: {e}")
return False
def list_backups(self) -> List[BackupInfo]:
"""
List all backups.
Returns:
List of BackupInfo objects
"""
backups = []
for metadata_file in sorted(self.backup_dir.glob("*.json")):
try:
with open(metadata_file) as f:
data = json.load(f)
backup_info = BackupInfo(
filename=data["filename"],
database_type=data["database_type"],
database_name=data["database_name"],
timestamp=datetime.fromisoformat(data["timestamp"]),
size_bytes=data["size_bytes"],
compressed=data["compressed"],
verified=data.get("verified", False)
)
backups.append(backup_info)
except Exception as e:
print(f"Error reading metadata {metadata_file}: {e}")
return backups
def cleanup_old_backups(self, retention_days: int, dry_run: bool = False) -> int:
"""
Remove backups older than retention period.
Args:
retention_days: Number of days to retain backups
dry_run: If True, only show what would be deleted
Returns:
Number of backups removed
"""
cutoff = datetime.now().timestamp() - (retention_days * 24 * 3600)
removed = 0
for backup_file in self.backup_dir.glob("*"):
if backup_file.suffix == ".json":
continue
if backup_file.stat().st_mtime < cutoff:
if dry_run:
print(f"Would remove: {backup_file.name}")
else:
print(f"Removing: {backup_file.name}")
backup_file.unlink()
# Remove metadata
metadata_file = backup_file.with_suffix(".json")
if metadata_file.exists():
metadata_file.unlink()
removed += 1
return removed
def _verify_backup(self, backup_info: BackupInfo) -> bool:
"""
Verify backup integrity.
Args:
backup_info: Backup information
Returns:
True if backup is valid, False otherwise
"""
backup_path = self.backup_dir / backup_info.filename
if not backup_path.exists():
return False
# Basic verification: file exists and has size > 0
if backup_path.stat().st_size == 0:
return False
# Could add more verification here (checksums, test restore, etc.)
return True
def _get_size(self, path: Path) -> int:
"""Get total size of file or directory."""
if path.is_file():
return path.stat().st_size
elif path.is_dir():
total = 0
for item in path.rglob("*"):
if item.is_file():
total += item.stat().st_size
return total
return 0
def _format_size(self, size_bytes: int) -> str:
"""Format size in human-readable format."""
for unit in ["B", "KB", "MB", "GB", "TB"]:
if size_bytes < 1024:
return f"{size_bytes:.2f} {unit}"
size_bytes /= 1024
return f"{size_bytes:.2f} PB"
def _save_metadata(self, backup_info: BackupInfo):
"""Save backup metadata to JSON file."""
metadata_path = self.backup_dir / f"{backup_info.filename}.json"
metadata = {
"filename": backup_info.filename,
"database_type": backup_info.database_type,
"database_name": backup_info.database_name,
"timestamp": backup_info.timestamp.isoformat(),
"size_bytes": backup_info.size_bytes,
"compressed": backup_info.compressed,
"verified": backup_info.verified
}
with open(metadata_path, "w") as f:
json.dump(metadata, f, indent=2)
def main():
"""Main entry point."""
parser = argparse.ArgumentParser(description="Database backup tool")
parser.add_argument("--db", required=True, choices=["mongodb", "postgres"],
help="Database type")
parser.add_argument("--backup-dir", default="./backups",
help="Backup directory")
subparsers = parser.add_subparsers(dest="command", required=True)
# Backup command
backup_parser = subparsers.add_parser("backup", help="Create backup")
backup_parser.add_argument("--uri", required=True, help="Database connection string")
backup_parser.add_argument("--database", help="Database name")
backup_parser.add_argument("--no-compress", action="store_true",
help="Disable compression")
backup_parser.add_argument("--no-verify", action="store_true",
help="Skip verification")
# Restore command
restore_parser = subparsers.add_parser("restore", help="Restore backup")
restore_parser.add_argument("filename", help="Backup filename")
restore_parser.add_argument("--uri", required=True, help="Database connection string")
restore_parser.add_argument("--dry-run", action="store_true",
help="Show what would be done")
# List command
subparsers.add_parser("list", help="List backups")
# Cleanup command
cleanup_parser = subparsers.add_parser("cleanup", help="Remove old backups")
cleanup_parser.add_argument("--retention-days", type=int, default=7,
help="Days to retain backups (default: 7)")
cleanup_parser.add_argument("--dry-run", action="store_true",
help="Show what would be removed")
args = parser.parse_args()
manager = BackupManager(args.db, args.backup_dir)
if args.command == "backup":
backup_info = manager.create_backup(
args.uri,
args.database,
compress=not args.no_compress,
verify=not args.no_verify
)
sys.exit(0 if backup_info else 1)
elif args.command == "restore":
success = manager.restore_backup(args.filename, args.uri, args.dry_run)
sys.exit(0 if success else 1)
elif args.command == "list":
backups = manager.list_backups()
print(f"Total backups: {len(backups)}\n")
for backup in backups:
verified_str = "" if backup.verified else "?"
print(f"[{verified_str}] {backup.filename}")
print(f" Database: {backup.database_name}")
print(f" Created: {backup.timestamp}")
print(f" Size: {manager._format_size(backup.size_bytes)}")
print()
elif args.command == "cleanup":
removed = manager.cleanup_old_backups(args.retention_days, args.dry_run)
print(f"Removed {removed} backup(s)")
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,414 @@
#!/usr/bin/env python3
"""
Database migration tool for MongoDB and PostgreSQL.
Generates and applies schema migrations with rollback support.
"""
import argparse
import json
import os
import sys
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional
try:
from pymongo import MongoClient
MONGO_AVAILABLE = True
except ImportError:
MONGO_AVAILABLE = False
try:
import psycopg2
from psycopg2 import sql
POSTGRES_AVAILABLE = True
except ImportError:
POSTGRES_AVAILABLE = False
@dataclass
class Migration:
"""Represents a database migration."""
id: str
name: str
timestamp: datetime
database_type: str
up_sql: Optional[str] = None
down_sql: Optional[str] = None
mongodb_operations: Optional[List[Dict[str, Any]]] = None
applied: bool = False
class MigrationManager:
"""Manages database migrations for MongoDB and PostgreSQL."""
def __init__(self, db_type: str, connection_string: str, migrations_dir: str = "./migrations"):
"""
Initialize migration manager.
Args:
db_type: Database type ('mongodb' or 'postgres')
connection_string: Database connection string
migrations_dir: Directory to store migration files
"""
self.db_type = db_type.lower()
self.connection_string = connection_string
self.migrations_dir = Path(migrations_dir)
self.migrations_dir.mkdir(exist_ok=True)
self.client = None
self.db = None
self.conn = None
def connect(self) -> bool:
"""
Connect to database.
Returns:
True if connection successful, False otherwise
"""
try:
if self.db_type == "mongodb":
if not MONGO_AVAILABLE:
print("Error: pymongo not installed")
return False
self.client = MongoClient(self.connection_string)
self.db = self.client.get_default_database()
# Test connection
self.client.server_info()
return True
elif self.db_type == "postgres":
if not POSTGRES_AVAILABLE:
print("Error: psycopg2 not installed")
return False
self.conn = psycopg2.connect(self.connection_string)
return True
else:
print(f"Error: Unsupported database type: {self.db_type}")
return False
except Exception as e:
print(f"Connection error: {e}")
return False
def disconnect(self):
"""Disconnect from database."""
try:
if self.client:
self.client.close()
if self.conn:
self.conn.close()
except Exception as e:
print(f"Disconnect error: {e}")
def _ensure_migrations_table(self):
"""Create migrations tracking table/collection if not exists."""
if self.db_type == "mongodb":
# MongoDB creates collection automatically
pass
elif self.db_type == "postgres":
with self.conn.cursor() as cur:
cur.execute("""
CREATE TABLE IF NOT EXISTS migrations (
id VARCHAR(255) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
)
""")
self.conn.commit()
def generate_migration(self, name: str, dry_run: bool = False) -> Optional[Migration]:
"""
Generate new migration file.
Args:
name: Migration name
dry_run: If True, only show what would be generated
Returns:
Migration object if successful, None otherwise
"""
timestamp = datetime.now()
migration_id = timestamp.strftime("%Y%m%d%H%M%S")
filename = f"{migration_id}_{name}.json"
filepath = self.migrations_dir / filename
migration = Migration(
id=migration_id,
name=name,
timestamp=timestamp,
database_type=self.db_type
)
if self.db_type == "mongodb":
migration.mongodb_operations = [
{
"operation": "createIndex",
"collection": "example_collection",
"index": {"field": 1},
"options": {}
}
]
elif self.db_type == "postgres":
migration.up_sql = "-- Add your SQL here\n"
migration.down_sql = "-- Add rollback SQL here\n"
migration_data = {
"id": migration.id,
"name": migration.name,
"timestamp": migration.timestamp.isoformat(),
"database_type": migration.database_type,
"up_sql": migration.up_sql,
"down_sql": migration.down_sql,
"mongodb_operations": migration.mongodb_operations
}
if dry_run:
print(f"Would create: {filepath}")
print(json.dumps(migration_data, indent=2))
return migration
try:
with open(filepath, "w") as f:
json.dump(migration_data, f, indent=2)
print(f"Created migration: {filepath}")
return migration
except Exception as e:
print(f"Error creating migration: {e}")
return None
def get_pending_migrations(self) -> List[Migration]:
"""
Get list of pending migrations.
Returns:
List of pending Migration objects
"""
# Get applied migrations
applied_ids = set()
try:
if self.db_type == "mongodb":
applied_ids = {
doc["id"] for doc in self.db.migrations.find({}, {"id": 1})
}
elif self.db_type == "postgres":
with self.conn.cursor() as cur:
cur.execute("SELECT id FROM migrations")
applied_ids = {row[0] for row in cur.fetchall()}
except Exception as e:
print(f"Error reading applied migrations: {e}")
# Get all migration files
pending = []
for filepath in sorted(self.migrations_dir.glob("*.json")):
try:
with open(filepath) as f:
data = json.load(f)
if data["id"] not in applied_ids:
migration = Migration(
id=data["id"],
name=data["name"],
timestamp=datetime.fromisoformat(data["timestamp"]),
database_type=data["database_type"],
up_sql=data.get("up_sql"),
down_sql=data.get("down_sql"),
mongodb_operations=data.get("mongodb_operations")
)
pending.append(migration)
except Exception as e:
print(f"Error reading {filepath}: {e}")
return pending
def apply_migration(self, migration: Migration, dry_run: bool = False) -> bool:
"""
Apply migration.
Args:
migration: Migration to apply
dry_run: If True, only show what would be executed
Returns:
True if successful, False otherwise
"""
print(f"Applying migration: {migration.id} - {migration.name}")
if dry_run:
if self.db_type == "mongodb":
print("MongoDB operations:")
print(json.dumps(migration.mongodb_operations, indent=2))
elif self.db_type == "postgres":
print("SQL to execute:")
print(migration.up_sql)
return True
try:
if self.db_type == "mongodb":
for op in migration.mongodb_operations or []:
if op["operation"] == "createIndex":
self.db[op["collection"]].create_index(
list(op["index"].items()),
**op.get("options", {})
)
# Record migration
self.db.migrations.insert_one({
"id": migration.id,
"name": migration.name,
"applied_at": datetime.now()
})
elif self.db_type == "postgres":
with self.conn.cursor() as cur:
cur.execute(migration.up_sql)
# Record migration
cur.execute(
"INSERT INTO migrations (id, name) VALUES (%s, %s)",
(migration.id, migration.name)
)
self.conn.commit()
print(f"✓ Applied: {migration.id}")
return True
except Exception as e:
print(f"✗ Error applying migration: {e}")
if self.conn:
self.conn.rollback()
return False
def rollback_migration(self, migration_id: str, dry_run: bool = False) -> bool:
"""
Rollback migration.
Args:
migration_id: Migration ID to rollback
dry_run: If True, only show what would be executed
Returns:
True if successful, False otherwise
"""
# Find migration file
migration_file = None
for filepath in self.migrations_dir.glob(f"{migration_id}_*.json"):
migration_file = filepath
break
if not migration_file:
print(f"Migration not found: {migration_id}")
return False
try:
with open(migration_file) as f:
data = json.load(f)
print(f"Rolling back: {migration_id} - {data['name']}")
if dry_run:
if self.db_type == "postgres":
print("SQL to execute:")
print(data.get("down_sql", "-- No rollback defined"))
return True
if self.db_type == "postgres" and data.get("down_sql"):
with self.conn.cursor() as cur:
cur.execute(data["down_sql"])
cur.execute("DELETE FROM migrations WHERE id = %s", (migration_id,))
self.conn.commit()
elif self.db_type == "mongodb":
self.db.migrations.delete_one({"id": migration_id})
print(f"✓ Rolled back: {migration_id}")
return True
except Exception as e:
print(f"✗ Error rolling back: {e}")
if self.conn:
self.conn.rollback()
return False
def main():
"""Main entry point."""
parser = argparse.ArgumentParser(description="Database migration tool")
parser.add_argument("--db", required=True, choices=["mongodb", "postgres"],
help="Database type")
parser.add_argument("--uri", help="Database connection string")
parser.add_argument("--migrations-dir", default="./migrations",
help="Migrations directory")
subparsers = parser.add_subparsers(dest="command", required=True)
# Generate command
gen_parser = subparsers.add_parser("generate", help="Generate new migration")
gen_parser.add_argument("name", help="Migration name")
gen_parser.add_argument("--dry-run", action="store_true",
help="Show what would be generated")
# Apply command
apply_parser = subparsers.add_parser("apply", help="Apply pending migrations")
apply_parser.add_argument("--dry-run", action="store_true",
help="Show what would be executed")
# Rollback command
rollback_parser = subparsers.add_parser("rollback", help="Rollback migration")
rollback_parser.add_argument("id", help="Migration ID to rollback")
rollback_parser.add_argument("--dry-run", action="store_true",
help="Show what would be executed")
# Status command
subparsers.add_parser("status", help="Show migration status")
args = parser.parse_args()
# For generate, we don't need connection
if args.command == "generate":
manager = MigrationManager(args.db, "", args.migrations_dir)
migration = manager.generate_migration(args.name, args.dry_run)
sys.exit(0 if migration else 1)
# Other commands need connection
if not args.uri:
print("Error: --uri required for this command")
sys.exit(1)
manager = MigrationManager(args.db, args.uri, args.migrations_dir)
if not manager.connect():
sys.exit(1)
try:
manager._ensure_migrations_table()
if args.command == "status":
pending = manager.get_pending_migrations()
print(f"Pending migrations: {len(pending)}")
for migration in pending:
print(f" {migration.id} - {migration.name}")
elif args.command == "apply":
pending = manager.get_pending_migrations()
if not pending:
print("No pending migrations")
else:
for migration in pending:
if not manager.apply_migration(migration, args.dry_run):
sys.exit(1)
elif args.command == "rollback":
if not manager.rollback_migration(args.id, args.dry_run):
sys.exit(1)
finally:
manager.disconnect()
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,444 @@
#!/usr/bin/env python3
"""
Database performance analysis tool for MongoDB and PostgreSQL.
Analyzes slow queries, recommends indexes, and generates reports.
"""
import argparse
import json
import sys
from dataclasses import dataclass, asdict
from datetime import datetime
from typing import Dict, List, Optional
try:
from pymongo import MongoClient
MONGO_AVAILABLE = True
except ImportError:
MONGO_AVAILABLE = False
try:
import psycopg2
from psycopg2.extras import RealDictCursor
POSTGRES_AVAILABLE = True
except ImportError:
POSTGRES_AVAILABLE = False
@dataclass
class SlowQuery:
"""Represents a slow query."""
query: str
execution_time_ms: float
count: int
collection_or_table: Optional[str] = None
index_used: Optional[str] = None
@dataclass
class IndexRecommendation:
"""Index recommendation."""
collection_or_table: str
fields: List[str]
reason: str
estimated_benefit: str
@dataclass
class PerformanceReport:
"""Performance analysis report."""
database_type: str
database_name: str
timestamp: datetime
slow_queries: List[SlowQuery]
index_recommendations: List[IndexRecommendation]
database_metrics: Dict[str, any]
class PerformanceAnalyzer:
"""Analyzes database performance."""
def __init__(self, db_type: str, connection_string: str, threshold_ms: int = 100):
"""
Initialize performance analyzer.
Args:
db_type: Database type ('mongodb' or 'postgres')
connection_string: Database connection string
threshold_ms: Slow query threshold in milliseconds
"""
self.db_type = db_type.lower()
self.connection_string = connection_string
self.threshold_ms = threshold_ms
self.client = None
self.db = None
self.conn = None
def connect(self) -> bool:
"""Connect to database."""
try:
if self.db_type == "mongodb":
if not MONGO_AVAILABLE:
print("Error: pymongo not installed")
return False
self.client = MongoClient(self.connection_string)
self.db = self.client.get_default_database()
self.client.server_info()
return True
elif self.db_type == "postgres":
if not POSTGRES_AVAILABLE:
print("Error: psycopg2 not installed")
return False
self.conn = psycopg2.connect(self.connection_string)
return True
else:
print(f"Error: Unsupported database type: {self.db_type}")
return False
except Exception as e:
print(f"Connection error: {e}")
return False
def disconnect(self):
"""Disconnect from database."""
try:
if self.client:
self.client.close()
if self.conn:
self.conn.close()
except Exception as e:
print(f"Disconnect error: {e}")
def analyze(self) -> Optional[PerformanceReport]:
"""
Analyze database performance.
Returns:
PerformanceReport if successful, None otherwise
"""
try:
if self.db_type == "mongodb":
return self._analyze_mongodb()
elif self.db_type == "postgres":
return self._analyze_postgres()
else:
return None
except Exception as e:
print(f"Analysis error: {e}")
return None
def _analyze_mongodb(self) -> PerformanceReport:
"""Analyze MongoDB performance."""
slow_queries = []
index_recommendations = []
# Enable profiling if not enabled
profiling_level = self.db.command("profile", -1)
if profiling_level.get("was", 0) == 0:
self.db.command("profile", 1, slowms=self.threshold_ms)
# Get slow queries from system.profile
for doc in self.db.system.profile.find(
{"millis": {"$gte": self.threshold_ms}},
limit=50
).sort("millis", -1):
query_str = json.dumps(doc.get("command", {}), default=str)
slow_queries.append(SlowQuery(
query=query_str,
execution_time_ms=doc.get("millis", 0),
count=1,
collection_or_table=doc.get("ns", "").split(".")[-1] if "ns" in doc else None,
index_used=doc.get("planSummary")
))
# Analyze collections for index recommendations
for coll_name in self.db.list_collection_names():
if coll_name.startswith("system."):
continue
coll = self.db[coll_name]
# Check for collections scans
stats = coll.aggregate([
{"$collStats": {"storageStats": {}}}
]).next()
# Check if collection has indexes
indexes = list(coll.list_indexes())
if len(indexes) <= 1: # Only _id index
# Recommend indexes based on common patterns
# Sample documents to find frequently queried fields
sample = list(coll.find().limit(100))
if sample:
# Find fields that appear in most documents
field_freq = {}
for doc in sample:
for field in doc.keys():
if field != "_id":
field_freq[field] = field_freq.get(field, 0) + 1
# Recommend index on most common field
if field_freq:
top_field = max(field_freq.items(), key=lambda x: x[1])[0]
index_recommendations.append(IndexRecommendation(
collection_or_table=coll_name,
fields=[top_field],
reason="Frequently queried field without index",
estimated_benefit="High"
))
# Get database metrics
server_status = self.client.admin.command("serverStatus")
db_stats = self.db.command("dbStats")
metrics = {
"connections": server_status.get("connections", {}).get("current", 0),
"operations_per_sec": server_status.get("opcounters", {}).get("query", 0),
"database_size_mb": db_stats.get("dataSize", 0) / (1024 * 1024),
"index_size_mb": db_stats.get("indexSize", 0) / (1024 * 1024),
"collections": db_stats.get("collections", 0)
}
return PerformanceReport(
database_type="mongodb",
database_name=self.db.name,
timestamp=datetime.now(),
slow_queries=slow_queries[:10], # Top 10
index_recommendations=index_recommendations,
database_metrics=metrics
)
def _analyze_postgres(self) -> PerformanceReport:
"""Analyze PostgreSQL performance."""
slow_queries = []
index_recommendations = []
with self.conn.cursor(cursor_factory=RealDictCursor) as cur:
# Check if pg_stat_statements extension is available
cur.execute("""
SELECT EXISTS (
SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements'
) AS has_extension
""")
has_pg_stat_statements = cur.fetchone()["has_extension"]
if has_pg_stat_statements:
# Get slow queries from pg_stat_statements
cur.execute("""
SELECT
query,
mean_exec_time,
calls,
total_exec_time
FROM pg_stat_statements
WHERE mean_exec_time >= %s
ORDER BY mean_exec_time DESC
LIMIT 10
""", (self.threshold_ms,))
for row in cur.fetchall():
slow_queries.append(SlowQuery(
query=row["query"],
execution_time_ms=row["mean_exec_time"],
count=row["calls"]
))
# Find tables with sequential scans (potential index candidates)
cur.execute("""
SELECT
schemaname,
tablename,
seq_scan,
seq_tup_read,
idx_scan
FROM pg_stat_user_tables
WHERE seq_scan > 1000
AND (idx_scan IS NULL OR seq_scan > idx_scan * 2)
ORDER BY seq_tup_read DESC
LIMIT 10
""")
for row in cur.fetchall():
index_recommendations.append(IndexRecommendation(
collection_or_table=f"{row['schemaname']}.{row['tablename']}",
fields=["<analyze query patterns>"],
reason=f"High sequential scans ({row['seq_scan']}) vs index scans ({row['idx_scan'] or 0})",
estimated_benefit="High" if row["seq_tup_read"] > 100000 else "Medium"
))
# Find unused indexes
cur.execute("""
SELECT
schemaname,
tablename,
indexname,
idx_scan
FROM pg_stat_user_indexes
WHERE idx_scan = 0
AND indexname NOT LIKE '%_pkey'
ORDER BY pg_relation_size(indexrelid) DESC
""")
unused_indexes = []
for row in cur.fetchall():
unused_indexes.append(
f"{row['schemaname']}.{row['tablename']}.{row['indexname']}"
)
# Database metrics
cur.execute("""
SELECT
sum(numbackends) AS connections,
sum(xact_commit) AS commits,
sum(xact_rollback) AS rollbacks
FROM pg_stat_database
WHERE datname = current_database()
""")
stats = cur.fetchone()
cur.execute("""
SELECT pg_database_size(current_database()) AS db_size
""")
db_size = cur.fetchone()["db_size"]
cur.execute("""
SELECT
sum(heap_blks_hit) / NULLIF(sum(heap_blks_hit) + sum(heap_blks_read), 0) AS cache_hit_ratio
FROM pg_statio_user_tables
""")
cache_ratio = cur.fetchone()["cache_hit_ratio"] or 0
metrics = {
"connections": stats["connections"],
"commits": stats["commits"],
"rollbacks": stats["rollbacks"],
"database_size_mb": db_size / (1024 * 1024),
"cache_hit_ratio": float(cache_ratio),
"unused_indexes": unused_indexes
}
return PerformanceReport(
database_type="postgres",
database_name=self.conn.info.dbname,
timestamp=datetime.now(),
slow_queries=slow_queries,
index_recommendations=index_recommendations,
database_metrics=metrics
)
def print_report(self, report: PerformanceReport):
"""Print performance report."""
print("=" * 80)
print(f"Database Performance Report - {report.database_type.upper()}")
print(f"Database: {report.database_name}")
print(f"Timestamp: {report.timestamp}")
print("=" * 80)
print("\n## Database Metrics")
print("-" * 80)
for key, value in report.database_metrics.items():
if isinstance(value, float):
print(f"{key}: {value:.2f}")
else:
print(f"{key}: {value}")
print("\n## Slow Queries")
print("-" * 80)
if report.slow_queries:
for i, query in enumerate(report.slow_queries, 1):
print(f"\n{i}. Execution Time: {query.execution_time_ms:.2f}ms | Count: {query.count}")
if query.collection_or_table:
print(f" Collection/Table: {query.collection_or_table}")
if query.index_used:
print(f" Index Used: {query.index_used}")
print(f" Query: {query.query[:200]}...")
else:
print("No slow queries found")
print("\n## Index Recommendations")
print("-" * 80)
if report.index_recommendations:
for i, rec in enumerate(report.index_recommendations, 1):
print(f"\n{i}. {rec.collection_or_table}")
print(f" Fields: {', '.join(rec.fields)}")
print(f" Reason: {rec.reason}")
print(f" Estimated Benefit: {rec.estimated_benefit}")
if report.database_type == "mongodb":
index_spec = {field: 1 for field in rec.fields}
print(f" Command: db.{rec.collection_or_table}.createIndex({json.dumps(index_spec)})")
elif report.database_type == "postgres":
fields_str = ", ".join(rec.fields)
print(f" Command: CREATE INDEX idx_{rec.collection_or_table.replace('.', '_')}_{rec.fields[0]} ON {rec.collection_or_table}({fields_str});")
else:
print("No index recommendations")
print("\n" + "=" * 80)
def save_report(self, report: PerformanceReport, filename: str):
"""Save report to JSON file."""
# Convert dataclasses to dict
report_dict = {
"database_type": report.database_type,
"database_name": report.database_name,
"timestamp": report.timestamp.isoformat(),
"slow_queries": [asdict(q) for q in report.slow_queries],
"index_recommendations": [asdict(r) for r in report.index_recommendations],
"database_metrics": report.database_metrics
}
with open(filename, "w") as f:
json.dump(report_dict, f, indent=2, default=str)
print(f"\nReport saved to: {filename}")
def main():
"""Main entry point."""
parser = argparse.ArgumentParser(description="Database performance analysis tool")
parser.add_argument("--db", required=True, choices=["mongodb", "postgres"],
help="Database type")
parser.add_argument("--uri", required=True, help="Database connection string")
parser.add_argument("--threshold", type=int, default=100,
help="Slow query threshold in milliseconds (default: 100)")
parser.add_argument("--output", help="Save report to JSON file")
args = parser.parse_args()
analyzer = PerformanceAnalyzer(args.db, args.uri, args.threshold)
if not analyzer.connect():
sys.exit(1)
try:
print(f"Analyzing {args.db} performance (threshold: {args.threshold}ms)...")
report = analyzer.analyze()
if report:
analyzer.print_report(report)
if args.output:
analyzer.save_report(report, args.output)
sys.exit(0)
else:
print("Analysis failed")
sys.exit(1)
finally:
analyzer.disconnect()
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,20 @@
# Databases Skill Dependencies
# Python 3.10+ required
# No Python package dependencies - uses only standard library
# Testing dependencies (dev)
pytest>=8.0.0
pytest-cov>=4.1.0
pytest-mock>=3.12.0
# Note: This skill requires database CLI tools:
#
# PostgreSQL:
# - psql CLI (comes with PostgreSQL)
# - Ubuntu/Debian: sudo apt-get install postgresql-client
# - macOS: brew install postgresql
#
# MongoDB:
# - mongosh CLI: https://www.mongodb.com/try/download/shell
# - mongodump/mongorestore: https://www.mongodb.com/try/download/database-tools

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,4 @@
pytest>=7.0.0
pytest-cov>=4.0.0
pytest-mock>=3.10.0
mongomock>=4.1.0

View File

@@ -0,0 +1,340 @@
"""Tests for db_backup.py"""
import json
import sys
from datetime import datetime
from pathlib import Path
from unittest.mock import Mock, patch, MagicMock, call
import pytest
# Add parent directory to path
sys.path.insert(0, str(Path(__file__).parent.parent))
from db_backup import BackupInfo, BackupManager
@pytest.fixture
def temp_backup_dir(tmp_path):
"""Create temporary backup directory."""
backup_dir = tmp_path / "backups"
backup_dir.mkdir()
return str(backup_dir)
@pytest.fixture
def sample_backup_info():
"""Create sample backup info."""
return BackupInfo(
filename="test_backup_20250101_120000.dump",
database_type="mongodb",
database_name="testdb",
timestamp=datetime.now(),
size_bytes=1024000,
compressed=True,
verified=True
)
class TestBackupInfo:
"""Test BackupInfo dataclass."""
def test_backup_info_creation(self):
"""Test creating backup info object."""
info = BackupInfo(
filename="backup.dump",
database_type="mongodb",
database_name="mydb",
timestamp=datetime.now(),
size_bytes=1024,
compressed=False
)
assert info.filename == "backup.dump"
assert info.database_type == "mongodb"
assert info.database_name == "mydb"
assert info.size_bytes == 1024
assert not info.compressed
assert not info.verified
class TestBackupManager:
"""Test BackupManager class."""
def test_init(self, temp_backup_dir):
"""Test manager initialization."""
manager = BackupManager("mongodb", temp_backup_dir)
assert manager.db_type == "mongodb"
assert Path(temp_backup_dir).exists()
@patch('subprocess.run')
def test_backup_mongodb(self, mock_run, temp_backup_dir):
"""Test MongoDB backup creation."""
mock_run.return_value = Mock(returncode=0, stderr="")
manager = BackupManager("mongodb", temp_backup_dir)
backup_info = manager.create_backup(
"mongodb://localhost",
"testdb",
compress=False,
verify=False
)
assert backup_info is not None
assert backup_info.database_type == "mongodb"
assert backup_info.database_name == "testdb"
mock_run.assert_called_once()
@patch('subprocess.run')
def test_backup_postgres(self, mock_run, temp_backup_dir):
"""Test PostgreSQL backup creation."""
mock_run.return_value = Mock(returncode=0, stderr="")
manager = BackupManager("postgres", temp_backup_dir)
with patch('builtins.open', create=True) as mock_open:
mock_open.return_value.__enter__.return_value = MagicMock()
backup_info = manager.create_backup(
"postgresql://localhost/testdb",
"testdb",
compress=False,
verify=False
)
assert backup_info is not None
assert backup_info.database_type == "postgres"
assert backup_info.database_name == "testdb"
def test_backup_postgres_no_database(self, temp_backup_dir):
"""Test PostgreSQL backup without database name."""
manager = BackupManager("postgres", temp_backup_dir)
backup_info = manager.create_backup(
"postgresql://localhost",
database=None,
compress=False,
verify=False
)
assert backup_info is None
@patch('subprocess.run')
def test_backup_with_compression(self, mock_run, temp_backup_dir):
"""Test backup with compression."""
mock_run.return_value = Mock(returncode=0, stderr="")
manager = BackupManager("mongodb", temp_backup_dir)
with patch('shutil.make_archive') as mock_archive, \
patch('shutil.rmtree') as mock_rmtree:
backup_info = manager.create_backup(
"mongodb://localhost",
"testdb",
compress=True,
verify=False
)
assert backup_info is not None
assert backup_info.compressed
mock_archive.assert_called_once()
def test_save_and_load_metadata(self, temp_backup_dir, sample_backup_info):
"""Test saving and loading backup metadata."""
manager = BackupManager("mongodb", temp_backup_dir)
# Save metadata
manager._save_metadata(sample_backup_info)
# Check file was created
metadata_file = Path(temp_backup_dir) / f"{sample_backup_info.filename}.json"
assert metadata_file.exists()
# Load metadata
with open(metadata_file) as f:
data = json.load(f)
assert data["filename"] == sample_backup_info.filename
assert data["database_type"] == "mongodb"
assert data["database_name"] == "testdb"
def test_list_backups(self, temp_backup_dir, sample_backup_info):
"""Test listing backups."""
manager = BackupManager("mongodb", temp_backup_dir)
# Create test backup metadata
manager._save_metadata(sample_backup_info)
# List backups
backups = manager.list_backups()
assert len(backups) == 1
assert backups[0].filename == sample_backup_info.filename
assert backups[0].database_name == "testdb"
@patch('subprocess.run')
def test_restore_mongodb(self, mock_run, temp_backup_dir):
"""Test MongoDB restore."""
mock_run.return_value = Mock(returncode=0, stderr="")
manager = BackupManager("mongodb", temp_backup_dir)
# Create dummy backup file
backup_file = Path(temp_backup_dir) / "test_backup.dump"
backup_file.touch()
result = manager.restore_backup(
"test_backup.dump",
"mongodb://localhost"
)
assert result is True
mock_run.assert_called_once()
@patch('subprocess.run')
def test_restore_postgres(self, mock_run, temp_backup_dir):
"""Test PostgreSQL restore."""
mock_run.return_value = Mock(returncode=0, stderr="")
manager = BackupManager("postgres", temp_backup_dir)
# Create dummy backup file
backup_file = Path(temp_backup_dir) / "test_backup.sql"
backup_file.write_text("SELECT 1;")
with patch('builtins.open', create=True) as mock_open:
mock_open.return_value.__enter__.return_value = MagicMock()
result = manager.restore_backup(
"test_backup.sql",
"postgresql://localhost/testdb"
)
assert result is True
def test_restore_nonexistent_backup(self, temp_backup_dir):
"""Test restore with non-existent backup file."""
manager = BackupManager("mongodb", temp_backup_dir)
result = manager.restore_backup(
"nonexistent.dump",
"mongodb://localhost"
)
assert result is False
def test_restore_dry_run(self, temp_backup_dir):
"""Test restore in dry-run mode."""
manager = BackupManager("mongodb", temp_backup_dir)
# Create dummy backup file
backup_file = Path(temp_backup_dir) / "test_backup.dump"
backup_file.touch()
result = manager.restore_backup(
"test_backup.dump",
"mongodb://localhost",
dry_run=True
)
assert result is True
def test_cleanup_old_backups(self, temp_backup_dir):
"""Test cleaning up old backups."""
manager = BackupManager("mongodb", temp_backup_dir)
# Create old backup file (simulate by setting mtime)
old_backup = Path(temp_backup_dir) / "old_backup.dump"
old_backup.touch()
# Set mtime to 10 days ago
old_time = datetime.now().timestamp() - (10 * 24 * 3600)
os.utime(old_backup, (old_time, old_time))
# Cleanup with 7-day retention
removed = manager.cleanup_old_backups(retention_days=7)
assert removed == 1
assert not old_backup.exists()
def test_cleanup_dry_run(self, temp_backup_dir):
"""Test cleanup in dry-run mode."""
manager = BackupManager("mongodb", temp_backup_dir)
# Create old backup file
old_backup = Path(temp_backup_dir) / "old_backup.dump"
old_backup.touch()
old_time = datetime.now().timestamp() - (10 * 24 * 3600)
os.utime(old_backup, (old_time, old_time))
# Cleanup with dry-run
removed = manager.cleanup_old_backups(retention_days=7, dry_run=True)
assert removed == 1
assert old_backup.exists() # File should still exist
def test_verify_backup(self, temp_backup_dir, sample_backup_info):
"""Test backup verification."""
manager = BackupManager("mongodb", temp_backup_dir)
# Create dummy backup file
backup_file = Path(temp_backup_dir) / sample_backup_info.filename
backup_file.write_text("backup data")
result = manager._verify_backup(sample_backup_info)
assert result is True
def test_verify_empty_backup(self, temp_backup_dir, sample_backup_info):
"""Test verification of empty backup file."""
manager = BackupManager("mongodb", temp_backup_dir)
# Create empty backup file
backup_file = Path(temp_backup_dir) / sample_backup_info.filename
backup_file.touch()
result = manager._verify_backup(sample_backup_info)
assert result is False
def test_format_size(self, temp_backup_dir):
"""Test size formatting."""
manager = BackupManager("mongodb", temp_backup_dir)
assert manager._format_size(500) == "500.00 B"
assert manager._format_size(1024) == "1.00 KB"
assert manager._format_size(1024 * 1024) == "1.00 MB"
assert manager._format_size(1024 * 1024 * 1024) == "1.00 GB"
def test_get_size_file(self, temp_backup_dir):
"""Test getting size of file."""
manager = BackupManager("mongodb", temp_backup_dir)
test_file = Path(temp_backup_dir) / "test.txt"
test_file.write_text("test data")
size = manager._get_size(test_file)
assert size > 0
def test_get_size_directory(self, temp_backup_dir):
"""Test getting size of directory."""
manager = BackupManager("mongodb", temp_backup_dir)
test_dir = Path(temp_backup_dir) / "test_dir"
test_dir.mkdir()
(test_dir / "file1.txt").write_text("data1")
(test_dir / "file2.txt").write_text("data2")
size = manager._get_size(test_dir)
assert size > 0
# Import os for cleanup test
import os
if __name__ == "__main__":
pytest.main([__file__, "-v"])

View File

@@ -0,0 +1,277 @@
"""Tests for db_migrate.py"""
import json
import os
import sys
from datetime import datetime
from pathlib import Path
from unittest.mock import Mock, patch, MagicMock
import pytest
# Add parent directory to path
sys.path.insert(0, str(Path(__file__).parent.parent))
from db_migrate import Migration, MigrationManager
@pytest.fixture
def temp_migrations_dir(tmp_path):
"""Create temporary migrations directory."""
migrations_dir = tmp_path / "migrations"
migrations_dir.mkdir()
return str(migrations_dir)
@pytest.fixture
def mock_mongo_client():
"""Mock MongoDB client."""
mock_client = MagicMock()
mock_db = MagicMock()
mock_client.get_default_database.return_value = mock_db
mock_client.server_info.return_value = {}
return mock_client, mock_db
@pytest.fixture
def mock_postgres_conn():
"""Mock PostgreSQL connection."""
mock_conn = MagicMock()
mock_cursor = MagicMock()
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
return mock_conn, mock_cursor
class TestMigration:
"""Test Migration dataclass."""
def test_migration_creation(self):
"""Test creating migration object."""
migration = Migration(
id="20250101120000",
name="test_migration",
timestamp=datetime.now(),
database_type="mongodb"
)
assert migration.id == "20250101120000"
assert migration.name == "test_migration"
assert migration.database_type == "mongodb"
assert not migration.applied
class TestMigrationManager:
"""Test MigrationManager class."""
def test_init(self, temp_migrations_dir):
"""Test manager initialization."""
manager = MigrationManager("mongodb", "mongodb://localhost", temp_migrations_dir)
assert manager.db_type == "mongodb"
assert manager.connection_string == "mongodb://localhost"
assert Path(temp_migrations_dir).exists()
@patch('db_migrate.MongoClient')
def test_connect_mongodb(self, mock_client_class, temp_migrations_dir, mock_mongo_client):
"""Test MongoDB connection."""
mock_client, mock_db = mock_mongo_client
mock_client_class.return_value = mock_client
manager = MigrationManager("mongodb", "mongodb://localhost", temp_migrations_dir)
result = manager.connect()
assert result is True
assert manager.client == mock_client
assert manager.db == mock_db
@patch('db_migrate.psycopg2')
def test_connect_postgres(self, mock_psycopg2, temp_migrations_dir, mock_postgres_conn):
"""Test PostgreSQL connection."""
mock_conn, mock_cursor = mock_postgres_conn
mock_psycopg2.connect.return_value = mock_conn
manager = MigrationManager("postgres", "postgresql://localhost", temp_migrations_dir)
result = manager.connect()
assert result is True
assert manager.conn == mock_conn
def test_connect_unsupported_db(self, temp_migrations_dir):
"""Test connection with unsupported database type."""
manager = MigrationManager("unsupported", "connection_string", temp_migrations_dir)
result = manager.connect()
assert result is False
def test_generate_migration(self, temp_migrations_dir):
"""Test migration generation."""
manager = MigrationManager("mongodb", "mongodb://localhost", temp_migrations_dir)
migration = manager.generate_migration("test_migration")
assert migration is not None
assert migration.name == "test_migration"
# Check file was created
migration_files = list(Path(temp_migrations_dir).glob("*.json"))
assert len(migration_files) == 1
# Check file content
with open(migration_files[0]) as f:
data = json.load(f)
assert data["name"] == "test_migration"
assert data["database_type"] == "mongodb"
def test_generate_migration_dry_run(self, temp_migrations_dir):
"""Test migration generation in dry-run mode."""
manager = MigrationManager("postgres", "postgresql://localhost", temp_migrations_dir)
migration = manager.generate_migration("test_migration", dry_run=True)
assert migration is not None
# Check no file was created
migration_files = list(Path(temp_migrations_dir).glob("*.json"))
assert len(migration_files) == 0
def test_get_pending_migrations(self, temp_migrations_dir):
"""Test getting pending migrations."""
manager = MigrationManager("mongodb", "mongodb://localhost", temp_migrations_dir)
# Create test migration file
migration_data = {
"id": "20250101120000",
"name": "test_migration",
"timestamp": datetime.now().isoformat(),
"database_type": "mongodb",
"mongodb_operations": []
}
migration_file = Path(temp_migrations_dir) / "20250101120000_test.json"
with open(migration_file, "w") as f:
json.dump(migration_data, f)
# Mock database connection
with patch.object(manager, 'db', MagicMock()):
manager.db.migrations.find.return_value = []
pending = manager.get_pending_migrations()
assert len(pending) == 1
assert pending[0].id == "20250101120000"
assert pending[0].name == "test_migration"
@patch('db_migrate.MongoClient')
def test_apply_mongodb_migration(self, mock_client_class, temp_migrations_dir, mock_mongo_client):
"""Test applying MongoDB migration."""
mock_client, mock_db = mock_mongo_client
mock_client_class.return_value = mock_client
manager = MigrationManager("mongodb", "mongodb://localhost", temp_migrations_dir)
manager.connect()
migration = Migration(
id="20250101120000",
name="test_migration",
timestamp=datetime.now(),
database_type="mongodb",
mongodb_operations=[
{
"operation": "createIndex",
"collection": "users",
"index": {"email": 1},
"options": {}
}
]
)
result = manager.apply_migration(migration)
assert result is True
mock_db["users"].create_index.assert_called_once()
mock_db.migrations.insert_one.assert_called_once()
def test_apply_migration_dry_run(self, temp_migrations_dir):
"""Test applying migration in dry-run mode."""
manager = MigrationManager("mongodb", "mongodb://localhost", temp_migrations_dir)
migration = Migration(
id="20250101120000",
name="test_migration",
timestamp=datetime.now(),
database_type="mongodb",
mongodb_operations=[]
)
result = manager.apply_migration(migration, dry_run=True)
assert result is True
@patch('db_migrate.psycopg2')
def test_rollback_postgres_migration(self, mock_psycopg2, temp_migrations_dir, mock_postgres_conn):
"""Test rolling back PostgreSQL migration."""
mock_conn, mock_cursor = mock_postgres_conn
mock_psycopg2.connect.return_value = mock_conn
manager = MigrationManager("postgres", "postgresql://localhost", temp_migrations_dir)
manager.connect()
# Create migration file
migration_data = {
"id": "20250101120000",
"name": "test_migration",
"timestamp": datetime.now().isoformat(),
"database_type": "postgres",
"up_sql": "CREATE TABLE test (id INT);",
"down_sql": "DROP TABLE test;"
}
migration_file = Path(temp_migrations_dir) / "20250101120000_test.json"
with open(migration_file, "w") as f:
json.dump(migration_data, f)
result = manager.rollback_migration("20250101120000")
assert result is True
# Verify SQL was executed
assert mock_cursor.execute.call_count >= 1
def test_rollback_migration_not_found(self, temp_migrations_dir):
"""Test rollback with non-existent migration."""
manager = MigrationManager("mongodb", "mongodb://localhost", temp_migrations_dir)
result = manager.rollback_migration("99999999999999")
assert result is False
def test_migration_sorting(temp_migrations_dir):
"""Test that migrations are applied in correct order."""
manager = MigrationManager("mongodb", "mongodb://localhost", temp_migrations_dir)
# Create multiple migration files
for i in range(3):
migration_data = {
"id": f"2025010112000{i}",
"name": f"migration_{i}",
"timestamp": datetime.now().isoformat(),
"database_type": "mongodb",
"mongodb_operations": []
}
migration_file = Path(temp_migrations_dir) / f"2025010112000{i}_test.json"
with open(migration_file, "w") as f:
json.dump(migration_data, f)
with patch.object(manager, 'db', MagicMock()):
manager.db.migrations.find.return_value = []
pending = manager.get_pending_migrations()
# Check they're in order
assert len(pending) == 3
assert pending[0].id == "20250101120000"
assert pending[1].id == "20250101120001"
assert pending[2].id == "20250101120002"
if __name__ == "__main__":
pytest.main([__file__, "-v"])

View File

@@ -0,0 +1,370 @@
"""Tests for db_performance_check.py"""
import json
import sys
from datetime import datetime
from pathlib import Path
from unittest.mock import Mock, patch, MagicMock
import pytest
# Add parent directory to path
sys.path.insert(0, str(Path(__file__).parent.parent))
from db_performance_check import (
SlowQuery, IndexRecommendation, PerformanceReport, PerformanceAnalyzer
)
@pytest.fixture
def mock_mongo_client():
"""Mock MongoDB client."""
mock_client = MagicMock()
mock_db = MagicMock()
mock_client.get_default_database.return_value = mock_db
mock_client.server_info.return_value = {}
return mock_client, mock_db
@pytest.fixture
def mock_postgres_conn():
"""Mock PostgreSQL connection."""
mock_conn = MagicMock()
mock_cursor = MagicMock()
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
return mock_conn, mock_cursor
class TestSlowQuery:
"""Test SlowQuery dataclass."""
def test_slow_query_creation(self):
"""Test creating slow query object."""
query = SlowQuery(
query="SELECT * FROM users",
execution_time_ms=150.5,
count=10
)
assert query.query == "SELECT * FROM users"
assert query.execution_time_ms == 150.5
assert query.count == 10
class TestIndexRecommendation:
"""Test IndexRecommendation dataclass."""
def test_recommendation_creation(self):
"""Test creating index recommendation."""
rec = IndexRecommendation(
collection_or_table="users",
fields=["email"],
reason="Frequently queried field",
estimated_benefit="High"
)
assert rec.collection_or_table == "users"
assert rec.fields == ["email"]
assert rec.reason == "Frequently queried field"
assert rec.estimated_benefit == "High"
class TestPerformanceReport:
"""Test PerformanceReport dataclass."""
def test_report_creation(self):
"""Test creating performance report."""
report = PerformanceReport(
database_type="mongodb",
database_name="testdb",
timestamp=datetime.now(),
slow_queries=[],
index_recommendations=[],
database_metrics={}
)
assert report.database_type == "mongodb"
assert report.database_name == "testdb"
assert isinstance(report.slow_queries, list)
assert isinstance(report.index_recommendations, list)
assert isinstance(report.database_metrics, dict)
class TestPerformanceAnalyzer:
"""Test PerformanceAnalyzer class."""
def test_init(self):
"""Test analyzer initialization."""
analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost", 100)
assert analyzer.db_type == "mongodb"
assert analyzer.connection_string == "mongodb://localhost"
assert analyzer.threshold_ms == 100
@patch('db_performance_check.MongoClient')
def test_connect_mongodb(self, mock_client_class, mock_mongo_client):
"""Test MongoDB connection."""
mock_client, mock_db = mock_mongo_client
mock_client_class.return_value = mock_client
analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost")
result = analyzer.connect()
assert result is True
assert analyzer.client == mock_client
assert analyzer.db == mock_db
@patch('db_performance_check.psycopg2')
def test_connect_postgres(self, mock_psycopg2, mock_postgres_conn):
"""Test PostgreSQL connection."""
mock_conn, mock_cursor = mock_postgres_conn
mock_psycopg2.connect.return_value = mock_conn
analyzer = PerformanceAnalyzer("postgres", "postgresql://localhost")
result = analyzer.connect()
assert result is True
assert analyzer.conn == mock_conn
def test_connect_unsupported_db(self):
"""Test connection with unsupported database type."""
analyzer = PerformanceAnalyzer("unsupported", "connection_string")
result = analyzer.connect()
assert result is False
@patch('db_performance_check.MongoClient')
def test_analyze_mongodb(self, mock_client_class, mock_mongo_client):
"""Test MongoDB performance analysis."""
mock_client, mock_db = mock_mongo_client
mock_client_class.return_value = mock_client
# Mock profiling
mock_db.command.side_effect = [
{"was": 0}, # profile -1 (get status)
{}, # profile 1 (enable)
]
# Mock slow queries
mock_profile_cursor = MagicMock()
mock_profile_cursor.sort.return_value = [
{
"command": {"find": "users"},
"millis": 150,
"ns": "testdb.users",
"planSummary": "COLLSCAN"
}
]
mock_db.system.profile.find.return_value = mock_profile_cursor
# Mock collections
mock_db.list_collection_names.return_value = ["users", "orders"]
# Mock collection stats
mock_coll = MagicMock()
mock_coll.aggregate.return_value = [{"storageStats": {}}]
mock_coll.list_indexes.return_value = [{"name": "_id_"}]
mock_coll.find.return_value.limit.return_value = [
{"_id": 1, "name": "Alice", "email": "alice@example.com"}
]
mock_db.__getitem__.return_value = mock_coll
# Mock server status and db stats
mock_client.admin.command.return_value = {
"connections": {"current": 10},
"opcounters": {"query": 1000}
}
mock_db.command.return_value = {
"dataSize": 1024 * 1024 * 100,
"indexSize": 1024 * 1024 * 10,
"collections": 5
}
analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost")
analyzer.connect()
report = analyzer.analyze()
assert report is not None
assert report.database_type == "mongodb"
assert isinstance(report.slow_queries, list)
assert isinstance(report.index_recommendations, list)
assert isinstance(report.database_metrics, dict)
@patch('db_performance_check.psycopg2')
def test_analyze_postgres(self, mock_psycopg2, mock_postgres_conn):
"""Test PostgreSQL performance analysis."""
mock_conn, mock_cursor = mock_postgres_conn
mock_psycopg2.connect.return_value = mock_conn
# Mock cursor results
mock_cursor.fetchone.side_effect = [
{"has_extension": True}, # pg_stat_statements check
{"connections": 10, "commits": 1000, "rollbacks": 5}, # stats
{"db_size": 1024 * 1024 * 500}, # database size
{"cache_hit_ratio": 0.95} # cache hit ratio
]
mock_cursor.fetchall.side_effect = [
# Slow queries
[
{
"query": "SELECT * FROM users",
"mean_exec_time": 150.5,
"calls": 100,
"total_exec_time": 15050
}
],
# Sequential scans
[
{
"schemaname": "public",
"tablename": "users",
"seq_scan": 5000,
"seq_tup_read": 500000,
"idx_scan": 100
}
],
# Unused indexes
[]
]
analyzer = PerformanceAnalyzer("postgres", "postgresql://localhost")
analyzer.connect()
report = analyzer.analyze()
assert report is not None
assert report.database_type == "postgres"
assert len(report.slow_queries) > 0
assert len(report.index_recommendations) > 0
def test_print_report(self, capsys):
"""Test report printing."""
analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost")
report = PerformanceReport(
database_type="mongodb",
database_name="testdb",
timestamp=datetime.now(),
slow_queries=[
SlowQuery(
query="db.users.find({age: {$gte: 18}})",
execution_time_ms=150.5,
count=10,
collection_or_table="users"
)
],
index_recommendations=[
IndexRecommendation(
collection_or_table="users",
fields=["age"],
reason="Frequently queried field",
estimated_benefit="High"
)
],
database_metrics={
"connections": 10,
"database_size_mb": 100.5
}
)
analyzer.print_report(report)
captured = capsys.readouterr()
assert "Database Performance Report" in captured.out
assert "testdb" in captured.out
assert "150.5ms" in captured.out
assert "users" in captured.out
def test_save_report(self, tmp_path):
"""Test saving report to JSON."""
analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost")
report = PerformanceReport(
database_type="mongodb",
database_name="testdb",
timestamp=datetime.now(),
slow_queries=[],
index_recommendations=[],
database_metrics={}
)
output_file = tmp_path / "report.json"
analyzer.save_report(report, str(output_file))
assert output_file.exists()
with open(output_file) as f:
data = json.load(f)
assert data["database_type"] == "mongodb"
assert data["database_name"] == "testdb"
def test_disconnect(self):
"""Test disconnection."""
analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost")
# Mock client and connection
analyzer.client = MagicMock()
analyzer.conn = MagicMock()
analyzer.disconnect()
analyzer.client.close.assert_called_once()
analyzer.conn.close.assert_called_once()
@patch('db_performance_check.MongoClient')
def test_analyze_error_handling(self, mock_client_class, mock_mongo_client):
"""Test error handling during analysis."""
mock_client, mock_db = mock_mongo_client
mock_client_class.return_value = mock_client
# Simulate error
mock_db.command.side_effect = Exception("Database error")
analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost")
analyzer.connect()
report = analyzer.analyze()
assert report is None
class TestIntegration:
"""Integration tests."""
@patch('db_performance_check.MongoClient')
def test_full_mongodb_workflow(self, mock_client_class, mock_mongo_client, tmp_path):
"""Test complete MongoDB analysis workflow."""
mock_client, mock_db = mock_mongo_client
mock_client_class.return_value = mock_client
# Setup mocks
mock_db.command.return_value = {"was": 0}
mock_db.system.profile.find.return_value.sort.return_value = []
mock_db.list_collection_names.return_value = []
mock_client.admin.command.return_value = {
"connections": {"current": 10},
"opcounters": {"query": 1000}
}
analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost", 100)
# Connect
assert analyzer.connect() is True
# Analyze
report = analyzer.analyze()
assert report is not None
# Save report
output_file = tmp_path / "report.json"
analyzer.save_report(report, str(output_file))
assert output_file.exists()
# Disconnect
analyzer.disconnect()
if __name__ == "__main__":
pytest.main([__file__, "-v"])