Initial commit
This commit is contained in:
502
skills/databases/scripts/db_backup.py
Normal file
502
skills/databases/scripts/db_backup.py
Normal file
@@ -0,0 +1,502 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Database backup and restore tool for MongoDB and PostgreSQL.
|
||||
Supports compression, scheduling, and verification.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class BackupInfo:
|
||||
"""Backup metadata."""
|
||||
|
||||
filename: str
|
||||
database_type: str
|
||||
database_name: str
|
||||
timestamp: datetime
|
||||
size_bytes: int
|
||||
compressed: bool
|
||||
verified: bool = False
|
||||
|
||||
|
||||
class BackupManager:
|
||||
"""Manages database backups for MongoDB and PostgreSQL."""
|
||||
|
||||
def __init__(self, db_type: str, backup_dir: str = "./backups"):
|
||||
"""
|
||||
Initialize backup manager.
|
||||
|
||||
Args:
|
||||
db_type: Database type ('mongodb' or 'postgres')
|
||||
backup_dir: Directory to store backups
|
||||
"""
|
||||
self.db_type = db_type.lower()
|
||||
self.backup_dir = Path(backup_dir)
|
||||
self.backup_dir.mkdir(exist_ok=True)
|
||||
|
||||
def create_backup(
|
||||
self,
|
||||
uri: str,
|
||||
database: Optional[str] = None,
|
||||
compress: bool = True,
|
||||
verify: bool = True
|
||||
) -> Optional[BackupInfo]:
|
||||
"""
|
||||
Create database backup.
|
||||
|
||||
Args:
|
||||
uri: Database connection string
|
||||
database: Database name (optional for MongoDB)
|
||||
compress: Compress backup file
|
||||
verify: Verify backup after creation
|
||||
|
||||
Returns:
|
||||
BackupInfo if successful, None otherwise
|
||||
"""
|
||||
timestamp = datetime.now()
|
||||
date_str = timestamp.strftime("%Y%m%d_%H%M%S")
|
||||
|
||||
if self.db_type == "mongodb":
|
||||
return self._backup_mongodb(uri, database, date_str, compress, verify)
|
||||
elif self.db_type == "postgres":
|
||||
return self._backup_postgres(uri, database, date_str, compress, verify)
|
||||
else:
|
||||
print(f"Error: Unsupported database type: {self.db_type}")
|
||||
return None
|
||||
|
||||
def _backup_mongodb(
|
||||
self,
|
||||
uri: str,
|
||||
database: Optional[str],
|
||||
date_str: str,
|
||||
compress: bool,
|
||||
verify: bool
|
||||
) -> Optional[BackupInfo]:
|
||||
"""Create MongoDB backup using mongodump."""
|
||||
db_name = database or "all"
|
||||
filename = f"mongodb_{db_name}_{date_str}"
|
||||
backup_path = self.backup_dir / filename
|
||||
|
||||
try:
|
||||
cmd = ["mongodump", "--uri", uri, "--out", str(backup_path)]
|
||||
|
||||
if database:
|
||||
cmd.extend(["--db", database])
|
||||
|
||||
print(f"Creating MongoDB backup: {filename}")
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
|
||||
if result.returncode != 0:
|
||||
print(f"Error: {result.stderr}")
|
||||
return None
|
||||
|
||||
# Compress if requested
|
||||
if compress:
|
||||
archive_path = backup_path.with_suffix(".tar.gz")
|
||||
print(f"Compressing backup...")
|
||||
shutil.make_archive(str(backup_path), "gztar", backup_path)
|
||||
shutil.rmtree(backup_path)
|
||||
backup_path = archive_path
|
||||
filename = archive_path.name
|
||||
|
||||
size_bytes = self._get_size(backup_path)
|
||||
|
||||
backup_info = BackupInfo(
|
||||
filename=filename,
|
||||
database_type="mongodb",
|
||||
database_name=db_name,
|
||||
timestamp=datetime.now(),
|
||||
size_bytes=size_bytes,
|
||||
compressed=compress
|
||||
)
|
||||
|
||||
if verify:
|
||||
backup_info.verified = self._verify_backup(backup_info)
|
||||
|
||||
self._save_metadata(backup_info)
|
||||
print(f"✓ Backup created: {filename} ({self._format_size(size_bytes)})")
|
||||
|
||||
return backup_info
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error creating MongoDB backup: {e}")
|
||||
return None
|
||||
|
||||
def _backup_postgres(
|
||||
self,
|
||||
uri: str,
|
||||
database: str,
|
||||
date_str: str,
|
||||
compress: bool,
|
||||
verify: bool
|
||||
) -> Optional[BackupInfo]:
|
||||
"""Create PostgreSQL backup using pg_dump."""
|
||||
if not database:
|
||||
print("Error: Database name required for PostgreSQL backup")
|
||||
return None
|
||||
|
||||
ext = ".sql.gz" if compress else ".sql"
|
||||
filename = f"postgres_{database}_{date_str}{ext}"
|
||||
backup_path = self.backup_dir / filename
|
||||
|
||||
try:
|
||||
cmd = ["pg_dump", uri]
|
||||
|
||||
if compress:
|
||||
# Use pg_dump with gzip
|
||||
with open(backup_path, "wb") as f:
|
||||
dump_proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
|
||||
gzip_proc = subprocess.Popen(
|
||||
["gzip"],
|
||||
stdin=dump_proc.stdout,
|
||||
stdout=f
|
||||
)
|
||||
dump_proc.stdout.close()
|
||||
gzip_proc.communicate()
|
||||
|
||||
if dump_proc.returncode != 0:
|
||||
print("Error: pg_dump failed")
|
||||
return None
|
||||
else:
|
||||
with open(backup_path, "w") as f:
|
||||
result = subprocess.run(cmd, stdout=f, stderr=subprocess.PIPE, text=True)
|
||||
|
||||
if result.returncode != 0:
|
||||
print(f"Error: {result.stderr}")
|
||||
return None
|
||||
|
||||
size_bytes = backup_path.stat().st_size
|
||||
|
||||
backup_info = BackupInfo(
|
||||
filename=filename,
|
||||
database_type="postgres",
|
||||
database_name=database,
|
||||
timestamp=datetime.now(),
|
||||
size_bytes=size_bytes,
|
||||
compressed=compress
|
||||
)
|
||||
|
||||
if verify:
|
||||
backup_info.verified = self._verify_backup(backup_info)
|
||||
|
||||
self._save_metadata(backup_info)
|
||||
print(f"✓ Backup created: {filename} ({self._format_size(size_bytes)})")
|
||||
|
||||
return backup_info
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error creating PostgreSQL backup: {e}")
|
||||
return None
|
||||
|
||||
def restore_backup(self, filename: str, uri: str, dry_run: bool = False) -> bool:
|
||||
"""
|
||||
Restore database from backup.
|
||||
|
||||
Args:
|
||||
filename: Backup filename
|
||||
uri: Database connection string
|
||||
dry_run: If True, only show what would be done
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
backup_path = self.backup_dir / filename
|
||||
|
||||
if not backup_path.exists():
|
||||
print(f"Error: Backup not found: {filename}")
|
||||
return False
|
||||
|
||||
# Load metadata
|
||||
metadata_path = backup_path.with_suffix(".json")
|
||||
if metadata_path.exists():
|
||||
with open(metadata_path) as f:
|
||||
metadata = json.load(f)
|
||||
print(f"Restoring backup from {metadata['timestamp']}")
|
||||
print(f"Database: {metadata['database_name']}")
|
||||
|
||||
if dry_run:
|
||||
print(f"Would restore from: {backup_path}")
|
||||
return True
|
||||
|
||||
print(f"Restoring backup: {filename}")
|
||||
|
||||
try:
|
||||
if self.db_type == "mongodb":
|
||||
return self._restore_mongodb(backup_path, uri)
|
||||
elif self.db_type == "postgres":
|
||||
return self._restore_postgres(backup_path, uri)
|
||||
else:
|
||||
print(f"Error: Unsupported database type: {self.db_type}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error restoring backup: {e}")
|
||||
return False
|
||||
|
||||
def _restore_mongodb(self, backup_path: Path, uri: str) -> bool:
|
||||
"""Restore MongoDB backup using mongorestore."""
|
||||
try:
|
||||
# Extract if compressed
|
||||
restore_path = backup_path
|
||||
if backup_path.suffix == ".gz":
|
||||
print("Extracting backup...")
|
||||
extract_path = backup_path.with_suffix("")
|
||||
shutil.unpack_archive(backup_path, extract_path)
|
||||
restore_path = extract_path
|
||||
|
||||
cmd = ["mongorestore", "--uri", uri, str(restore_path)]
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
|
||||
# Cleanup extracted files
|
||||
if restore_path != backup_path and restore_path.is_dir():
|
||||
shutil.rmtree(restore_path)
|
||||
|
||||
if result.returncode != 0:
|
||||
print(f"Error: {result.stderr}")
|
||||
return False
|
||||
|
||||
print("✓ Restore completed")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error restoring MongoDB: {e}")
|
||||
return False
|
||||
|
||||
def _restore_postgres(self, backup_path: Path, uri: str) -> bool:
|
||||
"""Restore PostgreSQL backup using psql."""
|
||||
try:
|
||||
if backup_path.suffix == ".gz":
|
||||
# Decompress and restore
|
||||
with gzip.open(backup_path, "rb") as f:
|
||||
cmd = ["psql", uri]
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
stdin=f,
|
||||
capture_output=True,
|
||||
text=False
|
||||
)
|
||||
else:
|
||||
with open(backup_path) as f:
|
||||
cmd = ["psql", uri]
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
stdin=f,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
print(f"Error: {result.stderr}")
|
||||
return False
|
||||
|
||||
print("✓ Restore completed")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error restoring PostgreSQL: {e}")
|
||||
return False
|
||||
|
||||
def list_backups(self) -> List[BackupInfo]:
|
||||
"""
|
||||
List all backups.
|
||||
|
||||
Returns:
|
||||
List of BackupInfo objects
|
||||
"""
|
||||
backups = []
|
||||
|
||||
for metadata_file in sorted(self.backup_dir.glob("*.json")):
|
||||
try:
|
||||
with open(metadata_file) as f:
|
||||
data = json.load(f)
|
||||
|
||||
backup_info = BackupInfo(
|
||||
filename=data["filename"],
|
||||
database_type=data["database_type"],
|
||||
database_name=data["database_name"],
|
||||
timestamp=datetime.fromisoformat(data["timestamp"]),
|
||||
size_bytes=data["size_bytes"],
|
||||
compressed=data["compressed"],
|
||||
verified=data.get("verified", False)
|
||||
)
|
||||
backups.append(backup_info)
|
||||
except Exception as e:
|
||||
print(f"Error reading metadata {metadata_file}: {e}")
|
||||
|
||||
return backups
|
||||
|
||||
def cleanup_old_backups(self, retention_days: int, dry_run: bool = False) -> int:
|
||||
"""
|
||||
Remove backups older than retention period.
|
||||
|
||||
Args:
|
||||
retention_days: Number of days to retain backups
|
||||
dry_run: If True, only show what would be deleted
|
||||
|
||||
Returns:
|
||||
Number of backups removed
|
||||
"""
|
||||
cutoff = datetime.now().timestamp() - (retention_days * 24 * 3600)
|
||||
removed = 0
|
||||
|
||||
for backup_file in self.backup_dir.glob("*"):
|
||||
if backup_file.suffix == ".json":
|
||||
continue
|
||||
|
||||
if backup_file.stat().st_mtime < cutoff:
|
||||
if dry_run:
|
||||
print(f"Would remove: {backup_file.name}")
|
||||
else:
|
||||
print(f"Removing: {backup_file.name}")
|
||||
backup_file.unlink()
|
||||
# Remove metadata
|
||||
metadata_file = backup_file.with_suffix(".json")
|
||||
if metadata_file.exists():
|
||||
metadata_file.unlink()
|
||||
removed += 1
|
||||
|
||||
return removed
|
||||
|
||||
def _verify_backup(self, backup_info: BackupInfo) -> bool:
|
||||
"""
|
||||
Verify backup integrity.
|
||||
|
||||
Args:
|
||||
backup_info: Backup information
|
||||
|
||||
Returns:
|
||||
True if backup is valid, False otherwise
|
||||
"""
|
||||
backup_path = self.backup_dir / backup_info.filename
|
||||
|
||||
if not backup_path.exists():
|
||||
return False
|
||||
|
||||
# Basic verification: file exists and has size > 0
|
||||
if backup_path.stat().st_size == 0:
|
||||
return False
|
||||
|
||||
# Could add more verification here (checksums, test restore, etc.)
|
||||
return True
|
||||
|
||||
def _get_size(self, path: Path) -> int:
|
||||
"""Get total size of file or directory."""
|
||||
if path.is_file():
|
||||
return path.stat().st_size
|
||||
elif path.is_dir():
|
||||
total = 0
|
||||
for item in path.rglob("*"):
|
||||
if item.is_file():
|
||||
total += item.stat().st_size
|
||||
return total
|
||||
return 0
|
||||
|
||||
def _format_size(self, size_bytes: int) -> str:
|
||||
"""Format size in human-readable format."""
|
||||
for unit in ["B", "KB", "MB", "GB", "TB"]:
|
||||
if size_bytes < 1024:
|
||||
return f"{size_bytes:.2f} {unit}"
|
||||
size_bytes /= 1024
|
||||
return f"{size_bytes:.2f} PB"
|
||||
|
||||
def _save_metadata(self, backup_info: BackupInfo):
|
||||
"""Save backup metadata to JSON file."""
|
||||
metadata_path = self.backup_dir / f"{backup_info.filename}.json"
|
||||
|
||||
metadata = {
|
||||
"filename": backup_info.filename,
|
||||
"database_type": backup_info.database_type,
|
||||
"database_name": backup_info.database_name,
|
||||
"timestamp": backup_info.timestamp.isoformat(),
|
||||
"size_bytes": backup_info.size_bytes,
|
||||
"compressed": backup_info.compressed,
|
||||
"verified": backup_info.verified
|
||||
}
|
||||
|
||||
with open(metadata_path, "w") as f:
|
||||
json.dump(metadata, f, indent=2)
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point."""
|
||||
parser = argparse.ArgumentParser(description="Database backup tool")
|
||||
parser.add_argument("--db", required=True, choices=["mongodb", "postgres"],
|
||||
help="Database type")
|
||||
parser.add_argument("--backup-dir", default="./backups",
|
||||
help="Backup directory")
|
||||
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
# Backup command
|
||||
backup_parser = subparsers.add_parser("backup", help="Create backup")
|
||||
backup_parser.add_argument("--uri", required=True, help="Database connection string")
|
||||
backup_parser.add_argument("--database", help="Database name")
|
||||
backup_parser.add_argument("--no-compress", action="store_true",
|
||||
help="Disable compression")
|
||||
backup_parser.add_argument("--no-verify", action="store_true",
|
||||
help="Skip verification")
|
||||
|
||||
# Restore command
|
||||
restore_parser = subparsers.add_parser("restore", help="Restore backup")
|
||||
restore_parser.add_argument("filename", help="Backup filename")
|
||||
restore_parser.add_argument("--uri", required=True, help="Database connection string")
|
||||
restore_parser.add_argument("--dry-run", action="store_true",
|
||||
help="Show what would be done")
|
||||
|
||||
# List command
|
||||
subparsers.add_parser("list", help="List backups")
|
||||
|
||||
# Cleanup command
|
||||
cleanup_parser = subparsers.add_parser("cleanup", help="Remove old backups")
|
||||
cleanup_parser.add_argument("--retention-days", type=int, default=7,
|
||||
help="Days to retain backups (default: 7)")
|
||||
cleanup_parser.add_argument("--dry-run", action="store_true",
|
||||
help="Show what would be removed")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
manager = BackupManager(args.db, args.backup_dir)
|
||||
|
||||
if args.command == "backup":
|
||||
backup_info = manager.create_backup(
|
||||
args.uri,
|
||||
args.database,
|
||||
compress=not args.no_compress,
|
||||
verify=not args.no_verify
|
||||
)
|
||||
sys.exit(0 if backup_info else 1)
|
||||
|
||||
elif args.command == "restore":
|
||||
success = manager.restore_backup(args.filename, args.uri, args.dry_run)
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
elif args.command == "list":
|
||||
backups = manager.list_backups()
|
||||
print(f"Total backups: {len(backups)}\n")
|
||||
for backup in backups:
|
||||
verified_str = "✓" if backup.verified else "?"
|
||||
print(f"[{verified_str}] {backup.filename}")
|
||||
print(f" Database: {backup.database_name}")
|
||||
print(f" Created: {backup.timestamp}")
|
||||
print(f" Size: {manager._format_size(backup.size_bytes)}")
|
||||
print()
|
||||
|
||||
elif args.command == "cleanup":
|
||||
removed = manager.cleanup_old_backups(args.retention_days, args.dry_run)
|
||||
print(f"Removed {removed} backup(s)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
414
skills/databases/scripts/db_migrate.py
Normal file
414
skills/databases/scripts/db_migrate.py
Normal file
@@ -0,0 +1,414 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Database migration tool for MongoDB and PostgreSQL.
|
||||
Generates and applies schema migrations with rollback support.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
try:
|
||||
from pymongo import MongoClient
|
||||
MONGO_AVAILABLE = True
|
||||
except ImportError:
|
||||
MONGO_AVAILABLE = False
|
||||
|
||||
try:
|
||||
import psycopg2
|
||||
from psycopg2 import sql
|
||||
POSTGRES_AVAILABLE = True
|
||||
except ImportError:
|
||||
POSTGRES_AVAILABLE = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class Migration:
|
||||
"""Represents a database migration."""
|
||||
|
||||
id: str
|
||||
name: str
|
||||
timestamp: datetime
|
||||
database_type: str
|
||||
up_sql: Optional[str] = None
|
||||
down_sql: Optional[str] = None
|
||||
mongodb_operations: Optional[List[Dict[str, Any]]] = None
|
||||
applied: bool = False
|
||||
|
||||
|
||||
class MigrationManager:
|
||||
"""Manages database migrations for MongoDB and PostgreSQL."""
|
||||
|
||||
def __init__(self, db_type: str, connection_string: str, migrations_dir: str = "./migrations"):
|
||||
"""
|
||||
Initialize migration manager.
|
||||
|
||||
Args:
|
||||
db_type: Database type ('mongodb' or 'postgres')
|
||||
connection_string: Database connection string
|
||||
migrations_dir: Directory to store migration files
|
||||
"""
|
||||
self.db_type = db_type.lower()
|
||||
self.connection_string = connection_string
|
||||
self.migrations_dir = Path(migrations_dir)
|
||||
self.migrations_dir.mkdir(exist_ok=True)
|
||||
|
||||
self.client = None
|
||||
self.db = None
|
||||
self.conn = None
|
||||
|
||||
def connect(self) -> bool:
|
||||
"""
|
||||
Connect to database.
|
||||
|
||||
Returns:
|
||||
True if connection successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
if self.db_type == "mongodb":
|
||||
if not MONGO_AVAILABLE:
|
||||
print("Error: pymongo not installed")
|
||||
return False
|
||||
self.client = MongoClient(self.connection_string)
|
||||
self.db = self.client.get_default_database()
|
||||
# Test connection
|
||||
self.client.server_info()
|
||||
return True
|
||||
|
||||
elif self.db_type == "postgres":
|
||||
if not POSTGRES_AVAILABLE:
|
||||
print("Error: psycopg2 not installed")
|
||||
return False
|
||||
self.conn = psycopg2.connect(self.connection_string)
|
||||
return True
|
||||
|
||||
else:
|
||||
print(f"Error: Unsupported database type: {self.db_type}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
print(f"Connection error: {e}")
|
||||
return False
|
||||
|
||||
def disconnect(self):
|
||||
"""Disconnect from database."""
|
||||
try:
|
||||
if self.client:
|
||||
self.client.close()
|
||||
if self.conn:
|
||||
self.conn.close()
|
||||
except Exception as e:
|
||||
print(f"Disconnect error: {e}")
|
||||
|
||||
def _ensure_migrations_table(self):
|
||||
"""Create migrations tracking table/collection if not exists."""
|
||||
if self.db_type == "mongodb":
|
||||
# MongoDB creates collection automatically
|
||||
pass
|
||||
elif self.db_type == "postgres":
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS migrations (
|
||||
id VARCHAR(255) PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
self.conn.commit()
|
||||
|
||||
def generate_migration(self, name: str, dry_run: bool = False) -> Optional[Migration]:
|
||||
"""
|
||||
Generate new migration file.
|
||||
|
||||
Args:
|
||||
name: Migration name
|
||||
dry_run: If True, only show what would be generated
|
||||
|
||||
Returns:
|
||||
Migration object if successful, None otherwise
|
||||
"""
|
||||
timestamp = datetime.now()
|
||||
migration_id = timestamp.strftime("%Y%m%d%H%M%S")
|
||||
filename = f"{migration_id}_{name}.json"
|
||||
filepath = self.migrations_dir / filename
|
||||
|
||||
migration = Migration(
|
||||
id=migration_id,
|
||||
name=name,
|
||||
timestamp=timestamp,
|
||||
database_type=self.db_type
|
||||
)
|
||||
|
||||
if self.db_type == "mongodb":
|
||||
migration.mongodb_operations = [
|
||||
{
|
||||
"operation": "createIndex",
|
||||
"collection": "example_collection",
|
||||
"index": {"field": 1},
|
||||
"options": {}
|
||||
}
|
||||
]
|
||||
elif self.db_type == "postgres":
|
||||
migration.up_sql = "-- Add your SQL here\n"
|
||||
migration.down_sql = "-- Add rollback SQL here\n"
|
||||
|
||||
migration_data = {
|
||||
"id": migration.id,
|
||||
"name": migration.name,
|
||||
"timestamp": migration.timestamp.isoformat(),
|
||||
"database_type": migration.database_type,
|
||||
"up_sql": migration.up_sql,
|
||||
"down_sql": migration.down_sql,
|
||||
"mongodb_operations": migration.mongodb_operations
|
||||
}
|
||||
|
||||
if dry_run:
|
||||
print(f"Would create: {filepath}")
|
||||
print(json.dumps(migration_data, indent=2))
|
||||
return migration
|
||||
|
||||
try:
|
||||
with open(filepath, "w") as f:
|
||||
json.dump(migration_data, f, indent=2)
|
||||
print(f"Created migration: {filepath}")
|
||||
return migration
|
||||
except Exception as e:
|
||||
print(f"Error creating migration: {e}")
|
||||
return None
|
||||
|
||||
def get_pending_migrations(self) -> List[Migration]:
|
||||
"""
|
||||
Get list of pending migrations.
|
||||
|
||||
Returns:
|
||||
List of pending Migration objects
|
||||
"""
|
||||
# Get applied migrations
|
||||
applied_ids = set()
|
||||
|
||||
try:
|
||||
if self.db_type == "mongodb":
|
||||
applied_ids = {
|
||||
doc["id"] for doc in self.db.migrations.find({}, {"id": 1})
|
||||
}
|
||||
elif self.db_type == "postgres":
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute("SELECT id FROM migrations")
|
||||
applied_ids = {row[0] for row in cur.fetchall()}
|
||||
except Exception as e:
|
||||
print(f"Error reading applied migrations: {e}")
|
||||
|
||||
# Get all migration files
|
||||
pending = []
|
||||
for filepath in sorted(self.migrations_dir.glob("*.json")):
|
||||
try:
|
||||
with open(filepath) as f:
|
||||
data = json.load(f)
|
||||
|
||||
if data["id"] not in applied_ids:
|
||||
migration = Migration(
|
||||
id=data["id"],
|
||||
name=data["name"],
|
||||
timestamp=datetime.fromisoformat(data["timestamp"]),
|
||||
database_type=data["database_type"],
|
||||
up_sql=data.get("up_sql"),
|
||||
down_sql=data.get("down_sql"),
|
||||
mongodb_operations=data.get("mongodb_operations")
|
||||
)
|
||||
pending.append(migration)
|
||||
except Exception as e:
|
||||
print(f"Error reading {filepath}: {e}")
|
||||
|
||||
return pending
|
||||
|
||||
def apply_migration(self, migration: Migration, dry_run: bool = False) -> bool:
|
||||
"""
|
||||
Apply migration.
|
||||
|
||||
Args:
|
||||
migration: Migration to apply
|
||||
dry_run: If True, only show what would be executed
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
print(f"Applying migration: {migration.id} - {migration.name}")
|
||||
|
||||
if dry_run:
|
||||
if self.db_type == "mongodb":
|
||||
print("MongoDB operations:")
|
||||
print(json.dumps(migration.mongodb_operations, indent=2))
|
||||
elif self.db_type == "postgres":
|
||||
print("SQL to execute:")
|
||||
print(migration.up_sql)
|
||||
return True
|
||||
|
||||
try:
|
||||
if self.db_type == "mongodb":
|
||||
for op in migration.mongodb_operations or []:
|
||||
if op["operation"] == "createIndex":
|
||||
self.db[op["collection"]].create_index(
|
||||
list(op["index"].items()),
|
||||
**op.get("options", {})
|
||||
)
|
||||
|
||||
# Record migration
|
||||
self.db.migrations.insert_one({
|
||||
"id": migration.id,
|
||||
"name": migration.name,
|
||||
"applied_at": datetime.now()
|
||||
})
|
||||
|
||||
elif self.db_type == "postgres":
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute(migration.up_sql)
|
||||
|
||||
# Record migration
|
||||
cur.execute(
|
||||
"INSERT INTO migrations (id, name) VALUES (%s, %s)",
|
||||
(migration.id, migration.name)
|
||||
)
|
||||
self.conn.commit()
|
||||
|
||||
print(f"✓ Applied: {migration.id}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"✗ Error applying migration: {e}")
|
||||
if self.conn:
|
||||
self.conn.rollback()
|
||||
return False
|
||||
|
||||
def rollback_migration(self, migration_id: str, dry_run: bool = False) -> bool:
|
||||
"""
|
||||
Rollback migration.
|
||||
|
||||
Args:
|
||||
migration_id: Migration ID to rollback
|
||||
dry_run: If True, only show what would be executed
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
# Find migration file
|
||||
migration_file = None
|
||||
for filepath in self.migrations_dir.glob(f"{migration_id}_*.json"):
|
||||
migration_file = filepath
|
||||
break
|
||||
|
||||
if not migration_file:
|
||||
print(f"Migration not found: {migration_id}")
|
||||
return False
|
||||
|
||||
try:
|
||||
with open(migration_file) as f:
|
||||
data = json.load(f)
|
||||
|
||||
print(f"Rolling back: {migration_id} - {data['name']}")
|
||||
|
||||
if dry_run:
|
||||
if self.db_type == "postgres":
|
||||
print("SQL to execute:")
|
||||
print(data.get("down_sql", "-- No rollback defined"))
|
||||
return True
|
||||
|
||||
if self.db_type == "postgres" and data.get("down_sql"):
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute(data["down_sql"])
|
||||
cur.execute("DELETE FROM migrations WHERE id = %s", (migration_id,))
|
||||
self.conn.commit()
|
||||
elif self.db_type == "mongodb":
|
||||
self.db.migrations.delete_one({"id": migration_id})
|
||||
|
||||
print(f"✓ Rolled back: {migration_id}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"✗ Error rolling back: {e}")
|
||||
if self.conn:
|
||||
self.conn.rollback()
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point."""
|
||||
parser = argparse.ArgumentParser(description="Database migration tool")
|
||||
parser.add_argument("--db", required=True, choices=["mongodb", "postgres"],
|
||||
help="Database type")
|
||||
parser.add_argument("--uri", help="Database connection string")
|
||||
parser.add_argument("--migrations-dir", default="./migrations",
|
||||
help="Migrations directory")
|
||||
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
# Generate command
|
||||
gen_parser = subparsers.add_parser("generate", help="Generate new migration")
|
||||
gen_parser.add_argument("name", help="Migration name")
|
||||
gen_parser.add_argument("--dry-run", action="store_true",
|
||||
help="Show what would be generated")
|
||||
|
||||
# Apply command
|
||||
apply_parser = subparsers.add_parser("apply", help="Apply pending migrations")
|
||||
apply_parser.add_argument("--dry-run", action="store_true",
|
||||
help="Show what would be executed")
|
||||
|
||||
# Rollback command
|
||||
rollback_parser = subparsers.add_parser("rollback", help="Rollback migration")
|
||||
rollback_parser.add_argument("id", help="Migration ID to rollback")
|
||||
rollback_parser.add_argument("--dry-run", action="store_true",
|
||||
help="Show what would be executed")
|
||||
|
||||
# Status command
|
||||
subparsers.add_parser("status", help="Show migration status")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# For generate, we don't need connection
|
||||
if args.command == "generate":
|
||||
manager = MigrationManager(args.db, "", args.migrations_dir)
|
||||
migration = manager.generate_migration(args.name, args.dry_run)
|
||||
sys.exit(0 if migration else 1)
|
||||
|
||||
# Other commands need connection
|
||||
if not args.uri:
|
||||
print("Error: --uri required for this command")
|
||||
sys.exit(1)
|
||||
|
||||
manager = MigrationManager(args.db, args.uri, args.migrations_dir)
|
||||
|
||||
if not manager.connect():
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
manager._ensure_migrations_table()
|
||||
|
||||
if args.command == "status":
|
||||
pending = manager.get_pending_migrations()
|
||||
print(f"Pending migrations: {len(pending)}")
|
||||
for migration in pending:
|
||||
print(f" {migration.id} - {migration.name}")
|
||||
|
||||
elif args.command == "apply":
|
||||
pending = manager.get_pending_migrations()
|
||||
if not pending:
|
||||
print("No pending migrations")
|
||||
else:
|
||||
for migration in pending:
|
||||
if not manager.apply_migration(migration, args.dry_run):
|
||||
sys.exit(1)
|
||||
|
||||
elif args.command == "rollback":
|
||||
if not manager.rollback_migration(args.id, args.dry_run):
|
||||
sys.exit(1)
|
||||
|
||||
finally:
|
||||
manager.disconnect()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
444
skills/databases/scripts/db_performance_check.py
Normal file
444
skills/databases/scripts/db_performance_check.py
Normal file
@@ -0,0 +1,444 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Database performance analysis tool for MongoDB and PostgreSQL.
|
||||
Analyzes slow queries, recommends indexes, and generates reports.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from dataclasses import dataclass, asdict
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
try:
|
||||
from pymongo import MongoClient
|
||||
MONGO_AVAILABLE = True
|
||||
except ImportError:
|
||||
MONGO_AVAILABLE = False
|
||||
|
||||
try:
|
||||
import psycopg2
|
||||
from psycopg2.extras import RealDictCursor
|
||||
POSTGRES_AVAILABLE = True
|
||||
except ImportError:
|
||||
POSTGRES_AVAILABLE = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class SlowQuery:
|
||||
"""Represents a slow query."""
|
||||
|
||||
query: str
|
||||
execution_time_ms: float
|
||||
count: int
|
||||
collection_or_table: Optional[str] = None
|
||||
index_used: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class IndexRecommendation:
|
||||
"""Index recommendation."""
|
||||
|
||||
collection_or_table: str
|
||||
fields: List[str]
|
||||
reason: str
|
||||
estimated_benefit: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class PerformanceReport:
|
||||
"""Performance analysis report."""
|
||||
|
||||
database_type: str
|
||||
database_name: str
|
||||
timestamp: datetime
|
||||
slow_queries: List[SlowQuery]
|
||||
index_recommendations: List[IndexRecommendation]
|
||||
database_metrics: Dict[str, any]
|
||||
|
||||
|
||||
class PerformanceAnalyzer:
|
||||
"""Analyzes database performance."""
|
||||
|
||||
def __init__(self, db_type: str, connection_string: str, threshold_ms: int = 100):
|
||||
"""
|
||||
Initialize performance analyzer.
|
||||
|
||||
Args:
|
||||
db_type: Database type ('mongodb' or 'postgres')
|
||||
connection_string: Database connection string
|
||||
threshold_ms: Slow query threshold in milliseconds
|
||||
"""
|
||||
self.db_type = db_type.lower()
|
||||
self.connection_string = connection_string
|
||||
self.threshold_ms = threshold_ms
|
||||
|
||||
self.client = None
|
||||
self.db = None
|
||||
self.conn = None
|
||||
|
||||
def connect(self) -> bool:
|
||||
"""Connect to database."""
|
||||
try:
|
||||
if self.db_type == "mongodb":
|
||||
if not MONGO_AVAILABLE:
|
||||
print("Error: pymongo not installed")
|
||||
return False
|
||||
self.client = MongoClient(self.connection_string)
|
||||
self.db = self.client.get_default_database()
|
||||
self.client.server_info()
|
||||
return True
|
||||
|
||||
elif self.db_type == "postgres":
|
||||
if not POSTGRES_AVAILABLE:
|
||||
print("Error: psycopg2 not installed")
|
||||
return False
|
||||
self.conn = psycopg2.connect(self.connection_string)
|
||||
return True
|
||||
|
||||
else:
|
||||
print(f"Error: Unsupported database type: {self.db_type}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
print(f"Connection error: {e}")
|
||||
return False
|
||||
|
||||
def disconnect(self):
|
||||
"""Disconnect from database."""
|
||||
try:
|
||||
if self.client:
|
||||
self.client.close()
|
||||
if self.conn:
|
||||
self.conn.close()
|
||||
except Exception as e:
|
||||
print(f"Disconnect error: {e}")
|
||||
|
||||
def analyze(self) -> Optional[PerformanceReport]:
|
||||
"""
|
||||
Analyze database performance.
|
||||
|
||||
Returns:
|
||||
PerformanceReport if successful, None otherwise
|
||||
"""
|
||||
try:
|
||||
if self.db_type == "mongodb":
|
||||
return self._analyze_mongodb()
|
||||
elif self.db_type == "postgres":
|
||||
return self._analyze_postgres()
|
||||
else:
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
print(f"Analysis error: {e}")
|
||||
return None
|
||||
|
||||
def _analyze_mongodb(self) -> PerformanceReport:
|
||||
"""Analyze MongoDB performance."""
|
||||
slow_queries = []
|
||||
index_recommendations = []
|
||||
|
||||
# Enable profiling if not enabled
|
||||
profiling_level = self.db.command("profile", -1)
|
||||
if profiling_level.get("was", 0) == 0:
|
||||
self.db.command("profile", 1, slowms=self.threshold_ms)
|
||||
|
||||
# Get slow queries from system.profile
|
||||
for doc in self.db.system.profile.find(
|
||||
{"millis": {"$gte": self.threshold_ms}},
|
||||
limit=50
|
||||
).sort("millis", -1):
|
||||
|
||||
query_str = json.dumps(doc.get("command", {}), default=str)
|
||||
|
||||
slow_queries.append(SlowQuery(
|
||||
query=query_str,
|
||||
execution_time_ms=doc.get("millis", 0),
|
||||
count=1,
|
||||
collection_or_table=doc.get("ns", "").split(".")[-1] if "ns" in doc else None,
|
||||
index_used=doc.get("planSummary")
|
||||
))
|
||||
|
||||
# Analyze collections for index recommendations
|
||||
for coll_name in self.db.list_collection_names():
|
||||
if coll_name.startswith("system."):
|
||||
continue
|
||||
|
||||
coll = self.db[coll_name]
|
||||
|
||||
# Check for collections scans
|
||||
stats = coll.aggregate([
|
||||
{"$collStats": {"storageStats": {}}}
|
||||
]).next()
|
||||
|
||||
# Check if collection has indexes
|
||||
indexes = list(coll.list_indexes())
|
||||
|
||||
if len(indexes) <= 1: # Only _id index
|
||||
# Recommend indexes based on common patterns
|
||||
# Sample documents to find frequently queried fields
|
||||
sample = list(coll.find().limit(100))
|
||||
|
||||
if sample:
|
||||
# Find fields that appear in most documents
|
||||
field_freq = {}
|
||||
for doc in sample:
|
||||
for field in doc.keys():
|
||||
if field != "_id":
|
||||
field_freq[field] = field_freq.get(field, 0) + 1
|
||||
|
||||
# Recommend index on most common field
|
||||
if field_freq:
|
||||
top_field = max(field_freq.items(), key=lambda x: x[1])[0]
|
||||
index_recommendations.append(IndexRecommendation(
|
||||
collection_or_table=coll_name,
|
||||
fields=[top_field],
|
||||
reason="Frequently queried field without index",
|
||||
estimated_benefit="High"
|
||||
))
|
||||
|
||||
# Get database metrics
|
||||
server_status = self.client.admin.command("serverStatus")
|
||||
db_stats = self.db.command("dbStats")
|
||||
|
||||
metrics = {
|
||||
"connections": server_status.get("connections", {}).get("current", 0),
|
||||
"operations_per_sec": server_status.get("opcounters", {}).get("query", 0),
|
||||
"database_size_mb": db_stats.get("dataSize", 0) / (1024 * 1024),
|
||||
"index_size_mb": db_stats.get("indexSize", 0) / (1024 * 1024),
|
||||
"collections": db_stats.get("collections", 0)
|
||||
}
|
||||
|
||||
return PerformanceReport(
|
||||
database_type="mongodb",
|
||||
database_name=self.db.name,
|
||||
timestamp=datetime.now(),
|
||||
slow_queries=slow_queries[:10], # Top 10
|
||||
index_recommendations=index_recommendations,
|
||||
database_metrics=metrics
|
||||
)
|
||||
|
||||
def _analyze_postgres(self) -> PerformanceReport:
|
||||
"""Analyze PostgreSQL performance."""
|
||||
slow_queries = []
|
||||
index_recommendations = []
|
||||
|
||||
with self.conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
# Check if pg_stat_statements extension is available
|
||||
cur.execute("""
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements'
|
||||
) AS has_extension
|
||||
""")
|
||||
has_pg_stat_statements = cur.fetchone()["has_extension"]
|
||||
|
||||
if has_pg_stat_statements:
|
||||
# Get slow queries from pg_stat_statements
|
||||
cur.execute("""
|
||||
SELECT
|
||||
query,
|
||||
mean_exec_time,
|
||||
calls,
|
||||
total_exec_time
|
||||
FROM pg_stat_statements
|
||||
WHERE mean_exec_time >= %s
|
||||
ORDER BY mean_exec_time DESC
|
||||
LIMIT 10
|
||||
""", (self.threshold_ms,))
|
||||
|
||||
for row in cur.fetchall():
|
||||
slow_queries.append(SlowQuery(
|
||||
query=row["query"],
|
||||
execution_time_ms=row["mean_exec_time"],
|
||||
count=row["calls"]
|
||||
))
|
||||
|
||||
# Find tables with sequential scans (potential index candidates)
|
||||
cur.execute("""
|
||||
SELECT
|
||||
schemaname,
|
||||
tablename,
|
||||
seq_scan,
|
||||
seq_tup_read,
|
||||
idx_scan
|
||||
FROM pg_stat_user_tables
|
||||
WHERE seq_scan > 1000
|
||||
AND (idx_scan IS NULL OR seq_scan > idx_scan * 2)
|
||||
ORDER BY seq_tup_read DESC
|
||||
LIMIT 10
|
||||
""")
|
||||
|
||||
for row in cur.fetchall():
|
||||
index_recommendations.append(IndexRecommendation(
|
||||
collection_or_table=f"{row['schemaname']}.{row['tablename']}",
|
||||
fields=["<analyze query patterns>"],
|
||||
reason=f"High sequential scans ({row['seq_scan']}) vs index scans ({row['idx_scan'] or 0})",
|
||||
estimated_benefit="High" if row["seq_tup_read"] > 100000 else "Medium"
|
||||
))
|
||||
|
||||
# Find unused indexes
|
||||
cur.execute("""
|
||||
SELECT
|
||||
schemaname,
|
||||
tablename,
|
||||
indexname,
|
||||
idx_scan
|
||||
FROM pg_stat_user_indexes
|
||||
WHERE idx_scan = 0
|
||||
AND indexname NOT LIKE '%_pkey'
|
||||
ORDER BY pg_relation_size(indexrelid) DESC
|
||||
""")
|
||||
|
||||
unused_indexes = []
|
||||
for row in cur.fetchall():
|
||||
unused_indexes.append(
|
||||
f"{row['schemaname']}.{row['tablename']}.{row['indexname']}"
|
||||
)
|
||||
|
||||
# Database metrics
|
||||
cur.execute("""
|
||||
SELECT
|
||||
sum(numbackends) AS connections,
|
||||
sum(xact_commit) AS commits,
|
||||
sum(xact_rollback) AS rollbacks
|
||||
FROM pg_stat_database
|
||||
WHERE datname = current_database()
|
||||
""")
|
||||
stats = cur.fetchone()
|
||||
|
||||
cur.execute("""
|
||||
SELECT pg_database_size(current_database()) AS db_size
|
||||
""")
|
||||
db_size = cur.fetchone()["db_size"]
|
||||
|
||||
cur.execute("""
|
||||
SELECT
|
||||
sum(heap_blks_hit) / NULLIF(sum(heap_blks_hit) + sum(heap_blks_read), 0) AS cache_hit_ratio
|
||||
FROM pg_statio_user_tables
|
||||
""")
|
||||
cache_ratio = cur.fetchone()["cache_hit_ratio"] or 0
|
||||
|
||||
metrics = {
|
||||
"connections": stats["connections"],
|
||||
"commits": stats["commits"],
|
||||
"rollbacks": stats["rollbacks"],
|
||||
"database_size_mb": db_size / (1024 * 1024),
|
||||
"cache_hit_ratio": float(cache_ratio),
|
||||
"unused_indexes": unused_indexes
|
||||
}
|
||||
|
||||
return PerformanceReport(
|
||||
database_type="postgres",
|
||||
database_name=self.conn.info.dbname,
|
||||
timestamp=datetime.now(),
|
||||
slow_queries=slow_queries,
|
||||
index_recommendations=index_recommendations,
|
||||
database_metrics=metrics
|
||||
)
|
||||
|
||||
def print_report(self, report: PerformanceReport):
|
||||
"""Print performance report."""
|
||||
print("=" * 80)
|
||||
print(f"Database Performance Report - {report.database_type.upper()}")
|
||||
print(f"Database: {report.database_name}")
|
||||
print(f"Timestamp: {report.timestamp}")
|
||||
print("=" * 80)
|
||||
|
||||
print("\n## Database Metrics")
|
||||
print("-" * 80)
|
||||
for key, value in report.database_metrics.items():
|
||||
if isinstance(value, float):
|
||||
print(f"{key}: {value:.2f}")
|
||||
else:
|
||||
print(f"{key}: {value}")
|
||||
|
||||
print("\n## Slow Queries")
|
||||
print("-" * 80)
|
||||
if report.slow_queries:
|
||||
for i, query in enumerate(report.slow_queries, 1):
|
||||
print(f"\n{i}. Execution Time: {query.execution_time_ms:.2f}ms | Count: {query.count}")
|
||||
if query.collection_or_table:
|
||||
print(f" Collection/Table: {query.collection_or_table}")
|
||||
if query.index_used:
|
||||
print(f" Index Used: {query.index_used}")
|
||||
print(f" Query: {query.query[:200]}...")
|
||||
else:
|
||||
print("No slow queries found")
|
||||
|
||||
print("\n## Index Recommendations")
|
||||
print("-" * 80)
|
||||
if report.index_recommendations:
|
||||
for i, rec in enumerate(report.index_recommendations, 1):
|
||||
print(f"\n{i}. {rec.collection_or_table}")
|
||||
print(f" Fields: {', '.join(rec.fields)}")
|
||||
print(f" Reason: {rec.reason}")
|
||||
print(f" Estimated Benefit: {rec.estimated_benefit}")
|
||||
|
||||
if report.database_type == "mongodb":
|
||||
index_spec = {field: 1 for field in rec.fields}
|
||||
print(f" Command: db.{rec.collection_or_table}.createIndex({json.dumps(index_spec)})")
|
||||
elif report.database_type == "postgres":
|
||||
fields_str = ", ".join(rec.fields)
|
||||
print(f" Command: CREATE INDEX idx_{rec.collection_or_table.replace('.', '_')}_{rec.fields[0]} ON {rec.collection_or_table}({fields_str});")
|
||||
else:
|
||||
print("No index recommendations")
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
|
||||
def save_report(self, report: PerformanceReport, filename: str):
|
||||
"""Save report to JSON file."""
|
||||
# Convert dataclasses to dict
|
||||
report_dict = {
|
||||
"database_type": report.database_type,
|
||||
"database_name": report.database_name,
|
||||
"timestamp": report.timestamp.isoformat(),
|
||||
"slow_queries": [asdict(q) for q in report.slow_queries],
|
||||
"index_recommendations": [asdict(r) for r in report.index_recommendations],
|
||||
"database_metrics": report.database_metrics
|
||||
}
|
||||
|
||||
with open(filename, "w") as f:
|
||||
json.dump(report_dict, f, indent=2, default=str)
|
||||
|
||||
print(f"\nReport saved to: {filename}")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point."""
|
||||
parser = argparse.ArgumentParser(description="Database performance analysis tool")
|
||||
parser.add_argument("--db", required=True, choices=["mongodb", "postgres"],
|
||||
help="Database type")
|
||||
parser.add_argument("--uri", required=True, help="Database connection string")
|
||||
parser.add_argument("--threshold", type=int, default=100,
|
||||
help="Slow query threshold in milliseconds (default: 100)")
|
||||
parser.add_argument("--output", help="Save report to JSON file")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
analyzer = PerformanceAnalyzer(args.db, args.uri, args.threshold)
|
||||
|
||||
if not analyzer.connect():
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
print(f"Analyzing {args.db} performance (threshold: {args.threshold}ms)...")
|
||||
report = analyzer.analyze()
|
||||
|
||||
if report:
|
||||
analyzer.print_report(report)
|
||||
|
||||
if args.output:
|
||||
analyzer.save_report(report, args.output)
|
||||
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("Analysis failed")
|
||||
sys.exit(1)
|
||||
|
||||
finally:
|
||||
analyzer.disconnect()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
20
skills/databases/scripts/requirements.txt
Normal file
20
skills/databases/scripts/requirements.txt
Normal file
@@ -0,0 +1,20 @@
|
||||
# Databases Skill Dependencies
|
||||
# Python 3.10+ required
|
||||
|
||||
# No Python package dependencies - uses only standard library
|
||||
|
||||
# Testing dependencies (dev)
|
||||
pytest>=8.0.0
|
||||
pytest-cov>=4.1.0
|
||||
pytest-mock>=3.12.0
|
||||
|
||||
# Note: This skill requires database CLI tools:
|
||||
#
|
||||
# PostgreSQL:
|
||||
# - psql CLI (comes with PostgreSQL)
|
||||
# - Ubuntu/Debian: sudo apt-get install postgresql-client
|
||||
# - macOS: brew install postgresql
|
||||
#
|
||||
# MongoDB:
|
||||
# - mongosh CLI: https://www.mongodb.com/try/download/shell
|
||||
# - mongodump/mongorestore: https://www.mongodb.com/try/download/database-tools
|
||||
1
skills/databases/scripts/tests/coverage-db.json
Normal file
1
skills/databases/scripts/tests/coverage-db.json
Normal file
File diff suppressed because one or more lines are too long
4
skills/databases/scripts/tests/requirements.txt
Normal file
4
skills/databases/scripts/tests/requirements.txt
Normal file
@@ -0,0 +1,4 @@
|
||||
pytest>=7.0.0
|
||||
pytest-cov>=4.0.0
|
||||
pytest-mock>=3.10.0
|
||||
mongomock>=4.1.0
|
||||
340
skills/databases/scripts/tests/test_db_backup.py
Normal file
340
skills/databases/scripts/tests/test_db_backup.py
Normal file
@@ -0,0 +1,340 @@
|
||||
"""Tests for db_backup.py"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch, MagicMock, call
|
||||
|
||||
import pytest
|
||||
|
||||
# Add parent directory to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from db_backup import BackupInfo, BackupManager
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_backup_dir(tmp_path):
|
||||
"""Create temporary backup directory."""
|
||||
backup_dir = tmp_path / "backups"
|
||||
backup_dir.mkdir()
|
||||
return str(backup_dir)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_backup_info():
|
||||
"""Create sample backup info."""
|
||||
return BackupInfo(
|
||||
filename="test_backup_20250101_120000.dump",
|
||||
database_type="mongodb",
|
||||
database_name="testdb",
|
||||
timestamp=datetime.now(),
|
||||
size_bytes=1024000,
|
||||
compressed=True,
|
||||
verified=True
|
||||
)
|
||||
|
||||
|
||||
class TestBackupInfo:
|
||||
"""Test BackupInfo dataclass."""
|
||||
|
||||
def test_backup_info_creation(self):
|
||||
"""Test creating backup info object."""
|
||||
info = BackupInfo(
|
||||
filename="backup.dump",
|
||||
database_type="mongodb",
|
||||
database_name="mydb",
|
||||
timestamp=datetime.now(),
|
||||
size_bytes=1024,
|
||||
compressed=False
|
||||
)
|
||||
|
||||
assert info.filename == "backup.dump"
|
||||
assert info.database_type == "mongodb"
|
||||
assert info.database_name == "mydb"
|
||||
assert info.size_bytes == 1024
|
||||
assert not info.compressed
|
||||
assert not info.verified
|
||||
|
||||
|
||||
class TestBackupManager:
|
||||
"""Test BackupManager class."""
|
||||
|
||||
def test_init(self, temp_backup_dir):
|
||||
"""Test manager initialization."""
|
||||
manager = BackupManager("mongodb", temp_backup_dir)
|
||||
|
||||
assert manager.db_type == "mongodb"
|
||||
assert Path(temp_backup_dir).exists()
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_backup_mongodb(self, mock_run, temp_backup_dir):
|
||||
"""Test MongoDB backup creation."""
|
||||
mock_run.return_value = Mock(returncode=0, stderr="")
|
||||
|
||||
manager = BackupManager("mongodb", temp_backup_dir)
|
||||
backup_info = manager.create_backup(
|
||||
"mongodb://localhost",
|
||||
"testdb",
|
||||
compress=False,
|
||||
verify=False
|
||||
)
|
||||
|
||||
assert backup_info is not None
|
||||
assert backup_info.database_type == "mongodb"
|
||||
assert backup_info.database_name == "testdb"
|
||||
mock_run.assert_called_once()
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_backup_postgres(self, mock_run, temp_backup_dir):
|
||||
"""Test PostgreSQL backup creation."""
|
||||
mock_run.return_value = Mock(returncode=0, stderr="")
|
||||
|
||||
manager = BackupManager("postgres", temp_backup_dir)
|
||||
|
||||
with patch('builtins.open', create=True) as mock_open:
|
||||
mock_open.return_value.__enter__.return_value = MagicMock()
|
||||
|
||||
backup_info = manager.create_backup(
|
||||
"postgresql://localhost/testdb",
|
||||
"testdb",
|
||||
compress=False,
|
||||
verify=False
|
||||
)
|
||||
|
||||
assert backup_info is not None
|
||||
assert backup_info.database_type == "postgres"
|
||||
assert backup_info.database_name == "testdb"
|
||||
|
||||
def test_backup_postgres_no_database(self, temp_backup_dir):
|
||||
"""Test PostgreSQL backup without database name."""
|
||||
manager = BackupManager("postgres", temp_backup_dir)
|
||||
backup_info = manager.create_backup(
|
||||
"postgresql://localhost",
|
||||
database=None,
|
||||
compress=False,
|
||||
verify=False
|
||||
)
|
||||
|
||||
assert backup_info is None
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_backup_with_compression(self, mock_run, temp_backup_dir):
|
||||
"""Test backup with compression."""
|
||||
mock_run.return_value = Mock(returncode=0, stderr="")
|
||||
|
||||
manager = BackupManager("mongodb", temp_backup_dir)
|
||||
|
||||
with patch('shutil.make_archive') as mock_archive, \
|
||||
patch('shutil.rmtree') as mock_rmtree:
|
||||
|
||||
backup_info = manager.create_backup(
|
||||
"mongodb://localhost",
|
||||
"testdb",
|
||||
compress=True,
|
||||
verify=False
|
||||
)
|
||||
|
||||
assert backup_info is not None
|
||||
assert backup_info.compressed
|
||||
mock_archive.assert_called_once()
|
||||
|
||||
def test_save_and_load_metadata(self, temp_backup_dir, sample_backup_info):
|
||||
"""Test saving and loading backup metadata."""
|
||||
manager = BackupManager("mongodb", temp_backup_dir)
|
||||
|
||||
# Save metadata
|
||||
manager._save_metadata(sample_backup_info)
|
||||
|
||||
# Check file was created
|
||||
metadata_file = Path(temp_backup_dir) / f"{sample_backup_info.filename}.json"
|
||||
assert metadata_file.exists()
|
||||
|
||||
# Load metadata
|
||||
with open(metadata_file) as f:
|
||||
data = json.load(f)
|
||||
assert data["filename"] == sample_backup_info.filename
|
||||
assert data["database_type"] == "mongodb"
|
||||
assert data["database_name"] == "testdb"
|
||||
|
||||
def test_list_backups(self, temp_backup_dir, sample_backup_info):
|
||||
"""Test listing backups."""
|
||||
manager = BackupManager("mongodb", temp_backup_dir)
|
||||
|
||||
# Create test backup metadata
|
||||
manager._save_metadata(sample_backup_info)
|
||||
|
||||
# List backups
|
||||
backups = manager.list_backups()
|
||||
|
||||
assert len(backups) == 1
|
||||
assert backups[0].filename == sample_backup_info.filename
|
||||
assert backups[0].database_name == "testdb"
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_restore_mongodb(self, mock_run, temp_backup_dir):
|
||||
"""Test MongoDB restore."""
|
||||
mock_run.return_value = Mock(returncode=0, stderr="")
|
||||
|
||||
manager = BackupManager("mongodb", temp_backup_dir)
|
||||
|
||||
# Create dummy backup file
|
||||
backup_file = Path(temp_backup_dir) / "test_backup.dump"
|
||||
backup_file.touch()
|
||||
|
||||
result = manager.restore_backup(
|
||||
"test_backup.dump",
|
||||
"mongodb://localhost"
|
||||
)
|
||||
|
||||
assert result is True
|
||||
mock_run.assert_called_once()
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_restore_postgres(self, mock_run, temp_backup_dir):
|
||||
"""Test PostgreSQL restore."""
|
||||
mock_run.return_value = Mock(returncode=0, stderr="")
|
||||
|
||||
manager = BackupManager("postgres", temp_backup_dir)
|
||||
|
||||
# Create dummy backup file
|
||||
backup_file = Path(temp_backup_dir) / "test_backup.sql"
|
||||
backup_file.write_text("SELECT 1;")
|
||||
|
||||
with patch('builtins.open', create=True) as mock_open:
|
||||
mock_open.return_value.__enter__.return_value = MagicMock()
|
||||
|
||||
result = manager.restore_backup(
|
||||
"test_backup.sql",
|
||||
"postgresql://localhost/testdb"
|
||||
)
|
||||
|
||||
assert result is True
|
||||
|
||||
def test_restore_nonexistent_backup(self, temp_backup_dir):
|
||||
"""Test restore with non-existent backup file."""
|
||||
manager = BackupManager("mongodb", temp_backup_dir)
|
||||
|
||||
result = manager.restore_backup(
|
||||
"nonexistent.dump",
|
||||
"mongodb://localhost"
|
||||
)
|
||||
|
||||
assert result is False
|
||||
|
||||
def test_restore_dry_run(self, temp_backup_dir):
|
||||
"""Test restore in dry-run mode."""
|
||||
manager = BackupManager("mongodb", temp_backup_dir)
|
||||
|
||||
# Create dummy backup file
|
||||
backup_file = Path(temp_backup_dir) / "test_backup.dump"
|
||||
backup_file.touch()
|
||||
|
||||
result = manager.restore_backup(
|
||||
"test_backup.dump",
|
||||
"mongodb://localhost",
|
||||
dry_run=True
|
||||
)
|
||||
|
||||
assert result is True
|
||||
|
||||
def test_cleanup_old_backups(self, temp_backup_dir):
|
||||
"""Test cleaning up old backups."""
|
||||
manager = BackupManager("mongodb", temp_backup_dir)
|
||||
|
||||
# Create old backup file (simulate by setting mtime)
|
||||
old_backup = Path(temp_backup_dir) / "old_backup.dump"
|
||||
old_backup.touch()
|
||||
|
||||
# Set mtime to 10 days ago
|
||||
old_time = datetime.now().timestamp() - (10 * 24 * 3600)
|
||||
os.utime(old_backup, (old_time, old_time))
|
||||
|
||||
# Cleanup with 7-day retention
|
||||
removed = manager.cleanup_old_backups(retention_days=7)
|
||||
|
||||
assert removed == 1
|
||||
assert not old_backup.exists()
|
||||
|
||||
def test_cleanup_dry_run(self, temp_backup_dir):
|
||||
"""Test cleanup in dry-run mode."""
|
||||
manager = BackupManager("mongodb", temp_backup_dir)
|
||||
|
||||
# Create old backup file
|
||||
old_backup = Path(temp_backup_dir) / "old_backup.dump"
|
||||
old_backup.touch()
|
||||
|
||||
old_time = datetime.now().timestamp() - (10 * 24 * 3600)
|
||||
os.utime(old_backup, (old_time, old_time))
|
||||
|
||||
# Cleanup with dry-run
|
||||
removed = manager.cleanup_old_backups(retention_days=7, dry_run=True)
|
||||
|
||||
assert removed == 1
|
||||
assert old_backup.exists() # File should still exist
|
||||
|
||||
def test_verify_backup(self, temp_backup_dir, sample_backup_info):
|
||||
"""Test backup verification."""
|
||||
manager = BackupManager("mongodb", temp_backup_dir)
|
||||
|
||||
# Create dummy backup file
|
||||
backup_file = Path(temp_backup_dir) / sample_backup_info.filename
|
||||
backup_file.write_text("backup data")
|
||||
|
||||
result = manager._verify_backup(sample_backup_info)
|
||||
|
||||
assert result is True
|
||||
|
||||
def test_verify_empty_backup(self, temp_backup_dir, sample_backup_info):
|
||||
"""Test verification of empty backup file."""
|
||||
manager = BackupManager("mongodb", temp_backup_dir)
|
||||
|
||||
# Create empty backup file
|
||||
backup_file = Path(temp_backup_dir) / sample_backup_info.filename
|
||||
backup_file.touch()
|
||||
|
||||
result = manager._verify_backup(sample_backup_info)
|
||||
|
||||
assert result is False
|
||||
|
||||
def test_format_size(self, temp_backup_dir):
|
||||
"""Test size formatting."""
|
||||
manager = BackupManager("mongodb", temp_backup_dir)
|
||||
|
||||
assert manager._format_size(500) == "500.00 B"
|
||||
assert manager._format_size(1024) == "1.00 KB"
|
||||
assert manager._format_size(1024 * 1024) == "1.00 MB"
|
||||
assert manager._format_size(1024 * 1024 * 1024) == "1.00 GB"
|
||||
|
||||
def test_get_size_file(self, temp_backup_dir):
|
||||
"""Test getting size of file."""
|
||||
manager = BackupManager("mongodb", temp_backup_dir)
|
||||
|
||||
test_file = Path(temp_backup_dir) / "test.txt"
|
||||
test_file.write_text("test data")
|
||||
|
||||
size = manager._get_size(test_file)
|
||||
|
||||
assert size > 0
|
||||
|
||||
def test_get_size_directory(self, temp_backup_dir):
|
||||
"""Test getting size of directory."""
|
||||
manager = BackupManager("mongodb", temp_backup_dir)
|
||||
|
||||
test_dir = Path(temp_backup_dir) / "test_dir"
|
||||
test_dir.mkdir()
|
||||
(test_dir / "file1.txt").write_text("data1")
|
||||
(test_dir / "file2.txt").write_text("data2")
|
||||
|
||||
size = manager._get_size(test_dir)
|
||||
|
||||
assert size > 0
|
||||
|
||||
|
||||
# Import os for cleanup test
|
||||
import os
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
277
skills/databases/scripts/tests/test_db_migrate.py
Normal file
277
skills/databases/scripts/tests/test_db_migrate.py
Normal file
@@ -0,0 +1,277 @@
|
||||
"""Tests for db_migrate.py"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
# Add parent directory to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from db_migrate import Migration, MigrationManager
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_migrations_dir(tmp_path):
|
||||
"""Create temporary migrations directory."""
|
||||
migrations_dir = tmp_path / "migrations"
|
||||
migrations_dir.mkdir()
|
||||
return str(migrations_dir)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_mongo_client():
|
||||
"""Mock MongoDB client."""
|
||||
mock_client = MagicMock()
|
||||
mock_db = MagicMock()
|
||||
mock_client.get_default_database.return_value = mock_db
|
||||
mock_client.server_info.return_value = {}
|
||||
return mock_client, mock_db
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_postgres_conn():
|
||||
"""Mock PostgreSQL connection."""
|
||||
mock_conn = MagicMock()
|
||||
mock_cursor = MagicMock()
|
||||
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
|
||||
return mock_conn, mock_cursor
|
||||
|
||||
|
||||
class TestMigration:
|
||||
"""Test Migration dataclass."""
|
||||
|
||||
def test_migration_creation(self):
|
||||
"""Test creating migration object."""
|
||||
migration = Migration(
|
||||
id="20250101120000",
|
||||
name="test_migration",
|
||||
timestamp=datetime.now(),
|
||||
database_type="mongodb"
|
||||
)
|
||||
|
||||
assert migration.id == "20250101120000"
|
||||
assert migration.name == "test_migration"
|
||||
assert migration.database_type == "mongodb"
|
||||
assert not migration.applied
|
||||
|
||||
|
||||
class TestMigrationManager:
|
||||
"""Test MigrationManager class."""
|
||||
|
||||
def test_init(self, temp_migrations_dir):
|
||||
"""Test manager initialization."""
|
||||
manager = MigrationManager("mongodb", "mongodb://localhost", temp_migrations_dir)
|
||||
|
||||
assert manager.db_type == "mongodb"
|
||||
assert manager.connection_string == "mongodb://localhost"
|
||||
assert Path(temp_migrations_dir).exists()
|
||||
|
||||
@patch('db_migrate.MongoClient')
|
||||
def test_connect_mongodb(self, mock_client_class, temp_migrations_dir, mock_mongo_client):
|
||||
"""Test MongoDB connection."""
|
||||
mock_client, mock_db = mock_mongo_client
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
manager = MigrationManager("mongodb", "mongodb://localhost", temp_migrations_dir)
|
||||
result = manager.connect()
|
||||
|
||||
assert result is True
|
||||
assert manager.client == mock_client
|
||||
assert manager.db == mock_db
|
||||
|
||||
@patch('db_migrate.psycopg2')
|
||||
def test_connect_postgres(self, mock_psycopg2, temp_migrations_dir, mock_postgres_conn):
|
||||
"""Test PostgreSQL connection."""
|
||||
mock_conn, mock_cursor = mock_postgres_conn
|
||||
mock_psycopg2.connect.return_value = mock_conn
|
||||
|
||||
manager = MigrationManager("postgres", "postgresql://localhost", temp_migrations_dir)
|
||||
result = manager.connect()
|
||||
|
||||
assert result is True
|
||||
assert manager.conn == mock_conn
|
||||
|
||||
def test_connect_unsupported_db(self, temp_migrations_dir):
|
||||
"""Test connection with unsupported database type."""
|
||||
manager = MigrationManager("unsupported", "connection_string", temp_migrations_dir)
|
||||
result = manager.connect()
|
||||
|
||||
assert result is False
|
||||
|
||||
def test_generate_migration(self, temp_migrations_dir):
|
||||
"""Test migration generation."""
|
||||
manager = MigrationManager("mongodb", "mongodb://localhost", temp_migrations_dir)
|
||||
migration = manager.generate_migration("test_migration")
|
||||
|
||||
assert migration is not None
|
||||
assert migration.name == "test_migration"
|
||||
|
||||
# Check file was created
|
||||
migration_files = list(Path(temp_migrations_dir).glob("*.json"))
|
||||
assert len(migration_files) == 1
|
||||
|
||||
# Check file content
|
||||
with open(migration_files[0]) as f:
|
||||
data = json.load(f)
|
||||
assert data["name"] == "test_migration"
|
||||
assert data["database_type"] == "mongodb"
|
||||
|
||||
def test_generate_migration_dry_run(self, temp_migrations_dir):
|
||||
"""Test migration generation in dry-run mode."""
|
||||
manager = MigrationManager("postgres", "postgresql://localhost", temp_migrations_dir)
|
||||
migration = manager.generate_migration("test_migration", dry_run=True)
|
||||
|
||||
assert migration is not None
|
||||
|
||||
# Check no file was created
|
||||
migration_files = list(Path(temp_migrations_dir).glob("*.json"))
|
||||
assert len(migration_files) == 0
|
||||
|
||||
def test_get_pending_migrations(self, temp_migrations_dir):
|
||||
"""Test getting pending migrations."""
|
||||
manager = MigrationManager("mongodb", "mongodb://localhost", temp_migrations_dir)
|
||||
|
||||
# Create test migration file
|
||||
migration_data = {
|
||||
"id": "20250101120000",
|
||||
"name": "test_migration",
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"database_type": "mongodb",
|
||||
"mongodb_operations": []
|
||||
}
|
||||
|
||||
migration_file = Path(temp_migrations_dir) / "20250101120000_test.json"
|
||||
with open(migration_file, "w") as f:
|
||||
json.dump(migration_data, f)
|
||||
|
||||
# Mock database connection
|
||||
with patch.object(manager, 'db', MagicMock()):
|
||||
manager.db.migrations.find.return_value = []
|
||||
|
||||
pending = manager.get_pending_migrations()
|
||||
|
||||
assert len(pending) == 1
|
||||
assert pending[0].id == "20250101120000"
|
||||
assert pending[0].name == "test_migration"
|
||||
|
||||
@patch('db_migrate.MongoClient')
|
||||
def test_apply_mongodb_migration(self, mock_client_class, temp_migrations_dir, mock_mongo_client):
|
||||
"""Test applying MongoDB migration."""
|
||||
mock_client, mock_db = mock_mongo_client
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
manager = MigrationManager("mongodb", "mongodb://localhost", temp_migrations_dir)
|
||||
manager.connect()
|
||||
|
||||
migration = Migration(
|
||||
id="20250101120000",
|
||||
name="test_migration",
|
||||
timestamp=datetime.now(),
|
||||
database_type="mongodb",
|
||||
mongodb_operations=[
|
||||
{
|
||||
"operation": "createIndex",
|
||||
"collection": "users",
|
||||
"index": {"email": 1},
|
||||
"options": {}
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
result = manager.apply_migration(migration)
|
||||
|
||||
assert result is True
|
||||
mock_db["users"].create_index.assert_called_once()
|
||||
mock_db.migrations.insert_one.assert_called_once()
|
||||
|
||||
def test_apply_migration_dry_run(self, temp_migrations_dir):
|
||||
"""Test applying migration in dry-run mode."""
|
||||
manager = MigrationManager("mongodb", "mongodb://localhost", temp_migrations_dir)
|
||||
|
||||
migration = Migration(
|
||||
id="20250101120000",
|
||||
name="test_migration",
|
||||
timestamp=datetime.now(),
|
||||
database_type="mongodb",
|
||||
mongodb_operations=[]
|
||||
)
|
||||
|
||||
result = manager.apply_migration(migration, dry_run=True)
|
||||
|
||||
assert result is True
|
||||
|
||||
@patch('db_migrate.psycopg2')
|
||||
def test_rollback_postgres_migration(self, mock_psycopg2, temp_migrations_dir, mock_postgres_conn):
|
||||
"""Test rolling back PostgreSQL migration."""
|
||||
mock_conn, mock_cursor = mock_postgres_conn
|
||||
mock_psycopg2.connect.return_value = mock_conn
|
||||
|
||||
manager = MigrationManager("postgres", "postgresql://localhost", temp_migrations_dir)
|
||||
manager.connect()
|
||||
|
||||
# Create migration file
|
||||
migration_data = {
|
||||
"id": "20250101120000",
|
||||
"name": "test_migration",
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"database_type": "postgres",
|
||||
"up_sql": "CREATE TABLE test (id INT);",
|
||||
"down_sql": "DROP TABLE test;"
|
||||
}
|
||||
|
||||
migration_file = Path(temp_migrations_dir) / "20250101120000_test.json"
|
||||
with open(migration_file, "w") as f:
|
||||
json.dump(migration_data, f)
|
||||
|
||||
result = manager.rollback_migration("20250101120000")
|
||||
|
||||
assert result is True
|
||||
# Verify SQL was executed
|
||||
assert mock_cursor.execute.call_count >= 1
|
||||
|
||||
def test_rollback_migration_not_found(self, temp_migrations_dir):
|
||||
"""Test rollback with non-existent migration."""
|
||||
manager = MigrationManager("mongodb", "mongodb://localhost", temp_migrations_dir)
|
||||
|
||||
result = manager.rollback_migration("99999999999999")
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
def test_migration_sorting(temp_migrations_dir):
|
||||
"""Test that migrations are applied in correct order."""
|
||||
manager = MigrationManager("mongodb", "mongodb://localhost", temp_migrations_dir)
|
||||
|
||||
# Create multiple migration files
|
||||
for i in range(3):
|
||||
migration_data = {
|
||||
"id": f"2025010112000{i}",
|
||||
"name": f"migration_{i}",
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"database_type": "mongodb",
|
||||
"mongodb_operations": []
|
||||
}
|
||||
|
||||
migration_file = Path(temp_migrations_dir) / f"2025010112000{i}_test.json"
|
||||
with open(migration_file, "w") as f:
|
||||
json.dump(migration_data, f)
|
||||
|
||||
with patch.object(manager, 'db', MagicMock()):
|
||||
manager.db.migrations.find.return_value = []
|
||||
|
||||
pending = manager.get_pending_migrations()
|
||||
|
||||
# Check they're in order
|
||||
assert len(pending) == 3
|
||||
assert pending[0].id == "20250101120000"
|
||||
assert pending[1].id == "20250101120001"
|
||||
assert pending[2].id == "20250101120002"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
370
skills/databases/scripts/tests/test_db_performance_check.py
Normal file
370
skills/databases/scripts/tests/test_db_performance_check.py
Normal file
@@ -0,0 +1,370 @@
|
||||
"""Tests for db_performance_check.py"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
# Add parent directory to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from db_performance_check import (
|
||||
SlowQuery, IndexRecommendation, PerformanceReport, PerformanceAnalyzer
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_mongo_client():
|
||||
"""Mock MongoDB client."""
|
||||
mock_client = MagicMock()
|
||||
mock_db = MagicMock()
|
||||
mock_client.get_default_database.return_value = mock_db
|
||||
mock_client.server_info.return_value = {}
|
||||
return mock_client, mock_db
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_postgres_conn():
|
||||
"""Mock PostgreSQL connection."""
|
||||
mock_conn = MagicMock()
|
||||
mock_cursor = MagicMock()
|
||||
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
|
||||
return mock_conn, mock_cursor
|
||||
|
||||
|
||||
class TestSlowQuery:
|
||||
"""Test SlowQuery dataclass."""
|
||||
|
||||
def test_slow_query_creation(self):
|
||||
"""Test creating slow query object."""
|
||||
query = SlowQuery(
|
||||
query="SELECT * FROM users",
|
||||
execution_time_ms=150.5,
|
||||
count=10
|
||||
)
|
||||
|
||||
assert query.query == "SELECT * FROM users"
|
||||
assert query.execution_time_ms == 150.5
|
||||
assert query.count == 10
|
||||
|
||||
|
||||
class TestIndexRecommendation:
|
||||
"""Test IndexRecommendation dataclass."""
|
||||
|
||||
def test_recommendation_creation(self):
|
||||
"""Test creating index recommendation."""
|
||||
rec = IndexRecommendation(
|
||||
collection_or_table="users",
|
||||
fields=["email"],
|
||||
reason="Frequently queried field",
|
||||
estimated_benefit="High"
|
||||
)
|
||||
|
||||
assert rec.collection_or_table == "users"
|
||||
assert rec.fields == ["email"]
|
||||
assert rec.reason == "Frequently queried field"
|
||||
assert rec.estimated_benefit == "High"
|
||||
|
||||
|
||||
class TestPerformanceReport:
|
||||
"""Test PerformanceReport dataclass."""
|
||||
|
||||
def test_report_creation(self):
|
||||
"""Test creating performance report."""
|
||||
report = PerformanceReport(
|
||||
database_type="mongodb",
|
||||
database_name="testdb",
|
||||
timestamp=datetime.now(),
|
||||
slow_queries=[],
|
||||
index_recommendations=[],
|
||||
database_metrics={}
|
||||
)
|
||||
|
||||
assert report.database_type == "mongodb"
|
||||
assert report.database_name == "testdb"
|
||||
assert isinstance(report.slow_queries, list)
|
||||
assert isinstance(report.index_recommendations, list)
|
||||
assert isinstance(report.database_metrics, dict)
|
||||
|
||||
|
||||
class TestPerformanceAnalyzer:
|
||||
"""Test PerformanceAnalyzer class."""
|
||||
|
||||
def test_init(self):
|
||||
"""Test analyzer initialization."""
|
||||
analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost", 100)
|
||||
|
||||
assert analyzer.db_type == "mongodb"
|
||||
assert analyzer.connection_string == "mongodb://localhost"
|
||||
assert analyzer.threshold_ms == 100
|
||||
|
||||
@patch('db_performance_check.MongoClient')
|
||||
def test_connect_mongodb(self, mock_client_class, mock_mongo_client):
|
||||
"""Test MongoDB connection."""
|
||||
mock_client, mock_db = mock_mongo_client
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost")
|
||||
result = analyzer.connect()
|
||||
|
||||
assert result is True
|
||||
assert analyzer.client == mock_client
|
||||
assert analyzer.db == mock_db
|
||||
|
||||
@patch('db_performance_check.psycopg2')
|
||||
def test_connect_postgres(self, mock_psycopg2, mock_postgres_conn):
|
||||
"""Test PostgreSQL connection."""
|
||||
mock_conn, mock_cursor = mock_postgres_conn
|
||||
mock_psycopg2.connect.return_value = mock_conn
|
||||
|
||||
analyzer = PerformanceAnalyzer("postgres", "postgresql://localhost")
|
||||
result = analyzer.connect()
|
||||
|
||||
assert result is True
|
||||
assert analyzer.conn == mock_conn
|
||||
|
||||
def test_connect_unsupported_db(self):
|
||||
"""Test connection with unsupported database type."""
|
||||
analyzer = PerformanceAnalyzer("unsupported", "connection_string")
|
||||
result = analyzer.connect()
|
||||
|
||||
assert result is False
|
||||
|
||||
@patch('db_performance_check.MongoClient')
|
||||
def test_analyze_mongodb(self, mock_client_class, mock_mongo_client):
|
||||
"""Test MongoDB performance analysis."""
|
||||
mock_client, mock_db = mock_mongo_client
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
# Mock profiling
|
||||
mock_db.command.side_effect = [
|
||||
{"was": 0}, # profile -1 (get status)
|
||||
{}, # profile 1 (enable)
|
||||
]
|
||||
|
||||
# Mock slow queries
|
||||
mock_profile_cursor = MagicMock()
|
||||
mock_profile_cursor.sort.return_value = [
|
||||
{
|
||||
"command": {"find": "users"},
|
||||
"millis": 150,
|
||||
"ns": "testdb.users",
|
||||
"planSummary": "COLLSCAN"
|
||||
}
|
||||
]
|
||||
mock_db.system.profile.find.return_value = mock_profile_cursor
|
||||
|
||||
# Mock collections
|
||||
mock_db.list_collection_names.return_value = ["users", "orders"]
|
||||
|
||||
# Mock collection stats
|
||||
mock_coll = MagicMock()
|
||||
mock_coll.aggregate.return_value = [{"storageStats": {}}]
|
||||
mock_coll.list_indexes.return_value = [{"name": "_id_"}]
|
||||
mock_coll.find.return_value.limit.return_value = [
|
||||
{"_id": 1, "name": "Alice", "email": "alice@example.com"}
|
||||
]
|
||||
mock_db.__getitem__.return_value = mock_coll
|
||||
|
||||
# Mock server status and db stats
|
||||
mock_client.admin.command.return_value = {
|
||||
"connections": {"current": 10},
|
||||
"opcounters": {"query": 1000}
|
||||
}
|
||||
mock_db.command.return_value = {
|
||||
"dataSize": 1024 * 1024 * 100,
|
||||
"indexSize": 1024 * 1024 * 10,
|
||||
"collections": 5
|
||||
}
|
||||
|
||||
analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost")
|
||||
analyzer.connect()
|
||||
|
||||
report = analyzer.analyze()
|
||||
|
||||
assert report is not None
|
||||
assert report.database_type == "mongodb"
|
||||
assert isinstance(report.slow_queries, list)
|
||||
assert isinstance(report.index_recommendations, list)
|
||||
assert isinstance(report.database_metrics, dict)
|
||||
|
||||
@patch('db_performance_check.psycopg2')
|
||||
def test_analyze_postgres(self, mock_psycopg2, mock_postgres_conn):
|
||||
"""Test PostgreSQL performance analysis."""
|
||||
mock_conn, mock_cursor = mock_postgres_conn
|
||||
mock_psycopg2.connect.return_value = mock_conn
|
||||
|
||||
# Mock cursor results
|
||||
mock_cursor.fetchone.side_effect = [
|
||||
{"has_extension": True}, # pg_stat_statements check
|
||||
{"connections": 10, "commits": 1000, "rollbacks": 5}, # stats
|
||||
{"db_size": 1024 * 1024 * 500}, # database size
|
||||
{"cache_hit_ratio": 0.95} # cache hit ratio
|
||||
]
|
||||
|
||||
mock_cursor.fetchall.side_effect = [
|
||||
# Slow queries
|
||||
[
|
||||
{
|
||||
"query": "SELECT * FROM users",
|
||||
"mean_exec_time": 150.5,
|
||||
"calls": 100,
|
||||
"total_exec_time": 15050
|
||||
}
|
||||
],
|
||||
# Sequential scans
|
||||
[
|
||||
{
|
||||
"schemaname": "public",
|
||||
"tablename": "users",
|
||||
"seq_scan": 5000,
|
||||
"seq_tup_read": 500000,
|
||||
"idx_scan": 100
|
||||
}
|
||||
],
|
||||
# Unused indexes
|
||||
[]
|
||||
]
|
||||
|
||||
analyzer = PerformanceAnalyzer("postgres", "postgresql://localhost")
|
||||
analyzer.connect()
|
||||
|
||||
report = analyzer.analyze()
|
||||
|
||||
assert report is not None
|
||||
assert report.database_type == "postgres"
|
||||
assert len(report.slow_queries) > 0
|
||||
assert len(report.index_recommendations) > 0
|
||||
|
||||
def test_print_report(self, capsys):
|
||||
"""Test report printing."""
|
||||
analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost")
|
||||
|
||||
report = PerformanceReport(
|
||||
database_type="mongodb",
|
||||
database_name="testdb",
|
||||
timestamp=datetime.now(),
|
||||
slow_queries=[
|
||||
SlowQuery(
|
||||
query="db.users.find({age: {$gte: 18}})",
|
||||
execution_time_ms=150.5,
|
||||
count=10,
|
||||
collection_or_table="users"
|
||||
)
|
||||
],
|
||||
index_recommendations=[
|
||||
IndexRecommendation(
|
||||
collection_or_table="users",
|
||||
fields=["age"],
|
||||
reason="Frequently queried field",
|
||||
estimated_benefit="High"
|
||||
)
|
||||
],
|
||||
database_metrics={
|
||||
"connections": 10,
|
||||
"database_size_mb": 100.5
|
||||
}
|
||||
)
|
||||
|
||||
analyzer.print_report(report)
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert "Database Performance Report" in captured.out
|
||||
assert "testdb" in captured.out
|
||||
assert "150.5ms" in captured.out
|
||||
assert "users" in captured.out
|
||||
|
||||
def test_save_report(self, tmp_path):
|
||||
"""Test saving report to JSON."""
|
||||
analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost")
|
||||
|
||||
report = PerformanceReport(
|
||||
database_type="mongodb",
|
||||
database_name="testdb",
|
||||
timestamp=datetime.now(),
|
||||
slow_queries=[],
|
||||
index_recommendations=[],
|
||||
database_metrics={}
|
||||
)
|
||||
|
||||
output_file = tmp_path / "report.json"
|
||||
analyzer.save_report(report, str(output_file))
|
||||
|
||||
assert output_file.exists()
|
||||
|
||||
with open(output_file) as f:
|
||||
data = json.load(f)
|
||||
assert data["database_type"] == "mongodb"
|
||||
assert data["database_name"] == "testdb"
|
||||
|
||||
def test_disconnect(self):
|
||||
"""Test disconnection."""
|
||||
analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost")
|
||||
|
||||
# Mock client and connection
|
||||
analyzer.client = MagicMock()
|
||||
analyzer.conn = MagicMock()
|
||||
|
||||
analyzer.disconnect()
|
||||
|
||||
analyzer.client.close.assert_called_once()
|
||||
analyzer.conn.close.assert_called_once()
|
||||
|
||||
@patch('db_performance_check.MongoClient')
|
||||
def test_analyze_error_handling(self, mock_client_class, mock_mongo_client):
|
||||
"""Test error handling during analysis."""
|
||||
mock_client, mock_db = mock_mongo_client
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
# Simulate error
|
||||
mock_db.command.side_effect = Exception("Database error")
|
||||
|
||||
analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost")
|
||||
analyzer.connect()
|
||||
|
||||
report = analyzer.analyze()
|
||||
|
||||
assert report is None
|
||||
|
||||
|
||||
class TestIntegration:
|
||||
"""Integration tests."""
|
||||
|
||||
@patch('db_performance_check.MongoClient')
|
||||
def test_full_mongodb_workflow(self, mock_client_class, mock_mongo_client, tmp_path):
|
||||
"""Test complete MongoDB analysis workflow."""
|
||||
mock_client, mock_db = mock_mongo_client
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
# Setup mocks
|
||||
mock_db.command.return_value = {"was": 0}
|
||||
mock_db.system.profile.find.return_value.sort.return_value = []
|
||||
mock_db.list_collection_names.return_value = []
|
||||
mock_client.admin.command.return_value = {
|
||||
"connections": {"current": 10},
|
||||
"opcounters": {"query": 1000}
|
||||
}
|
||||
|
||||
analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost", 100)
|
||||
|
||||
# Connect
|
||||
assert analyzer.connect() is True
|
||||
|
||||
# Analyze
|
||||
report = analyzer.analyze()
|
||||
assert report is not None
|
||||
|
||||
# Save report
|
||||
output_file = tmp_path / "report.json"
|
||||
analyzer.save_report(report, str(output_file))
|
||||
assert output_file.exists()
|
||||
|
||||
# Disconnect
|
||||
analyzer.disconnect()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
Reference in New Issue
Block a user