389 lines
14 KiB
Python
389 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Metrics Tracker
|
|
Tracks actual time saved vs estimates
|
|
Measures real impact of automation
|
|
"""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from datetime import datetime
|
|
from typing import Dict, List, Optional
|
|
|
|
class MetricsTracker:
|
|
"""Tracks automation effectiveness metrics"""
|
|
|
|
def __init__(self, session_id: str, storage_path: str = None):
|
|
self.session_id = session_id
|
|
if storage_path:
|
|
self.storage_path = Path(storage_path)
|
|
else:
|
|
self.storage_path = Path(f".claude/meta-automation/metrics/{session_id}.json")
|
|
|
|
self.storage_path.parent.mkdir(parents=True, exist_ok=True)
|
|
self.metrics = self._load_or_create()
|
|
|
|
def _load_or_create(self) -> Dict:
|
|
"""Load existing metrics or create new"""
|
|
if self.storage_path.exists():
|
|
try:
|
|
with open(self.storage_path, 'r') as f:
|
|
return json.load(f)
|
|
except:
|
|
return self._create_new()
|
|
return self._create_new()
|
|
|
|
def _create_new(self) -> Dict:
|
|
"""Create new metrics structure"""
|
|
return {
|
|
'session_id': self.session_id,
|
|
'created_at': datetime.now().isoformat(),
|
|
'project_info': {},
|
|
'automation_generated': {
|
|
'agents': [],
|
|
'skills': [],
|
|
'commands': [],
|
|
'hooks': []
|
|
},
|
|
'time_tracking': {
|
|
'setup_time_minutes': 0,
|
|
'estimated_time_saved_hours': 0,
|
|
'actual_time_saved_hours': 0,
|
|
'accuracy': 0
|
|
},
|
|
'usage_metrics': {
|
|
'skills_run_count': {},
|
|
'commands_run_count': {},
|
|
'automation_frequency': []
|
|
},
|
|
'value_metrics': {
|
|
'issues_prevented': 0,
|
|
'quality_improvements': [],
|
|
'deployment_count': 0,
|
|
'test_failures_caught': 0
|
|
},
|
|
'cost_metrics': {
|
|
'setup_cost': 0,
|
|
'ongoing_cost': 0,
|
|
'total_cost': 0
|
|
},
|
|
'user_feedback': {
|
|
'satisfaction_ratings': [],
|
|
'comments': [],
|
|
'pain_points_resolved': []
|
|
}
|
|
}
|
|
|
|
def _save(self):
|
|
"""Save metrics to disk"""
|
|
with open(self.storage_path, 'w') as f:
|
|
json.dump(self.metrics, f, indent=2)
|
|
|
|
def set_project_info(self, info: Dict):
|
|
"""Set project information"""
|
|
self.metrics['project_info'] = {
|
|
**info,
|
|
'recorded_at': datetime.now().isoformat()
|
|
}
|
|
self._save()
|
|
|
|
def record_automation_generated(self, category: str, items: List[str]):
|
|
"""
|
|
Record what automation was generated
|
|
|
|
Args:
|
|
category: 'agents', 'skills', 'commands', 'hooks'
|
|
items: List of generated items
|
|
"""
|
|
if category in self.metrics['automation_generated']:
|
|
self.metrics['automation_generated'][category].extend(items)
|
|
self._save()
|
|
|
|
def record_setup_time(self, minutes: int):
|
|
"""Record time spent setting up automation"""
|
|
self.metrics['time_tracking']['setup_time_minutes'] = minutes
|
|
self._save()
|
|
|
|
def record_estimated_time_saved(self, hours: float):
|
|
"""Record estimated time savings"""
|
|
self.metrics['time_tracking']['estimated_time_saved_hours'] = hours
|
|
self._save()
|
|
|
|
def record_actual_time_saved(self, hours: float, description: str):
|
|
"""
|
|
Record actual time saved from automation
|
|
|
|
Args:
|
|
hours: Hours actually saved
|
|
description: What was automated
|
|
"""
|
|
current = self.metrics['time_tracking']['actual_time_saved_hours']
|
|
self.metrics['time_tracking']['actual_time_saved_hours'] = current + hours
|
|
|
|
# Calculate accuracy
|
|
estimated = self.metrics['time_tracking']['estimated_time_saved_hours']
|
|
if estimated > 0:
|
|
actual = self.metrics['time_tracking']['actual_time_saved_hours']
|
|
self.metrics['time_tracking']['accuracy'] = round((actual / estimated) * 100, 1)
|
|
|
|
# Track individual savings
|
|
if 'time_savings_breakdown' not in self.metrics:
|
|
self.metrics['time_savings_breakdown'] = []
|
|
|
|
self.metrics['time_savings_breakdown'].append({
|
|
'hours_saved': hours,
|
|
'description': description,
|
|
'recorded_at': datetime.now().isoformat()
|
|
})
|
|
|
|
self._save()
|
|
|
|
def record_skill_usage(self, skill_name: str):
|
|
"""Record that a skill was used"""
|
|
if skill_name not in self.metrics['usage_metrics']['skills_run_count']:
|
|
self.metrics['usage_metrics']['skills_run_count'][skill_name] = 0
|
|
|
|
self.metrics['usage_metrics']['skills_run_count'][skill_name] += 1
|
|
self._save()
|
|
|
|
def record_command_usage(self, command_name: str):
|
|
"""Record that a command was used"""
|
|
if command_name not in self.metrics['usage_metrics']['commands_run_count']:
|
|
self.metrics['usage_metrics']['commands_run_count'][command_name] = 0
|
|
|
|
self.metrics['usage_metrics']['commands_run_count'][command_name] += 1
|
|
self._save()
|
|
|
|
def record_issue_prevented(self, issue_type: str, description: str):
|
|
"""Record that automation prevented an issue"""
|
|
self.metrics['value_metrics']['issues_prevented'] += 1
|
|
|
|
if 'prevented_issues' not in self.metrics['value_metrics']:
|
|
self.metrics['value_metrics']['prevented_issues'] = []
|
|
|
|
self.metrics['value_metrics']['prevented_issues'].append({
|
|
'type': issue_type,
|
|
'description': description,
|
|
'prevented_at': datetime.now().isoformat()
|
|
})
|
|
|
|
self._save()
|
|
|
|
def record_quality_improvement(self, metric: str, before: float, after: float):
|
|
"""
|
|
Record quality improvement
|
|
|
|
Args:
|
|
metric: What improved (e.g., 'test_coverage', 'build_success_rate')
|
|
before: Value before automation
|
|
after: Value after automation
|
|
"""
|
|
improvement = {
|
|
'metric': metric,
|
|
'before': before,
|
|
'after': after,
|
|
'improvement_percent': round(((after - before) / before) * 100, 1) if before > 0 else 0,
|
|
'recorded_at': datetime.now().isoformat()
|
|
}
|
|
|
|
self.metrics['value_metrics']['quality_improvements'].append(improvement)
|
|
self._save()
|
|
|
|
def record_user_feedback(self, rating: int, comment: str = None):
|
|
"""
|
|
Record user satisfaction
|
|
|
|
Args:
|
|
rating: 1-5 rating
|
|
comment: Optional comment
|
|
"""
|
|
self.metrics['user_feedback']['satisfaction_ratings'].append({
|
|
'rating': rating,
|
|
'comment': comment,
|
|
'recorded_at': datetime.now().isoformat()
|
|
})
|
|
|
|
self._save()
|
|
|
|
def get_roi(self) -> Dict:
|
|
"""Calculate return on investment"""
|
|
setup_time = self.metrics['time_tracking']['setup_time_minutes'] / 60 # hours
|
|
actual_saved = self.metrics['time_tracking']['actual_time_saved_hours']
|
|
|
|
if setup_time == 0:
|
|
return {
|
|
'roi': 0,
|
|
'message': 'No setup time recorded'
|
|
}
|
|
|
|
roi = actual_saved / setup_time
|
|
|
|
return {
|
|
'roi': round(roi, 1),
|
|
'setup_hours': round(setup_time, 1),
|
|
'saved_hours': round(actual_saved, 1),
|
|
'net_gain_hours': round(actual_saved - setup_time, 1),
|
|
'break_even_reached': actual_saved > setup_time
|
|
}
|
|
|
|
def get_effectiveness(self) -> Dict:
|
|
"""Calculate automation effectiveness"""
|
|
generated = self.metrics['automation_generated']
|
|
usage = self.metrics['usage_metrics']
|
|
|
|
total_generated = sum(len(items) for items in generated.values())
|
|
total_used = (
|
|
len(usage['skills_run_count']) +
|
|
len(usage['commands_run_count'])
|
|
)
|
|
|
|
if total_generated == 0:
|
|
return {
|
|
'effectiveness': 0,
|
|
'message': 'No automation generated yet'
|
|
}
|
|
|
|
effectiveness = (total_used / total_generated) * 100
|
|
|
|
return {
|
|
'effectiveness_percent': round(effectiveness, 1),
|
|
'total_generated': total_generated,
|
|
'total_used': total_used,
|
|
'unused': total_generated - total_used
|
|
}
|
|
|
|
def get_summary(self) -> Dict:
|
|
"""Get comprehensive metrics summary"""
|
|
roi = self.get_roi()
|
|
effectiveness = self.get_effectiveness()
|
|
|
|
avg_satisfaction = 0
|
|
if self.metrics['user_feedback']['satisfaction_ratings']:
|
|
ratings = [r['rating'] for r in self.metrics['user_feedback']['satisfaction_ratings']]
|
|
avg_satisfaction = round(sum(ratings) / len(ratings), 1)
|
|
|
|
return {
|
|
'session_id': self.session_id,
|
|
'project': self.metrics['project_info'].get('project_type', 'unknown'),
|
|
'automation_generated': {
|
|
category: len(items)
|
|
for category, items in self.metrics['automation_generated'].items()
|
|
},
|
|
'time_metrics': {
|
|
'setup_time_hours': round(self.metrics['time_tracking']['setup_time_minutes'] / 60, 1),
|
|
'estimated_saved_hours': self.metrics['time_tracking']['estimated_time_saved_hours'],
|
|
'actual_saved_hours': self.metrics['time_tracking']['actual_time_saved_hours'],
|
|
'accuracy': f"{self.metrics['time_tracking']['accuracy']}%",
|
|
'net_gain_hours': roi.get('net_gain_hours', 0)
|
|
},
|
|
'roi': roi,
|
|
'effectiveness': effectiveness,
|
|
'value': {
|
|
'issues_prevented': self.metrics['value_metrics']['issues_prevented'],
|
|
'quality_improvements_count': len(self.metrics['value_metrics']['quality_improvements']),
|
|
'average_satisfaction': avg_satisfaction
|
|
},
|
|
'most_used': {
|
|
'skills': sorted(
|
|
self.metrics['usage_metrics']['skills_run_count'].items(),
|
|
key=lambda x: x[1],
|
|
reverse=True
|
|
)[:3],
|
|
'commands': sorted(
|
|
self.metrics['usage_metrics']['commands_run_count'].items(),
|
|
key=lambda x: x[1],
|
|
reverse=True
|
|
)[:3]
|
|
}
|
|
}
|
|
|
|
def export_report(self) -> str:
|
|
"""Export formatted metrics report"""
|
|
summary = self.get_summary()
|
|
|
|
report = f"""
|
|
# Automation Metrics Report
|
|
**Session:** {summary['session_id']}
|
|
**Project Type:** {summary['project']}
|
|
|
|
## Automation Generated
|
|
- **Agents:** {summary['automation_generated']['agents']}
|
|
- **Skills:** {summary['automation_generated']['skills']}
|
|
- **Commands:** {summary['automation_generated']['commands']}
|
|
- **Hooks:** {summary['automation_generated']['hooks']}
|
|
|
|
## Time Savings
|
|
- **Setup Time:** {summary['time_metrics']['setup_time_hours']} hours
|
|
- **Estimated Savings:** {summary['time_metrics']['estimated_saved_hours']} hours
|
|
- **Actual Savings:** {summary['time_metrics']['actual_saved_hours']} hours
|
|
- **Accuracy:** {summary['time_metrics']['accuracy']}
|
|
- **Net Gain:** {summary['time_metrics']['net_gain_hours']} hours
|
|
|
|
## ROI
|
|
- **Return on Investment:** {summary['roi']['roi']}x
|
|
- **Break-Even:** {'✅ Yes' if summary['roi']['break_even_reached'] else '❌ Not yet'}
|
|
|
|
## Effectiveness
|
|
- **Usage Rate:** {summary['effectiveness']['effectiveness_percent']}%
|
|
- **Generated:** {summary['effectiveness']['total_generated']} items
|
|
- **Actually Used:** {summary['effectiveness']['total_used']} items
|
|
- **Unused:** {summary['effectiveness']['unused']} items
|
|
|
|
## Value Delivered
|
|
- **Issues Prevented:** {summary['value']['issues_prevented']}
|
|
- **Quality Improvements:** {summary['value']['quality_improvements_count']}
|
|
- **User Satisfaction:** {summary['value']['average_satisfaction']}/5
|
|
|
|
## Most Used Automation
|
|
"""
|
|
|
|
if summary['most_used']['skills']:
|
|
report += "\n**Skills:**\n"
|
|
for skill, count in summary['most_used']['skills']:
|
|
report += f"- {skill}: {count} times\n"
|
|
|
|
if summary['most_used']['commands']:
|
|
report += "\n**Commands:**\n"
|
|
for cmd, count in summary['most_used']['commands']:
|
|
report += f"- {cmd}: {count} times\n"
|
|
|
|
return report
|
|
|
|
# Example usage
|
|
if __name__ == '__main__':
|
|
tracker = MetricsTracker('test-session-123')
|
|
|
|
# Simulate automation setup
|
|
tracker.set_project_info({
|
|
'project_type': 'programming',
|
|
'project_name': 'my-web-app'
|
|
})
|
|
|
|
tracker.record_automation_generated('skills', ['security-scanner', 'test-generator'])
|
|
tracker.record_automation_generated('commands', ['/security-check', '/generate-tests'])
|
|
|
|
tracker.record_setup_time(30) # 30 minutes to set up
|
|
tracker.record_estimated_time_saved(50) # Estimated 50 hours saved
|
|
|
|
# Simulate usage over time
|
|
tracker.record_skill_usage('security-scanner')
|
|
tracker.record_skill_usage('security-scanner')
|
|
tracker.record_skill_usage('test-generator')
|
|
|
|
tracker.record_command_usage('/security-check')
|
|
|
|
# Record actual time saved
|
|
tracker.record_actual_time_saved(5, 'Security scan caught 3 vulnerabilities before deployment')
|
|
tracker.record_actual_time_saved(8, 'Auto-generated 15 test scaffolds')
|
|
|
|
# Record quality improvements
|
|
tracker.record_quality_improvement('test_coverage', 42, 75)
|
|
|
|
# Record issues prevented
|
|
tracker.record_issue_prevented('security', 'SQL injection vulnerability caught')
|
|
|
|
# User feedback
|
|
tracker.record_user_feedback(5, 'This saved me so much time!')
|
|
|
|
print(tracker.export_report())
|