From b1f89ff340ba6657a89ab9ba3bf8946db506cc1a Mon Sep 17 00:00:00 2001 From: Zhongwei Li Date: Sun, 30 Nov 2025 08:20:20 +0800 Subject: [PATCH] Initial commit --- .claude-plugin/plugin.json | 18 ++ README.md | 3 + commands/log.md | 34 ++++ commands/report.md | 123 ++++++++++++ commands/search.md | 131 +++++++++++++ mcp/server.json | 20 ++ plugin.lock.json | 113 +++++++++++ skills/skill-adapter/assets/README.md | 8 + .../skill-adapter/assets/config-template.json | 32 ++++ .../assets/dashboard_template.html | 125 ++++++++++++ .../assets/example_experiments.json | 110 +++++++++++ .../assets/experiment_report_template.docx | 178 ++++++++++++++++++ skills/skill-adapter/assets/skill-schema.json | 28 +++ skills/skill-adapter/assets/test-data.json | 27 +++ .../assets/visualization_styles.css | 100 ++++++++++ skills/skill-adapter/references/README.md | 8 + .../references/best-practices.md | 69 +++++++ skills/skill-adapter/references/examples.md | 70 +++++++ skills/skill-adapter/scripts/README.md | 8 + .../skill-adapter/scripts/helper-template.sh | 42 +++++ skills/skill-adapter/scripts/validation.sh | 32 ++++ 21 files changed, 1279 insertions(+) create mode 100644 .claude-plugin/plugin.json create mode 100644 README.md create mode 100644 commands/log.md create mode 100644 commands/report.md create mode 100644 commands/search.md create mode 100644 mcp/server.json create mode 100644 plugin.lock.json create mode 100644 skills/skill-adapter/assets/README.md create mode 100644 skills/skill-adapter/assets/config-template.json create mode 100644 skills/skill-adapter/assets/dashboard_template.html create mode 100644 skills/skill-adapter/assets/example_experiments.json create mode 100644 skills/skill-adapter/assets/experiment_report_template.docx create mode 100644 skills/skill-adapter/assets/skill-schema.json create mode 100644 skills/skill-adapter/assets/test-data.json create mode 100644 skills/skill-adapter/assets/visualization_styles.css create mode 100644 skills/skill-adapter/references/README.md create mode 100644 skills/skill-adapter/references/best-practices.md create mode 100644 skills/skill-adapter/references/examples.md create mode 100644 skills/skill-adapter/scripts/README.md create mode 100755 skills/skill-adapter/scripts/helper-template.sh create mode 100755 skills/skill-adapter/scripts/validation.sh diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..16ebf8f --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,18 @@ +{ + "name": "ai-experiment-logger", + "description": "Track and analyze AI experiments with a web dashboard and MCP tools", + "version": "1.0.0", + "author": { + "name": "Claude Code Plugins", + "email": "plugins@example.com" + }, + "skills": [ + "./skills" + ], + "commands": [ + "./commands" + ], + "mcp": [ + "./mcp" + ] +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..b571eaf --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# ai-experiment-logger + +Track and analyze AI experiments with a web dashboard and MCP tools diff --git a/commands/log.md b/commands/log.md new file mode 100644 index 0000000..e5d8813 --- /dev/null +++ b/commands/log.md @@ -0,0 +1,34 @@ +--- +description: Log a new AI experiment with terminal report +shortcut: log-exp +--- + +# Log AI Experiment + +When the user runs `/log-exp` or `/log`, help them log a new AI experiment by gathering the following information: + +## Required Information + +1. **AI Tool Used** - e.g., "ChatGPT o1-preview", "Claude Sonnet 3.5", "Gemini Pro" +2. **Prompt/Query** - What they asked the AI +3. **Result Summary** - Brief summary of the AI's response +4. **Effectiveness Rating** - 1 (poor) to 5 (excellent) +5. **Tags** (optional) - Comma-separated tags like "code-generation, python, debugging" +6. **Date** (optional) - Defaults to now if not provided + +## Process + +1. Ask for the information conversationally (don't make it feel like a form) +2. Once you have all required info, use the `log_experiment` MCP tool +3. Display a formatted terminal report showing: + - ✅ Success confirmation + - 📊 Experiment ID + - 🤖 AI Tool + - ⭐ Rating + - 🏷️ Tags + - 📅 Date/Time + +## Example Interaction + +``` +User: /log-exp \ No newline at end of file diff --git a/commands/report.md b/commands/report.md new file mode 100644 index 0000000..9895c2b --- /dev/null +++ b/commands/report.md @@ -0,0 +1,123 @@ +--- +description: Generate comprehensive terminal report of AI experiments +shortcut: ai-report +--- + +# AI Experiments Report + +When the user runs `/ai-report`, generate a comprehensive, well-formatted terminal report of their AI experiments. + +## Report Sections + +### 1. Summary Statistics + +Display at the top: +``` +╔══════════════════════════════════════════════════════════════╗ +║ AI EXPERIMENT LOGGER - SUMMARY REPORT ║ +╚══════════════════════════════════════════════════════════════╝ + +📊 OVERVIEW +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Total Experiments: [count] +Average Rating: [X.XX] ⭐ +Date Range: [earliest] → [latest] +``` + +### 2. Top Performing AI Tools + +Show the top 5 AI tools by average rating: +``` +🏆 TOP AI TOOLS BY PERFORMANCE +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +1. ChatGPT o1-preview ⭐ 4.85 avg (20 experiments) +2. Claude Sonnet 3.5 ⭐ 4.75 avg (18 experiments) +3. Gemini Pro ⭐ 4.20 avg (12 experiments) +``` + +### 3. Rating Distribution + +Show distribution visually: +``` +⭐ RATING DISTRIBUTION +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +5 stars ████████████████████████████ (28) +4 stars ████████████████████ (20) +3 stars ██████████ (10) +2 stars ████ (4) +1 star ██ (2) +``` + +### 4. Most Used Tags + +Show top 10 tags: +``` +🏷️ POPULAR TAGS +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +code-generation (15) debugging (12) research (10) +creative-writing (8) data-analysis (7) testing (6) +``` + +### 5. Recent Activity + +Show last 7 days: +``` +📈 RECENT ACTIVITY (Last 7 Days) +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +2025-10-13: ████████ (8 experiments) +2025-10-12: ██████ (6 experiments) +2025-10-11: ████ (4 experiments) +``` + +### 6. Latest Experiments + +Show the 5 most recent: +``` +📝 LATEST EXPERIMENTS +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +[Oct 13, 2:30 PM] ChatGPT o1-preview ⭐⭐⭐⭐⭐ + Prompt: Write a Python function to calculate Fibonacci + Tags: code-generation, python + +[Oct 13, 1:15 PM] Claude Sonnet 3.5 ⭐⭐⭐⭐ + Prompt: Debug React useState issue with async + Tags: debugging, react, javascript +``` + +## Data Sources + +Use the `get_statistics` and `list_experiments` MCP tools to gather all data. + +## Optional Filters + +Support these filters via arguments: +- `/ai-report tool:ChatGPT` - Show report for specific tool +- `/ai-report tag:code-generation` - Show report for specific tag +- `/ai-report days:30` - Show last 30 days only +- `/ai-report rating:5` - Show only 5-star experiments + +## Error Handling + +If no experiments exist: +``` +╔══════════════════════════════════════════════════════════════╗ +║ AI EXPERIMENT LOGGER - SUMMARY REPORT ║ +╚══════════════════════════════════════════════════════════════╝ + +📊 NO EXPERIMENTS YET + +Get started by logging your first AI experiment: + /log-exp + +Or use the MCP tool directly: + Use the log_experiment tool +``` + +## Implementation Notes + +- Use Unicode box drawing characters for clean terminal output +- Ensure all sections are properly aligned +- Use emoji for visual appeal but keep it professional +- Calculate percentages and averages with 2 decimal precision +- Sort all rankings by count/rating descending +- Format dates in user-friendly format (Oct 13, 2:30 PM) diff --git a/commands/search.md b/commands/search.md new file mode 100644 index 0000000..933b412 --- /dev/null +++ b/commands/search.md @@ -0,0 +1,131 @@ +--- +description: Search AI experiments and display formatted results +shortcut: search-exp +--- + +# Search AI Experiments + +When the user runs `/search-exp [query]`, search through all AI experiments and display formatted terminal results. + +## Search Behavior + +Search across these fields: +- AI tool name +- Prompt text +- Result text +- Tags + +## Usage Examples + +```bash +/search-exp code # Search for "code" +/search-exp tool:ChatGPT # Filter by tool +/search-exp tag:debugging # Filter by tag +/search-exp rating:5 # Filter by rating +/search-exp "python function" # Search phrase +``` + +## Output Format + +``` +╔══════════════════════════════════════════════════════════════╗ +║ AI EXPERIMENTS - SEARCH RESULTS ║ +╚══════════════════════════════════════════════════════════════╝ + +Found 12 experiments matching "code generation" + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +[1] Oct 13, 2:30 PM | ChatGPT o1-preview | ⭐⭐⭐⭐⭐ + +📝 Prompt: +Write a Python function to calculate Fibonacci numbers recursively + +💡 Result: +Provided clean recursive implementation with base cases. Included +time complexity analysis (O(2^n)). + +🏷️ Tags: code-generation, python, algorithms +🆔 ID: exp_1697234567890_abc123def + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +[2] Oct 12, 4:15 PM | Claude Sonnet 3.5 | ⭐⭐⭐⭐ + +📝 Prompt: +Generate TypeScript interfaces from JSON schema + +💡 Result: +Created well-typed interfaces with proper nested types and +optional fields. + +🏷️ Tags: code-generation, typescript, types +🆔 ID: exp_1697148912345_xyz789abc + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +📊 QUICK STATS +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Total results: 12 experiments +Average rating: 4.42 ⭐ +Most common tool: ChatGPT o1-preview (7 experiments) +Most common tag: code-generation (12 experiments) +``` + +## Advanced Filters + +Support combined filters: +```bash +/search-exp tool:ChatGPT rating:5 tag:python +``` + +Parse and apply each filter: +- `tool:X` → Filter by AI tool +- `rating:X` → Filter by rating (1-5) +- `tag:X` → Filter by tag +- `date:YYYY-MM-DD` → Filter by specific date +- `from:YYYY-MM-DD` → From date +- `to:YYYY-MM-DD` → To date + +## Empty Results + +``` +╔══════════════════════════════════════════════════════════════╗ +║ AI EXPERIMENTS - SEARCH RESULTS ║ +╚══════════════════════════════════════════════════════════════╝ + +🔍 No experiments found matching "xyz123" + +Try: + • Broadening your search terms + • Checking spelling + • Using /ai-report to see all experiments + • Using different filter criteria +``` + +## Sorting Options + +Support sorting with suffix: +```bash +/search-exp code :date # Sort by date (newest first) +/search-exp code :rating # Sort by rating (highest first) +/search-exp code :tool # Sort by tool name (alphabetical) +``` + +## Limit Results + +Default to showing 10 results, with pagination hint: +``` +Showing 10 of 45 results. Use filters to narrow down. +``` + +## Implementation + +Use the `list_experiments` MCP tool with appropriate filters: +- `searchQuery` for text search +- `aiTool` for tool filter +- `rating` for rating filter +- `tags` for tag filter +- `dateFrom` / `dateTo` for date range + +Format output to be readable in terminal with proper spacing and visual hierarchy. diff --git a/mcp/server.json b/mcp/server.json new file mode 100644 index 0000000..def8b0e --- /dev/null +++ b/mcp/server.json @@ -0,0 +1,20 @@ +{ + "mcpServers": { + "ai-experiment-logger": { + "command": "node", + "args": [ + "/absolute/path/to/plugins/mcp/ai-experiment-logger/dist/index.js" + ], + "description": "Track and analyze AI experiments with comprehensive logging and statistics", + "tools": [ + "log_experiment", + "list_experiments", + "get_experiment", + "update_experiment", + "delete_experiment", + "get_statistics", + "export_experiments" + ] + } + } +} diff --git a/plugin.lock.json b/plugin.lock.json new file mode 100644 index 0000000..d211d94 --- /dev/null +++ b/plugin.lock.json @@ -0,0 +1,113 @@ +{ + "$schema": "internal://schemas/plugin.lock.v1.json", + "pluginId": "gh:jeremylongshore/claude-code-plugins-plus:plugins/mcp/ai-experiment-logger", + "normalized": { + "repo": null, + "ref": "refs/tags/v20251128.0", + "commit": "dfd0aa72a7ad0ffccdd736ced8b800df31873db7", + "treeHash": "00689cfead76b232bddd9d8b444e69729881cf81ab79931b7a545e3522241352", + "generatedAt": "2025-11-28T10:18:03.639854Z", + "toolVersion": "publish_plugins.py@0.2.0" + }, + "origin": { + "remote": "git@github.com:zhongweili/42plugin-data.git", + "branch": "master", + "commit": "aa1497ed0949fd50e99e70d6324a29c5b34f9390", + "repoRoot": "/Users/zhongweili/projects/openmind/42plugin-data" + }, + "manifest": { + "name": "ai-experiment-logger", + "description": "Track and analyze AI experiments with a web dashboard and MCP tools", + "version": "1.0.0" + }, + "content": { + "files": [ + { + "path": "README.md", + "sha256": "8b5eeb5c76b7bc73b0cddd9d4a50777b7e355cd18eacc9f117882ec34107feb9" + }, + { + "path": "mcp/server.json", + "sha256": "191b2293909340d12f32f71da96a59652f64cf6a7563e05324f5ccde1ab45330" + }, + { + "path": ".claude-plugin/plugin.json", + "sha256": "4259ead45e690a94e29b720ad047e02ab1bee5b36c443bcb16e5468aa92445c2" + }, + { + "path": "commands/search.md", + "sha256": "88f88dd4e5e95558d33ef3e29bd5d8bc3df57c93ddbfaa8a6128944ef4e10814" + }, + { + "path": "commands/report.md", + "sha256": "e66f787b04d4193edc480c35e570cc703047bb21af6c01df0209894550f34d93" + }, + { + "path": "commands/log.md", + "sha256": "27ec2c20411d5d27aa94ac1acd79df5e49fb9edaf7f0667d7e2bcf9f6b55c63d" + }, + { + "path": "skills/skill-adapter/references/examples.md", + "sha256": "922bbc3c4ebf38b76f515b5c1998ebde6bf902233e00e2c5a0e9176f975a7572" + }, + { + "path": "skills/skill-adapter/references/best-practices.md", + "sha256": "c8f32b3566252f50daacd346d7045a1060c718ef5cfb07c55a0f2dec5f1fb39e" + }, + { + "path": "skills/skill-adapter/references/README.md", + "sha256": "e4e929cba28cf715ac5747fb78d87f1094e450fca4eae606d8a510addb8c0c28" + }, + { + "path": "skills/skill-adapter/scripts/helper-template.sh", + "sha256": "0881d5660a8a7045550d09ae0acc15642c24b70de6f08808120f47f86ccdf077" + }, + { + "path": "skills/skill-adapter/scripts/validation.sh", + "sha256": "92551a29a7f512d2036e4f1fb46c2a3dc6bff0f7dde4a9f699533e446db48502" + }, + { + "path": "skills/skill-adapter/scripts/README.md", + "sha256": "67d5b375e4021d083ec5f5faf524fbbc89e21e0a642a75dc2dcbf5754c654f35" + }, + { + "path": "skills/skill-adapter/assets/visualization_styles.css", + "sha256": "469fee226e5d7f3b7112f297f59308ef82b458a5afa4a5e44f3915f266ede44f" + }, + { + "path": "skills/skill-adapter/assets/test-data.json", + "sha256": "ac17dca3d6e253a5f39f2a2f1b388e5146043756b05d9ce7ac53a0042eee139d" + }, + { + "path": "skills/skill-adapter/assets/example_experiments.json", + "sha256": "75470e18c7d9fa0812197a674bc64d04d9eca673d94251be47764cee51f7f4a7" + }, + { + "path": "skills/skill-adapter/assets/README.md", + "sha256": "51cb44be8de439a943eda401a36941f5b3ab18c975012c253aeb97116d108caa" + }, + { + "path": "skills/skill-adapter/assets/skill-schema.json", + "sha256": "f5639ba823a24c9ac4fb21444c0717b7aefde1a4993682897f5bf544f863c2cd" + }, + { + "path": "skills/skill-adapter/assets/experiment_report_template.docx", + "sha256": "5a9bbd91710bcdf182b5fa3e1c19b584d549063898b71c6cf55beff45a11c7b1" + }, + { + "path": "skills/skill-adapter/assets/dashboard_template.html", + "sha256": "85e8f54349691cdcd2a78902eabb5beecf59da6a02b0b078febc6d36cccd52fd" + }, + { + "path": "skills/skill-adapter/assets/config-template.json", + "sha256": "0c2ba33d2d3c5ccb266c0848fc43caa68a2aa6a80ff315d4b378352711f83e1c" + } + ], + "dirSha256": "00689cfead76b232bddd9d8b444e69729881cf81ab79931b7a545e3522241352" + }, + "security": { + "scannedAt": null, + "scannerVersion": null, + "flags": [] + } +} \ No newline at end of file diff --git a/skills/skill-adapter/assets/README.md b/skills/skill-adapter/assets/README.md new file mode 100644 index 0000000..ace5fa1 --- /dev/null +++ b/skills/skill-adapter/assets/README.md @@ -0,0 +1,8 @@ +# Assets + +Bundled resources for ai-experiment-logger skill + +- [ ] dashboard_template.html: HTML template for the web dashboard, providing a starting point for customization. +- [ ] example_experiments.json: Example JSON file containing sample experiment data. +- [ ] visualization_styles.css: CSS file to style visualizations generated by the analysis scripts. +- [ ] experiment_report_template.docx: Word template for generating experiment reports. diff --git a/skills/skill-adapter/assets/config-template.json b/skills/skill-adapter/assets/config-template.json new file mode 100644 index 0000000..16f1712 --- /dev/null +++ b/skills/skill-adapter/assets/config-template.json @@ -0,0 +1,32 @@ +{ + "skill": { + "name": "skill-name", + "version": "1.0.0", + "enabled": true, + "settings": { + "verbose": false, + "autoActivate": true, + "toolRestrictions": true + } + }, + "triggers": { + "keywords": [ + "example-trigger-1", + "example-trigger-2" + ], + "patterns": [] + }, + "tools": { + "allowed": [ + "Read", + "Grep", + "Bash" + ], + "restricted": [] + }, + "metadata": { + "author": "Plugin Author", + "category": "general", + "tags": [] + } +} diff --git a/skills/skill-adapter/assets/dashboard_template.html b/skills/skill-adapter/assets/dashboard_template.html new file mode 100644 index 0000000..7dabe5b --- /dev/null +++ b/skills/skill-adapter/assets/dashboard_template.html @@ -0,0 +1,125 @@ + + + + + + AI Experiment Logger Dashboard + + + + +
+ +

AI Experiment Logger Dashboard

+ + +
+

Experiment Summary

+

Total Experiments: {{total_experiments}}

+

Successful Experiments: {{successful_experiments}}

+

Average Success Rate: {{average_success_rate}}%

+
+ + +
+

Recent Experiments

+
    + +
  • + Experiment ID: {{experiment_id_1}}
    + Description: {{experiment_description_1}}
    + Status: {{experiment_status_1}} +
    • +
  • + Experiment ID: {{experiment_id_2}}
    + Description: {{experiment_description_2}}
    + Status: {{experiment_status_2}} +
    • +
  • + Experiment ID: {{experiment_id_3}}
    + Description: {{experiment_description_3}}
    + Status: {{experiment_status_3}} +
    • + +
+
+ +
+ + + \ No newline at end of file diff --git a/skills/skill-adapter/assets/example_experiments.json b/skills/skill-adapter/assets/example_experiments.json new file mode 100644 index 0000000..64c129d --- /dev/null +++ b/skills/skill-adapter/assets/example_experiments.json @@ -0,0 +1,110 @@ +{ + "_comment": "Example JSON file for AI Experiment Logger plugin. Represents a collection of experiments.", + "experiments": [ + { + "_comment": "First experiment - a simple sentiment analysis task", + "experiment_id": "sentiment-analysis-v1", + "experiment_name": "Sentiment Analysis with Claude", + "description": "Experiment to evaluate Claude's sentiment analysis accuracy on customer reviews.", + "model": "Claude-v2", + "dataset": "Customer Reviews - Product A", + "date_created": "2024-01-15", + "date_completed": "2024-01-16", + "status": "completed", + "parameters": { + "temperature": 0.2, + "max_tokens": 200, + "prompt_template": "Analyze the sentiment of the following review: {review}" + }, + "metrics": { + "accuracy": 0.92, + "precision": 0.90, + "recall": 0.94, + "f1_score": 0.92 + }, + "artifacts": { + "results_csv": "sentiment_analysis_results.csv", + "model_config": "claude_config.json" + }, + "notes": "High accuracy achieved. Consider testing with more diverse datasets." + }, + { + "_comment": "Second experiment - a more complex text summarization task", + "experiment_id": "text-summarization-v2", + "experiment_name": "Text Summarization Experiment with Gemini", + "description": "Experiment to compare different summarization techniques using Gemini.", + "model": "Gemini-Pro", + "dataset": "News Articles - Technology", + "date_created": "2024-01-20", + "date_completed": "2024-01-22", + "status": "completed", + "parameters": { + "temperature": 0.5, + "max_tokens": 500, + "summarization_type": "extractive", + "prompt_template": "Summarize the following article: {article}" + }, + "metrics": { + "rouge_1": 0.45, + "rouge_2": 0.22, + "rouge_l": 0.38, + "bleu_score": 0.28 + }, + "artifacts": { + "summaries_json": "summaries.json", + "hyperparameter_search_log": "hyperparameter_search.log" + }, + "notes": "Extractive summarization performed better than abstractive. Needs further tuning." + }, + { + "_comment": "Third experiment - still in progress", + "experiment_id": "image-classification-v1", + "experiment_name": "Image Classification with a Custom Model", + "description": "Training and evaluating a custom image classification model.", + "model": "Custom CNN", + "dataset": "Custom Image Dataset - Flowers", + "date_created": "2024-01-25", + "date_completed": null, + "status": "running", + "parameters": { + "learning_rate": 0.001, + "batch_size": 32, + "epochs": 10 + }, + "metrics": { + "training_loss": 0.25, + "validation_accuracy": 0.85 + }, + "artifacts": { + "model_checkpoint": "model_checkpoint_epoch_5.pth", + "training_logs": "training.log" + }, + "notes": "Training in progress. Validation accuracy is improving." + }, + { + "_comment": "Fourth experiment - trying to compare the models", + "experiment_id": "model-comparison-v1", + "experiment_name": "Model Comparison for Question Answering", + "description": "Comparing Claude and Gemini for question answering accuracy", + "model": "Claude-v2, Gemini-Pro", + "dataset": "SQuAD", + "date_created": "2024-02-01", + "date_completed": "2024-02-03", + "status": "completed", + "parameters": { + "question_length": "short", + "context_length": "medium" + }, + "metrics": { + "claude_exact_match": 0.75, + "claude_f1": 0.82, + "gemini_exact_match": 0.78, + "gemini_f1": 0.85 + }, + "artifacts": { + "results_table": "qa_results.csv" + }, + "notes": "Gemini slightly outperforms Claude on F1 score. Further investigation needed with longer contexts." + } + ] +} \ No newline at end of file diff --git a/skills/skill-adapter/assets/experiment_report_template.docx b/skills/skill-adapter/assets/experiment_report_template.docx new file mode 100644 index 0000000..f6e9f9e --- /dev/null +++ b/skills/skill-adapter/assets/experiment_report_template.docx @@ -0,0 +1,178 @@ + + + + + + AI Experiment Report + + + + + + + + + + + + + Experiment Title: [EXPERIMENT_TITLE] + + + + + + + + + + + Generated on: [REPORT_GENERATION_DATE] + + + + + + 1. Introduction + + + + + This report summarizes the results of the AI experiment titled "[EXPERIMENT_TITLE]". The experiment aimed to [EXPERIMENT_OBJECTIVE]. + + + + + + 2. Experiment Setup + + + + + 2.1 AI Model: + + + + + [AI_MODEL_USED] + + + + + 2.2 Input Data: + + + + + [INPUT_DATA_DESCRIPTION] (e.g., dataset name, size, preprocessing steps) + + + + + 2.3 Parameters: + + + + + [PARAMETER_SETTINGS] (e.g., learning rate, batch size, number of epochs) + + + + + 2.4 Evaluation Metrics: + + + + + [EVALUATION_METRICS] (e.g., accuracy, precision, recall, F1-score) + + + + + + 3. Results + + + + + 3.1 Overall Performance: + + + + + [OVERALL_PERFORMANCE_SUMMARY] (e.g., The model achieved an accuracy of 92%.) + + + + + 3.2 Detailed Results: + + + + + [DETAILED_RESULTS_TABLE] (Insert a table or chart showing the results for each evaluation metric.) + + + + + [INCLUDE_VISUALIZATIONS_HERE] (Optional: Insert charts or graphs visualizing the results. For example, a graph of accuracy vs. epoch.) + + + + + + 4. Analysis + + + + + [ANALYSIS_OF_RESULTS] (Discuss the results, including strengths, weaknesses, and potential areas for improvement.) + + + + + + 5. Conclusion + + + + + [CONCLUSION_SUMMARY] (Summarize the main findings of the experiment and their implications.) + + + + + + 6. Recommendations + + + + + [RECOMMENDATIONS_FOR_FUTURE_EXPERIMENTS] (Suggest next steps or modifications for future experiments.) + + + + + + 7. Appendix (Optional) + + + + + [APPENDIX_CONTENT] (Include any additional information, such as code snippets, raw data, or detailed logs.) + + + + + + + + + \ No newline at end of file diff --git a/skills/skill-adapter/assets/skill-schema.json b/skills/skill-adapter/assets/skill-schema.json new file mode 100644 index 0000000..8dc154c --- /dev/null +++ b/skills/skill-adapter/assets/skill-schema.json @@ -0,0 +1,28 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Claude Skill Configuration", + "type": "object", + "required": ["name", "description"], + "properties": { + "name": { + "type": "string", + "pattern": "^[a-z0-9-]+$", + "maxLength": 64, + "description": "Skill identifier (lowercase, hyphens only)" + }, + "description": { + "type": "string", + "maxLength": 1024, + "description": "What the skill does and when to use it" + }, + "allowed-tools": { + "type": "string", + "description": "Comma-separated list of allowed tools" + }, + "version": { + "type": "string", + "pattern": "^\\d+\\.\\d+\\.\\d+$", + "description": "Semantic version (x.y.z)" + } + } +} diff --git a/skills/skill-adapter/assets/test-data.json b/skills/skill-adapter/assets/test-data.json new file mode 100644 index 0000000..f0cd871 --- /dev/null +++ b/skills/skill-adapter/assets/test-data.json @@ -0,0 +1,27 @@ +{ + "testCases": [ + { + "name": "Basic activation test", + "input": "trigger phrase example", + "expected": { + "activated": true, + "toolsUsed": ["Read", "Grep"], + "success": true + } + }, + { + "name": "Complex workflow test", + "input": "multi-step trigger example", + "expected": { + "activated": true, + "steps": 3, + "toolsUsed": ["Read", "Write", "Bash"], + "success": true + } + } + ], + "fixtures": { + "sampleInput": "example data", + "expectedOutput": "processed result" + } +} diff --git a/skills/skill-adapter/assets/visualization_styles.css b/skills/skill-adapter/assets/visualization_styles.css new file mode 100644 index 0000000..d5ca979 --- /dev/null +++ b/skills/skill-adapter/assets/visualization_styles.css @@ -0,0 +1,100 @@ +/* + visualization_styles.css - Stylesheet for AI Experiment Logger visualizations. + + This file provides CSS rules to style visualizations generated by the + analysis scripts within the AI Experiment Logger plugin. + + Customize these styles to match your desired aesthetic and ensure + clarity and readability of your visualizations. + + Remember to link this stylesheet to your HTML files where visualizations + are displayed. Example: + +*/ + +/* General Styles */ +body { + font-family: sans-serif; /* Placeholder: Choose your preferred font */ + margin: 0; + padding: 0; +} + +.visualization-container { + /* Styles for the container holding the entire visualization */ + width: 100%; /* Adjust as needed */ + margin: 10px auto; /* Center horizontally */ + padding: 15px; + border: 1px solid #ddd; /* Light gray border */ + background-color: #f9f9f9; /* Light background */ + box-sizing: border-box; /* Include padding and border in element's total width and height */ +} + +h2 { + /* Styles for visualization titles */ + font-size: 1.5em; + margin-bottom: 10px; + color: #333; /* Dark gray title color */ +} + +/* Styles for tables */ +table { + width: 100%; + border-collapse: collapse; /* Remove spacing between table cells */ + margin-bottom: 20px; +} + +th, td { + border: 1px solid #ddd; + padding: 8px; + text-align: left; +} + +th { + background-color: #f2f2f2; /* Light gray background for header cells */ + font-weight: bold; +} + +/* Styles for charts (modify based on your charting library) */ +.chart-container { + /* Placeholder: Styles for containing chart elements */ + width: 100%; + height: 300px; /* Adjust height as needed */ + margin-bottom: 20px; +} + +/* Placeholder: Add styles for specific chart elements + Example (using a hypothetical chart library): + .chart-bar { + background-color: steelblue; + color: white; + } +*/ + +/* Styles for labels */ +.label { + font-size: 0.8em; + color: #777; + margin-top: 5px; + display: block; /* Ensure label is on its own line */ +} + +/* Styles for error messages */ +.error-message { + color: red; + font-weight: bold; + margin-top: 10px; +} + +/* Add more styles as needed for different visualization types and elements */ + +/* Example: Styles for a specific type of chart */ +.experiment-comparison-chart { + /* Add specific styling for experiment comparison charts here */ + /* Example: adjust colors, fonts, etc. */ +} + +/* Example: Styles for a specific element within a chart */ +.experiment-comparison-chart .axis-label { + font-size: 0.9em; + color: #555; +} \ No newline at end of file diff --git a/skills/skill-adapter/references/README.md b/skills/skill-adapter/references/README.md new file mode 100644 index 0000000..a5e6bdb --- /dev/null +++ b/skills/skill-adapter/references/README.md @@ -0,0 +1,8 @@ +# References + +Bundled resources for ai-experiment-logger skill + +- [ ] experiment_schema.md: Detailed documentation of the experiment data schema, including field descriptions, data types, and validation rules. +- [ ] statistics_definitions.md: Definitions and explanations of the statistical metrics used in the analysis reports. +- [ ] best_practices.md: Best practices for designing and conducting AI experiments, including prompt engineering tips and tool selection guidelines. +- [ ] api_documentation.md: Documentation for interacting with the experiment logger's data programmatically. diff --git a/skills/skill-adapter/references/best-practices.md b/skills/skill-adapter/references/best-practices.md new file mode 100644 index 0000000..3505048 --- /dev/null +++ b/skills/skill-adapter/references/best-practices.md @@ -0,0 +1,69 @@ +# Skill Best Practices + +Guidelines for optimal skill usage and development. + +## For Users + +### Activation Best Practices + +1. **Use Clear Trigger Phrases** + - Match phrases from skill description + - Be specific about intent + - Provide necessary context + +2. **Provide Sufficient Context** + - Include relevant file paths + - Specify scope of analysis + - Mention any constraints + +3. **Understand Tool Permissions** + - Check allowed-tools in frontmatter + - Know what the skill can/cannot do + - Request appropriate actions + +### Workflow Optimization + +- Start with simple requests +- Build up to complex workflows +- Verify each step before proceeding +- Use skill consistently for related tasks + +## For Developers + +### Skill Development Guidelines + +1. **Clear Descriptions** + - Include explicit trigger phrases + - Document all capabilities + - Specify limitations + +2. **Proper Tool Permissions** + - Use minimal necessary tools + - Document security implications + - Test with restricted tools + +3. **Comprehensive Documentation** + - Provide usage examples + - Document common pitfalls + - Include troubleshooting guide + +### Maintenance + +- Keep version updated +- Test after tool updates +- Monitor user feedback +- Iterate on descriptions + +## Performance Tips + +- Scope skills to specific domains +- Avoid overlapping trigger phrases +- Keep descriptions under 1024 chars +- Test activation reliability + +## Security Considerations + +- Never include secrets in skill files +- Validate all inputs +- Use read-only tools when possible +- Document security requirements diff --git a/skills/skill-adapter/references/examples.md b/skills/skill-adapter/references/examples.md new file mode 100644 index 0000000..b1d8bd2 --- /dev/null +++ b/skills/skill-adapter/references/examples.md @@ -0,0 +1,70 @@ +# Skill Usage Examples + +This document provides practical examples of how to use this skill effectively. + +## Basic Usage + +### Example 1: Simple Activation + +**User Request:** +``` +[Describe trigger phrase here] +``` + +**Skill Response:** +1. Analyzes the request +2. Performs the required action +3. Returns results + +### Example 2: Complex Workflow + +**User Request:** +``` +[Describe complex scenario] +``` + +**Workflow:** +1. Step 1: Initial analysis +2. Step 2: Data processing +3. Step 3: Result generation +4. Step 4: Validation + +## Advanced Patterns + +### Pattern 1: Chaining Operations + +Combine this skill with other tools: +``` +Step 1: Use this skill for [purpose] +Step 2: Chain with [other tool] +Step 3: Finalize with [action] +``` + +### Pattern 2: Error Handling + +If issues occur: +- Check trigger phrase matches +- Verify context is available +- Review allowed-tools permissions + +## Tips & Best Practices + +- ✅ Be specific with trigger phrases +- ✅ Provide necessary context +- ✅ Check tool permissions match needs +- ❌ Avoid vague requests +- ❌ Don't mix unrelated tasks + +## Common Issues + +**Issue:** Skill doesn't activate +**Solution:** Use exact trigger phrases from description + +**Issue:** Unexpected results +**Solution:** Check input format and context + +## See Also + +- Main SKILL.md for full documentation +- scripts/ for automation helpers +- assets/ for configuration examples diff --git a/skills/skill-adapter/scripts/README.md b/skills/skill-adapter/scripts/README.md new file mode 100644 index 0000000..67e4828 --- /dev/null +++ b/skills/skill-adapter/scripts/README.md @@ -0,0 +1,8 @@ +# Scripts + +Bundled resources for ai-experiment-logger skill + +- [ ] analyze_experiments.py: Script to perform in-depth statistical analysis of logged experiments, generating reports on tool performance, prompt effectiveness, and other key metrics. +- [ ] export_to_db.py: Script to export experiment data to a database (e.g., PostgreSQL) for more robust storage and querying. +- [ ] import_from_csv.py: Script to import experiment data from a CSV file, allowing users to migrate existing data into the logger. +- [ ] validate_experiment.py: Script to validate experiment data before logging, ensuring data integrity and consistency. diff --git a/skills/skill-adapter/scripts/helper-template.sh b/skills/skill-adapter/scripts/helper-template.sh new file mode 100755 index 0000000..c4aae90 --- /dev/null +++ b/skills/skill-adapter/scripts/helper-template.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Helper script template for skill automation +# Customize this for your skill's specific needs + +set -e + +function show_usage() { + echo "Usage: $0 [options]" + echo "" + echo "Options:" + echo " -h, --help Show this help message" + echo " -v, --verbose Enable verbose output" + echo "" +} + +# Parse arguments +VERBOSE=false + +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + show_usage + exit 0 + ;; + -v|--verbose) + VERBOSE=true + shift + ;; + *) + echo "Unknown option: $1" + show_usage + exit 1 + ;; + esac +done + +# Your skill logic here +if [ "$VERBOSE" = true ]; then + echo "Running skill automation..." +fi + +echo "✅ Complete" diff --git a/skills/skill-adapter/scripts/validation.sh b/skills/skill-adapter/scripts/validation.sh new file mode 100755 index 0000000..590af58 --- /dev/null +++ b/skills/skill-adapter/scripts/validation.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Skill validation helper +# Validates skill activation and functionality + +set -e + +echo "🔍 Validating skill..." + +# Check if SKILL.md exists +if [ ! -f "../SKILL.md" ]; then + echo "❌ Error: SKILL.md not found" + exit 1 +fi + +# Validate frontmatter +if ! grep -q "^---$" "../SKILL.md"; then + echo "❌ Error: No frontmatter found" + exit 1 +fi + +# Check required fields +if ! grep -q "^name:" "../SKILL.md"; then + echo "❌ Error: Missing 'name' field" + exit 1 +fi + +if ! grep -q "^description:" "../SKILL.md"; then + echo "❌ Error: Missing 'description' field" + exit 1 +fi + +echo "✅ Skill validation passed"