From 5e822e4e98f81b539e5d8cc5fab58dce470a967a Mon Sep 17 00:00:00 2001 From: Zhongwei Li Date: Sun, 30 Nov 2025 08:19:52 +0800 Subject: [PATCH] Initial commit --- .claude-plugin/plugin.json | 15 + README.md | 3 + commands/log-setup.md | 25 ++ plugin.lock.json | 85 ++++++ skills/log-aggregation-setup/SKILL.md | 52 ++++ skills/log-aggregation-setup/assets/README.md | 11 + .../assets/dashboard_elk.json | 204 +++++++++++++ .../assets/dashboard_loki.json | 288 ++++++++++++++++++ .../assets/dashboard_splunk.json | 141 +++++++++ .../assets/example_log_data.json | 107 +++++++ .../assets/loki_config_template.yaml | 75 +++++ .../assets/splunk_config_template.conf | 114 +++++++ .../references/README.md | 12 + .../log-aggregation-setup/scripts/README.md | 12 + 14 files changed, 1144 insertions(+) create mode 100644 .claude-plugin/plugin.json create mode 100644 README.md create mode 100644 commands/log-setup.md create mode 100644 plugin.lock.json create mode 100644 skills/log-aggregation-setup/SKILL.md create mode 100644 skills/log-aggregation-setup/assets/README.md create mode 100644 skills/log-aggregation-setup/assets/dashboard_elk.json create mode 100644 skills/log-aggregation-setup/assets/dashboard_loki.json create mode 100644 skills/log-aggregation-setup/assets/dashboard_splunk.json create mode 100644 skills/log-aggregation-setup/assets/example_log_data.json create mode 100644 skills/log-aggregation-setup/assets/loki_config_template.yaml create mode 100644 skills/log-aggregation-setup/assets/splunk_config_template.conf create mode 100644 skills/log-aggregation-setup/references/README.md create mode 100644 skills/log-aggregation-setup/scripts/README.md diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..f028db2 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,15 @@ +{ + "name": "log-aggregation-setup", + "description": "Set up log aggregation (ELK, Loki, Splunk)", + "version": "1.0.0", + "author": { + "name": "Claude Code Plugins", + "email": "[email protected]" + }, + "skills": [ + "./skills" + ], + "commands": [ + "./commands" + ] +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..c135bd6 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# log-aggregation-setup + +Set up log aggregation (ELK, Loki, Splunk) diff --git a/commands/log-setup.md b/commands/log-setup.md new file mode 100644 index 0000000..31f5ee9 --- /dev/null +++ b/commands/log-setup.md @@ -0,0 +1,25 @@ +--- +description: $(echo "$description" | cut -d' ' -f1-5) +--- + +# $(echo "$name" | sed 's/-/ /g' | sed 's/\b\(.\)/\u\1/g') + +$(echo "$description") + +## Key Features + +- Production-ready configurations +- Best practices implementation +- Security-first approach +- Scalable architecture +- Comprehensive documentation +- Multi-platform support + +## Example Usage + +This plugin generates complete configurations for your DevOps needs. +Specify your requirements and get production-ready code instantly. + +## When Invoked + +Generate configurations and setup code based on your specific requirements and infrastructure needs. diff --git a/plugin.lock.json b/plugin.lock.json new file mode 100644 index 0000000..218a27b --- /dev/null +++ b/plugin.lock.json @@ -0,0 +1,85 @@ +{ + "$schema": "internal://schemas/plugin.lock.v1.json", + "pluginId": "gh:jeremylongshore/claude-code-plugins-plus:plugins/devops/log-aggregation-setup", + "normalized": { + "repo": null, + "ref": "refs/tags/v20251128.0", + "commit": "3cbb065588fae10ff28f23a0d91950579de9f483", + "treeHash": "ce2c68f785eb71daae10d2ff5b0201839a260813753a2ffa6f7ef5f82417910d", + "generatedAt": "2025-11-28T10:18:32.618281Z", + "toolVersion": "publish_plugins.py@0.2.0" + }, + "origin": { + "remote": "git@github.com:zhongweili/42plugin-data.git", + "branch": "master", + "commit": "aa1497ed0949fd50e99e70d6324a29c5b34f9390", + "repoRoot": "/Users/zhongweili/projects/openmind/42plugin-data" + }, + "manifest": { + "name": "log-aggregation-setup", + "description": "Set up log aggregation (ELK, Loki, Splunk)", + "version": "1.0.0" + }, + "content": { + "files": [ + { + "path": "README.md", + "sha256": "082b8c65191f7344e6da19cd11a9648734d0537d952312fe63ba6806ccf5d27b" + }, + { + "path": ".claude-plugin/plugin.json", + "sha256": "858e1a44a88d1363d304a2f160cfcb942262cb42be907cdb41e1a71c5e8d0d9a" + }, + { + "path": "commands/log-setup.md", + "sha256": "353f80054a90cda1e6716da3628115ce829307fbbb83a15b64f1d37c96224a99" + }, + { + "path": "skills/log-aggregation-setup/SKILL.md", + "sha256": "de945437dc843fc26c5322ec985a05b8bba1b58c22dcfc12076bb01e31a7454d" + }, + { + "path": "skills/log-aggregation-setup/references/README.md", + "sha256": "da4a91a3b4dc543083e1fa035ba172a1e4dc5d8240baa4a612c427bcd4cefdb6" + }, + { + "path": "skills/log-aggregation-setup/scripts/README.md", + "sha256": "b7f0dd6709b8c198bc11fbed2a08e8062b62783b58c77eb1b3be4ed19c65e4cc" + }, + { + "path": "skills/log-aggregation-setup/assets/example_log_data.json", + "sha256": "1544addad466cc41a1a77b6a345bc1f630f6bdd7605369ba02f048deba5913d1" + }, + { + "path": "skills/log-aggregation-setup/assets/splunk_config_template.conf", + "sha256": "f97f2d4fb2cf8bf4ecbdbf2a0875e3999555e1db6ba9cd459b68abdaac1a394c" + }, + { + "path": "skills/log-aggregation-setup/assets/README.md", + "sha256": "8740fa1656a47b09c794eef6dec17110e2089ced3d413e3010370ad19f61a91f" + }, + { + "path": "skills/log-aggregation-setup/assets/dashboard_elk.json", + "sha256": "89e4b3ef58ceca702699a9247124ddca617c5dc657fe6ca1fb2e21c00e2d298c" + }, + { + "path": "skills/log-aggregation-setup/assets/dashboard_splunk.json", + "sha256": "f21530c49d54d7294cf9e74994d4b8c02ef7e86e3d728d39079139da9acdda1a" + }, + { + "path": "skills/log-aggregation-setup/assets/dashboard_loki.json", + "sha256": "1dd05ebe3ae5847c3710bf2f00e2b116d3cef463ad17bc99cdbd05127f89f947" + }, + { + "path": "skills/log-aggregation-setup/assets/loki_config_template.yaml", + "sha256": "f2c5e2a6a2f3a63a266f72b7b55cefbb27bd89d5aab68bbd2a0cfa2ece831392" + } + ], + "dirSha256": "ce2c68f785eb71daae10d2ff5b0201839a260813753a2ffa6f7ef5f82417910d" + }, + "security": { + "scannedAt": null, + "scannerVersion": null, + "flags": [] + } +} \ No newline at end of file diff --git a/skills/log-aggregation-setup/SKILL.md b/skills/log-aggregation-setup/SKILL.md new file mode 100644 index 0000000..b52beed --- /dev/null +++ b/skills/log-aggregation-setup/SKILL.md @@ -0,0 +1,52 @@ +--- +name: setting-up-log-aggregation +description: | + This skill sets up log aggregation solutions using ELK (Elasticsearch, Logstash, Kibana), Loki, or Splunk. It generates production-ready configurations and setup code based on specific requirements and infrastructure. Use this skill when the user requests to set up logging infrastructure, configure log aggregation, deploy ELK stack, deploy Loki, deploy Splunk, or needs help with observability. It is triggered by terms like "log aggregation," "ELK setup," "Loki configuration," "Splunk deployment," or similar requests for centralized logging solutions. +allowed-tools: Read, Write, Edit, Grep, Glob, Bash +version: 1.0.0 +--- + +## Overview + +This skill simplifies the deployment and configuration of log aggregation systems. It automates the process of setting up ELK, Loki, or Splunk, providing production-ready configurations tailored to your environment. + +## How It Works + +1. **Requirement Gathering**: The skill identifies the user's specific requirements, including the desired log aggregation platform (ELK, Loki, or Splunk), infrastructure details, and security considerations. +2. **Configuration Generation**: Based on the gathered requirements, the skill generates the necessary configuration files for the chosen platform. This includes configurations for data ingestion, processing, storage, and visualization. +3. **Setup Code Generation**: The skill provides the setup code needed to deploy and configure the log aggregation solution on the target infrastructure. This might include scripts, Docker Compose files, or other deployment artifacts. + +## When to Use This Skill + +This skill activates when you need to: +- Deploy a new log aggregation system. +- Configure an existing log aggregation system. +- Migrate from one log aggregation system to another. + +## Examples + +### Example 1: Deploying an ELK Stack + +User request: "Set up an ELK stack for my Kubernetes cluster to aggregate application logs." + +The skill will: +1. Generate Elasticsearch, Logstash, and Kibana configuration files optimized for Kubernetes. +2. Provide a Docker Compose file or Kubernetes manifests for deploying the ELK stack. + +### Example 2: Configuring Loki for a Docker Swarm + +User request: "Configure Loki to aggregate logs from my Docker Swarm environment." + +The skill will: +1. Generate a Loki configuration file optimized for Docker Swarm. +2. Provide instructions for deploying Loki as a service within the Swarm. + +## Best Practices + +- **Security**: Ensure that all generated configurations adhere to security best practices, including proper authentication and authorization mechanisms. +- **Scalability**: Design the log aggregation system to be scalable, allowing it to handle increasing log volumes over time. +- **Monitoring**: Implement monitoring for the log aggregation system itself to ensure its health and performance. + +## Integration + +This skill can integrate with other deployment and infrastructure management tools in the Claude Code ecosystem to automate the entire deployment process. It can also work with security analysis tools to ensure log data is securely handled. \ No newline at end of file diff --git a/skills/log-aggregation-setup/assets/README.md b/skills/log-aggregation-setup/assets/README.md new file mode 100644 index 0000000..8c831c6 --- /dev/null +++ b/skills/log-aggregation-setup/assets/README.md @@ -0,0 +1,11 @@ +# Assets + +Bundled resources for log-aggregation-setup skill + +- [ ] elk_config_template.conf: Template configuration file for Logstash. +- [ ] loki_config_template.yaml: Template configuration file for Loki. +- [ ] splunk_config_template.conf: Template configuration file for Splunk. +- [ ] example_log_data.json: Example log data in JSON format for testing the log aggregation setup. +- [ ] dashboard_elk.json: Example Kibana dashboard configuration. +- [ ] dashboard_loki.json: Example Grafana dashboard configuration for Loki. +- [ ] dashboard_splunk.json: Example Splunk dashboard configuration. diff --git a/skills/log-aggregation-setup/assets/dashboard_elk.json b/skills/log-aggregation-setup/assets/dashboard_elk.json new file mode 100644 index 0000000..752c6fc --- /dev/null +++ b/skills/log-aggregation-setup/assets/dashboard_elk.json @@ -0,0 +1,204 @@ +{ + "_comment": "Kibana Dashboard Configuration for ELK Stack", + "title": "System Performance and Log Analysis", + "description": "Dashboard providing insights into system performance and log data.", + "panels": [ + { + "id": "cpu_usage", + "type": "visualization", + "title": "CPU Usage", + "description": "Displays CPU usage over time.", + "visState": { + "type": "timeseries", + "params": { + "indexPattern": "system-metrics-*", + "timeField": "@timestamp", + "interval": "auto", + "metrics": [ + { + "field": "system.cpu.usage", + "type": "avg", + "alias": "Average CPU Usage" + } + ], + "xAxisMode": "timeseries", + "yAxisMode": "normal" + } + }, + "gridData": { + "x": 0, + "y": 0, + "w": 12, + "h": 6 + } + }, + { + "id": "memory_usage", + "type": "visualization", + "title": "Memory Usage", + "description": "Displays memory usage over time.", + "visState": { + "type": "timeseries", + "params": { + "indexPattern": "system-metrics-*", + "timeField": "@timestamp", + "interval": "auto", + "metrics": [ + { + "field": "system.memory.actual.used.pct", + "type": "avg", + "alias": "Average Memory Usage" + } + ], + "xAxisMode": "timeseries", + "yAxisMode": "normal" + } + }, + "gridData": { + "x": 0, + "y": 6, + "w": 12, + "h": 6 + } + }, + { + "id": "disk_usage", + "type": "visualization", + "title": "Disk Usage", + "description": "Displays disk usage over time.", + "visState": { + "type": "timeseries", + "params": { + "indexPattern": "system-metrics-*", + "timeField": "@timestamp", + "interval": "auto", + "metrics": [ + { + "field": "system.disk.used.pct", + "type": "avg", + "alias": "Average Disk Usage" + } + ], + "xAxisMode": "timeseries", + "yAxisMode": "normal" + } + }, + "gridData": { + "x": 12, + "y": 0, + "w": 12, + "h": 6 + } + }, + { + "id": "log_level_distribution", + "type": "visualization", + "title": "Log Level Distribution", + "description": "Displays the distribution of log levels.", + "visState": { + "type": "pie", + "params": { + "indexPattern": "application-logs-*", + "timeField": "@timestamp", + "interval": "auto", + "metrics": [ + { + "field": "log.level", + "type": "count", + "alias": "Count" + } + ], + "xAxisMode": "categorical", + "yAxisMode": "normal", + "terms": { + "field": "log.level", + "size": 5 + } + } + }, + "gridData": { + "x": 12, + "y": 6, + "w": 6, + "h": 6 + } + }, + { + "id": "error_rate", + "type": "visualization", + "title": "Error Rate", + "description": "Displays the rate of error logs over time.", + "visState": { + "type": "timeseries", + "params": { + "indexPattern": "application-logs-*", + "timeField": "@timestamp", + "interval": "auto", + "metrics": [ + { + "field": "log.level", + "type": "count", + "alias": "Error Count", + "filters": [ + { + "field": "log.level", + "operator": "is", + "value": "error" + } + ] + } + ], + "xAxisMode": "timeseries", + "yAxisMode": "normal" + } + }, + "gridData": { + "x": 18, + "y": 6, + "w": 6, + "h": 6 + } + }, + { + "id": "log_table", + "type": "visualization", + "title": "Recent Logs", + "description": "Displays a table of recent log entries.", + "visState": { + "type": "table", + "params": { + "indexPattern": "application-logs-*", + "timeField": "@timestamp", + "columns": [ + "@timestamp", + "log.level", + "message", + "service.name" + ], + "sort": { + "field": "@timestamp", + "direction": "desc" + }, + "pageSize": 10 + } + }, + "gridData": { + "x": 0, + "y": 12, + "w": 24, + "h": 6 + } + } + ], + "timeRestore": true, + "timeTo": "now", + "timeFrom": "now-15m", + "refreshInterval": { + "pause": false, + "value": 15000 + }, + "indexPatternRefName": "kibana_index_pattern_ref", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\": []}" + } +} \ No newline at end of file diff --git a/skills/log-aggregation-setup/assets/dashboard_loki.json b/skills/log-aggregation-setup/assets/dashboard_loki.json new file mode 100644 index 0000000..e0f9c5b --- /dev/null +++ b/skills/log-aggregation-setup/assets/dashboard_loki.json @@ -0,0 +1,288 @@ +{ + "_comment": "Grafana dashboard for Loki", + "dashboard": { + "annotations": { + "list": [] + }, + "description": "Example Grafana dashboard for Loki log aggregation.", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "_comment": "Panel: Logs overview", + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "display": "auto", + "filterable": true + }, + "mappings": [], + "min": null, + "max": null, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "dedupStrategy": "none", + "enableExemplar": true, + "prettifyJson": true, + "showCommonContext": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLines": true + }, + "pluginVersion": "7.5.7", + "targets": [ + { + "datasource": "${DS_LOKI}", + "editorMode": "code", + "expr": "{job=\"my-app\"} |= \"error\"", + "instant": false, + "queryType": "range", + "refId": "A" + } + ], + "title": "Error Logs", + "type": "logs" + }, + { + "_comment": "Panel: Log volume over time", + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "showPoints": "auto", + "spanNulls": false, + "stacking": "normal", + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": null, + "max": null, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "7.5.7", + "targets": [ + { + "datasource": "${DS_LOKI}", + "editorMode": "code", + "expr": "rate({job=\"my-app\"} |= `error` [1m])", + "instant": false, + "legendFormat": "{{job}}", + "queryType": "range", + "refId": "A" + } + ], + "title": "Error Log Volume", + "type": "timeseries" + }, + { + "_comment": "Panel: HTTP Request Latency", + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 1, + "showPoints": "auto", + "spanNulls": false, + "stacking": "normal", + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": null, + "max": null, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "7.5.7", + "targets": [ + { + "datasource": "${DS_LOKI}", + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate({job=\"my-app\", endpoint=\"/api/users\"} | json | unwrap duration [1m])) by (le))", + "instant": false, + "legendFormat": "99th percentile", + "queryType": "range", + "refId": "A" + } + ], + "title": "HTTP Request Latency (99th percentile)", + "type": "timeseries" + } + ], + "refresh": "1m", + "schemaVersion": 30, + "style": "dark", + "tags": [ + "loki", + "logs", + "example" + ], + "templating": { + "list": [ + { + "_comment": "Loki datasource variable", + "current": { + "text": "Loki", + "value": "Loki" + }, + "datasource": null, + "definition": "Loki", + "hide": 0, + "includeAll": false, + "label": "Loki Datasource", + "multi": false, + "name": "DS_LOKI", + "options": [], + "query": "Loki", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "datasource", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Loki Log Aggregation Dashboard", + "uid": "logi-aggregation-dashboard", + "version": 1 + } +} \ No newline at end of file diff --git a/skills/log-aggregation-setup/assets/dashboard_splunk.json b/skills/log-aggregation-setup/assets/dashboard_splunk.json new file mode 100644 index 0000000..89543fd --- /dev/null +++ b/skills/log-aggregation-setup/assets/dashboard_splunk.json @@ -0,0 +1,141 @@ +{ + "_comment": "Splunk Dashboard Configuration - Example", + "dashboard": { + "label": "Application Performance Overview", + "description": "A dashboard providing insights into application performance and health.", + "version": "1.0", + "layout": { + "type": "absolute", + "options": { + "width": "100%", + "height": "100%" + } + }, + "panels": [ + { + "id": "panel1", + "title": "Requests per Minute", + "description": "Shows the rate of incoming requests.", + "type": "timeseries", + "options": { + "xAxisTitle": "Time", + "yAxisTitle": "Requests/Minute" + }, + "search": { + "query": "index=main sourcetype=access_combined | timechart count by _time span=1m", + "earliest": "-15m", + "latest": "now" + }, + "position": { + "x": 0, + "y": 0, + "width": 6, + "height": 4 + } + }, + { + "id": "panel2", + "title": "Error Rate", + "description": "Displays the percentage of error responses.", + "type": "singlevalue", + "options": { + "unit": "%", + "underLabel": "Error Rate (Last 15 minutes)" + }, + "search": { + "query": "index=main sourcetype=access_combined status>=500 | stats count as errors | eval total = [search index=main sourcetype=access_combined | stats count] | eval error_rate=round((errors/total)*100,2)", + "earliest": "-15m", + "latest": "now" + }, + "position": { + "x": 6, + "y": 0, + "width": 3, + "height": 4 + } + }, + { + "id": "panel3", + "title": "Average Response Time", + "description": "Measures the average time taken to process requests.", + "type": "singlevalue", + "options": { + "unit": "ms", + "underLabel": "Average Response Time (Last 15 minutes)" + }, + "search": { + "query": "index=main sourcetype=access_combined | stats avg(response_time) as avg_rt | eval avg_rt=round(avg_rt,2)", + "earliest": "-15m", + "latest": "now" + }, + "position": { + "x": 9, + "y": 0, + "width": 3, + "height": 4 + } + }, + { + "id": "panel4", + "title": "Top 10 Slowest Endpoints", + "description": "Lists the endpoints with the highest average response times.", + "type": "table", + "options": { + "drilldown": "none" + }, + "search": { + "query": "index=main sourcetype=access_combined | stats avg(response_time) as avg_rt by uri | sort -avg_rt | head 10", + "earliest": "-1h", + "latest": "now" + }, + "position": { + "x": 0, + "y": 4, + "width": 6, + "height": 4 + } + }, + { + "id": "panel5", + "title": "Server CPU Utilization", + "description": "Displays the CPU utilization across all servers.", + "type": "timeseries", + "options": { + "xAxisTitle": "Time", + "yAxisTitle": "% CPU Utilization" + }, + "search": { + "query": "index=os sourcetype=cpu | timechart avg(percentIdle) as idle by host span=1m | eval cpu_utilization=100-idle", + "earliest": "-15m", + "latest": "now" + }, + "position": { + "x": 6, + "y": 4, + "width": 6, + "height": 4 + } + }, + { + "id": "panel6", + "title": "Recent Error Logs", + "description": "Shows the most recent error logs.", + "type": "event", + "options": { + "count": 5 + }, + "search": { + "query": "index=main sourcetype=application log_level=ERROR", + "earliest": "-1h", + "latest": "now" + }, + "position": { + "x": 0, + "y": 8, + "width": 12, + "height": 4 + } + } + ] + } +} \ No newline at end of file diff --git a/skills/log-aggregation-setup/assets/example_log_data.json b/skills/log-aggregation-setup/assets/example_log_data.json new file mode 100644 index 0000000..182be94 --- /dev/null +++ b/skills/log-aggregation-setup/assets/example_log_data.json @@ -0,0 +1,107 @@ +[ + { + "_comment": "Example log entry from a web server", + "timestamp": "2024-01-26T10:00:00.000Z", + "log_level": "INFO", + "component": "web_server", + "message": "Request received", + "request_id": "a1b2c3d4e5f6", + "client_ip": "192.168.1.100", + "http_method": "GET", + "http_path": "/api/users", + "http_status_code": 200, + "response_time_ms": 123 + }, + { + "_comment": "Example log entry from a database", + "timestamp": "2024-01-26T10:00:01.000Z", + "log_level": "DEBUG", + "component": "database", + "message": "SQL query executed", + "query": "SELECT * FROM users WHERE id = 1", + "execution_time_ms": 5, + "rows_returned": 1 + }, + { + "_comment": "Example log entry from an application", + "timestamp": "2024-01-26T10:00:02.000Z", + "log_level": "ERROR", + "component": "application", + "message": "Error processing request", + "error_code": 500, + "error_message": "Internal server error", + "request_id": "a1b2c3d4e5f6", + "user_id": 123 + }, + { + "_comment": "Example log entry from a system", + "timestamp": "2024-01-26T10:00:03.000Z", + "log_level": "WARN", + "component": "system", + "message": "Disk space nearing capacity", + "disk_usage_percent": 90, + "disk_path": "/var/log" + }, + { + "_comment": "Example log entry from a security component", + "timestamp": "2024-01-26T10:00:04.000Z", + "log_level": "INFO", + "component": "security", + "message": "Authentication successful", + "user_id": 456, + "username": "testuser", + "client_ip": "192.168.1.200" + }, + { + "_comment": "Example log entry for authentication failure", + "timestamp": "2024-01-26T10:00:05.000Z", + "log_level": "WARN", + "component": "security", + "message": "Authentication failed", + "username": "invaliduser", + "client_ip": "192.168.1.200", + "reason": "Invalid password" + }, + { + "_comment": "Example log entry from a microservice", + "timestamp": "2024-01-26T10:00:06.000Z", + "log_level": "INFO", + "component": "microservice-auth", + "message": "User authenticated", + "user_id": 789, + "username": "validuser", + "service_name": "auth-service" + }, + { + "_comment": "Example log entry with exception details", + "timestamp": "2024-01-26T10:00:07.000Z", + "log_level": "ERROR", + "component": "application", + "message": "Unhandled exception", + "exception_type": "NullPointerException", + "exception_message": "Object reference not set to an instance of an object.", + "stack_trace": "at MyApp.Main.DoSomething() in MyApp.cs:line 20", + "request_id": "g7h8i9j0k1l2" + }, + { + "_comment": "Example log entry with metrics", + "timestamp": "2024-01-26T10:00:08.000Z", + "log_level": "INFO", + "component": "monitoring", + "message": "System metrics", + "cpu_usage_percent": 35, + "memory_usage_percent": 60, + "network_throughput_kbps": 1024 + }, + { + "_comment": "Example log entry with audit information", + "timestamp": "2024-01-26T10:00:09.000Z", + "log_level": "INFO", + "component": "audit", + "message": "User profile updated", + "user_id": 123, + "updated_field": "email", + "old_value": "old@example.com", + "new_value": "new@example.com" + } +] \ No newline at end of file diff --git a/skills/log-aggregation-setup/assets/loki_config_template.yaml b/skills/log-aggregation-setup/assets/loki_config_template.yaml new file mode 100644 index 0000000..d58908c --- /dev/null +++ b/skills/log-aggregation-setup/assets/loki_config_template.yaml @@ -0,0 +1,75 @@ +# Loki Configuration File +# This file configures the Loki log aggregation system. + +auth_enabled: false # Disable authentication for simplicity (REPLACE_ME: Enable authentication in production) + +server: + http_listen_port: 3100 # Port Loki listens on for HTTP requests + grpc_listen_port: 9096 # Port Loki listens on for gRPC requests + +ingester: + lifecycler: + address: 127.0.0.1 # Address of the ingester + ring: + kvstore: + store: inmemory # Use in-memory store for simplicity (REPLACE_ME: Use a persistent store like Consul or etcd in production) + replication_factor: 1 # Number of replicas for log data + wal: + enabled: true # Enable Write-Ahead Log for durability + dir: /tmp/loki/wal # Directory for the Write-Ahead Log (REPLACE_ME: Use a persistent volume in production) + chunk_idle_period: 1h # Time after which an inactive chunk is flushed to storage + chunk_block_size: 262144 # Size of each chunk block (256KB) + chunk_retain_period: 24h # Time after which a chunk is deleted from the ingester + max_transfer_retries: 0 # Maximum number of retries for transferring chunks + +schema_config: + configs: + - from: 2020-10-24 # Start date for this schema + store: boltdb-shipper # Use BoltDB shipper for index storage + object_store: filesystem # Use filesystem for chunk storage + schema: v11 # Schema version + index: + prefix: index_ # Prefix for index keys + period: 24h # Index rotation period + +storage_config: + boltdb_shipper: + active_index_directory: /tmp/loki/index # Directory for the active index (REPLACE_ME: Use a persistent volume in production) + shared_dir: /tmp/loki/chunks # Directory for shared chunks (REPLACE_ME: Use a persistent volume in production) + filesystem: + path: /tmp/loki/chunks # Directory for chunk storage (REPLACE_ME: Use a persistent volume in production) + +limits_config: + enforce_metric_name: false # Disable enforcement of metric names + reject_old_samples: true # Reject samples older than the configured time + reject_old_samples_max_age: 168h # Maximum age of samples (7 days) + max_global_streams_per_user: 0 # 0 means unlimited + max_streams_per_user: 0 # 0 means unlimited + ingestion_rate_mb: 100 # Maximum ingestion rate in MB/s + ingestion_burst_size_mb: 200 # Maximum burst size in MB + max_line_size: 512000 # Maximum line size in bytes (500KB) + max_line_length: 512000 # DEPRECATED: use max_line_size instead + max_query_lookback: 720h # Maximum query lookback (30 days) + split_queries_by_interval: 12h # Split queries by this interval + max_concurrent_queries: 30 # Maximum number of concurrent queries + max_query_series: 1000 # Maximum number of series returned by a query + max_query_parallelism: 16 # Maximum query parallelism + max_query_length: 720h # Maximum query length (30 days) + +compactor: + working_directory: /tmp/loki/compactor # Directory for compactor working files (REPLACE_ME: Use a persistent volume in production) + shared_store: filesystem # Use filesystem for shared storage + compaction_interval: 1h # Interval between compactor runs + retention_enabled: true # Enable retention of old chunks + retention_delete_delay: 24h # Delay before deleting old chunks + retention_max_age: 720h # Maximum age of chunks to retain (30 days) + +ruler: + storage: + type: local + local: + directory: /tmp/loki/rules # Directory to store the rules (REPLACE_ME: Use a persistent volume in production) + rule_path: /tmp/loki/rules # Path where rules are stored (REPLACE_ME: Use a persistent volume in production) + alertmanager_url: "" # URL of the Alertmanager instance (REPLACE_ME: YOUR_ALERTMANAGER_URL) + poll_interval: 30s # Interval to poll for rule changes + enable_api: true # Enable the API for managing rules \ No newline at end of file diff --git a/skills/log-aggregation-setup/assets/splunk_config_template.conf b/skills/log-aggregation-setup/assets/splunk_config_template.conf new file mode 100644 index 0000000..21552db --- /dev/null +++ b/skills/log-aggregation-setup/assets/splunk_config_template.conf @@ -0,0 +1,114 @@ +# Splunk Configuration Template + +# This file provides a template for configuring Splunk to collect and index logs. +# It includes examples for various log sources and configurations. +# Please review and modify this file according to your specific environment and requirements. + +# ============================================================================== +# Global Settings +# ============================================================================== + +[default] +host = # Replace with the actual hostname of the Splunk instance + +# ============================================================================== +# Input Configuration: System Logs (Syslog) +# ============================================================================== + +# Configure a UDP input for receiving syslog messages. +# Ensure your syslog daemon is configured to forward logs to this Splunk instance. + +[udp://514] +connection_host = ip +sourcetype = syslog +index = main # Change if you want to index into a different index +disabled = false + +# ============================================================================== +# Input Configuration: File Monitoring (Tail) +# ============================================================================== + +# Monitor a specific log file. Useful for application logs. +# Adjust the path and sourcetype accordingly. + +[monitor:///var/log//.log] +sourcetype = _log +index = main # Change if you want to index into a different index +disabled = false +# Optional: Multiline event breaking (if needed) +# MUST_BREAK_AFTER = ^\d{4}-\d{2}-\d{2} + +# ============================================================================== +# Input Configuration: Windows Event Logs (Windows) +# ============================================================================== + +# Configure Splunk to collect Windows Event Logs. +# Adjust the event logs to monitor as needed. + +[WinEventLog://Application] +disabled = false +index = wineventlog +sourcetype = WinEventLog:Application +# Optional: Filter events by event code +# evt_resolve_ad_obj = 1 # Resolve AD objects +# whitelist = 4624,4625 # Example: Only collect events with ID 4624 and 4625 + +[WinEventLog://System] +disabled = false +index = wineventlog +sourcetype = WinEventLog:System + +[WinEventLog://Security] +disabled = false +index = wineventlog +sourcetype = WinEventLog:Security +# IMPORTANT: Consider the volume of security logs and storage implications. + +# ============================================================================== +# Input Configuration: Scripted Input (Example: CPU Utilization) +# ============================================================================== + +# Example of a scripted input to collect CPU utilization. +# Requires a script (e.g., cpu_utilization.sh or cpu_utilization.ps1) +# that outputs the CPU utilization in a structured format (e.g., CSV, JSON). + +[script://$SPLUNK_HOME/etc/apps//bin/cpu_utilization.sh] +interval = 60 # Run every 60 seconds +sourcetype = cpu_utilization +index = metrics # Consider a dedicated metrics index +disabled = false + +# ============================================================================== +# Transformations (Optional) +# ============================================================================== + +# Use transformations to modify events before they are indexed. +# Example: Masking sensitive data. + +# [transform-null] +# REGEX = (.*).* +# DEST_KEY = _raw +# FORMAT = $1MASKED + +# ============================================================================== +# Index Configuration (Optional) +# ============================================================================== + +# Configure index-specific settings. + +# [] +# homePath = $SPLUNK_DB//db +# coldPath = $SPLUNK_DB//colddb +# thawedPath = $SPLUNK_DB//thaweddb +# maxDataSize = auto +# frozenTimePeriodInSecs = 90d # 90 days retention + +# ============================================================================== +# Notes +# ============================================================================== + +# * Replace placeholders with actual values. +# * Ensure proper permissions are set for log files and scripts. +# * Test configurations thoroughly before deploying to production. +# * Consider using Splunk's monitoring console for health checks and troubleshooting. +# * Review Splunk documentation for detailed information on configuration options. \ No newline at end of file diff --git a/skills/log-aggregation-setup/references/README.md b/skills/log-aggregation-setup/references/README.md new file mode 100644 index 0000000..cdfbede --- /dev/null +++ b/skills/log-aggregation-setup/references/README.md @@ -0,0 +1,12 @@ +# References + +Bundled resources for log-aggregation-setup skill + +- [ ] elk_best_practices.md: A document detailing best practices for configuring and maintaining an ELK stack. +- [ ] loki_best_practices.md: A document detailing best practices for configuring and maintaining Loki. +- [ ] splunk_best_practices.md: A document detailing best practices for configuring and maintaining Splunk. +- [ ] elk_config_schema.json: JSON schema for validating ELK configuration files. +- [ ] loki_config_schema.json: JSON schema for validating Loki configuration files. +- [ ] splunk_config_schema.json: JSON schema for validating Splunk configuration files. +- [ ] supported_platforms.md: Lists the supported operating systems and infrastructure providers for log aggregation setup. +- [ ] security_considerations.md: Security considerations for setting up log aggregation. diff --git a/skills/log-aggregation-setup/scripts/README.md b/skills/log-aggregation-setup/scripts/README.md new file mode 100644 index 0000000..6368bb7 --- /dev/null +++ b/skills/log-aggregation-setup/scripts/README.md @@ -0,0 +1,12 @@ +# Scripts + +Bundled resources for log-aggregation-setup skill + +- [ ] setup_elk.sh: Automates the deployment and configuration of the ELK stack (Elasticsearch, Logstash, Kibana). +- [ ] setup_loki.sh: Automates the deployment and configuration of Loki. +- [ ] setup_splunk.sh: Automates the deployment and configuration of Splunk. +- [ ] configure_logging.py: A Python script to configure logging in various application frameworks (e.g., Flask, Django). +- [ ] validate_config.py: Validates the generated configuration files for ELK, Loki, and Splunk. +- [ ] teardown_elk.sh: Script to safely remove the ELK stack. +- [ ] teardown_loki.sh: Script to safely remove Loki. +- [ ] teardown_splunk.sh: Script to safely remove Splunk.