commit d36eeec1dcee062faef77b323189c6e684fa33db Author: Zhongwei Li Date: Sat Nov 29 18:23:11 2025 +0800 Initial commit diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..39a772f --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,14 @@ +{ + "name": "concat-glob-tool", + "description": "concat-glob-tool-plugin", + "version": "0.1.0", + "author": { + "name": "Dennis Vriend" + }, + "skills": [ + "./skills" + ], + "commands": [ + "./commands" + ] +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..25a6c64 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# concat-glob-tool + +concat-glob-tool-plugin diff --git a/commands/concat.md b/commands/concat.md new file mode 100644 index 0000000..283f2fa --- /dev/null +++ b/commands/concat.md @@ -0,0 +1,39 @@ +--- +description: Concatenate files matching glob patterns +argument-hint: patterns +--- + +Concatenate files matching PATTERNS to output file with separators. + +## Usage + +```bash +concat-glob-tool PATTERNS... --output-file FILE [OPTIONS] +``` + +## Arguments + +- `PATTERNS...`: Glob patterns (e.g., `*.py`, `src/**/*.md`) +- `-o, --output-file FILE`: Output file path (required) +- `--separator TEXT`: Separator text (default: `---`) +- `-n, --dry-run`: Preview (default, enabled) +- `--no-dry-run`: Execute concatenation +- `-f, --force`: Overwrite existing file +- `-v/-vv/-vvv`: Verbosity (INFO/DEBUG/TRACE) + +## Examples + +```bash +# Preview concatenation (dry-run) +concat-glob-tool '*.py' -o output.txt + +# Execute concatenation +concat-glob-tool '*.py' '*.md' -o output.txt --no-dry-run + +# Custom separator +concat-glob-tool '*.py' -o output.txt --separator '===' --no-dry-run +``` + +## Output + +Files concatenated with format: `\n---\n# FILENAME\n---\n` diff --git a/commands/help.md b/commands/help.md new file mode 100644 index 0000000..2df4557 --- /dev/null +++ b/commands/help.md @@ -0,0 +1,34 @@ +--- +description: Show help and examples +argument-hint: --help +--- + +Display help information with examples. + +## Usage + +```bash +concat-glob-tool --help +concat-glob-tool --version +``` + +## Examples + +```bash +# Show full help +concat-glob-tool --help + +# Show version +concat-glob-tool --version + +# Verbose help with examples +concat-glob-tool --help | less +``` + +## Output + +Comprehensive help with: +- Command syntax +- All options and flags +- Working examples +- Output format description diff --git a/commands/stdin.md b/commands/stdin.md new file mode 100644 index 0000000..a96630b --- /dev/null +++ b/commands/stdin.md @@ -0,0 +1,37 @@ +--- +description: Concatenate files from stdin input +argument-hint: --stdin +--- + +Read file paths from stdin and concatenate them. + +## Usage + +```bash +find ... | concat-glob-tool --stdin --output-file FILE [OPTIONS] +``` + +## Arguments + +- `-s, --stdin`: Read paths from stdin (required) +- `-o, --output-file FILE`: Output file path (required) +- `--separator TEXT`: Separator text (default: `---`) +- `--no-dry-run`: Execute concatenation +- `-f, --force`: Overwrite existing file + +## Examples + +```bash +# From find command +find . -name '*.py' | concat-glob-tool --stdin -o output.txt --no-dry-run + +# From fd command +fd -e py | concat-glob-tool --stdin -o output.txt --no-dry-run + +# Filtered with grep +find . -name '*.py' | grep -v test | concat-glob-tool --stdin -o output.txt --no-dry-run +``` + +## Output + +Files concatenated with separators including filenames. diff --git a/plugin.lock.json b/plugin.lock.json new file mode 100644 index 0000000..d3dd20d --- /dev/null +++ b/plugin.lock.json @@ -0,0 +1,57 @@ +{ + "$schema": "internal://schemas/plugin.lock.v1.json", + "pluginId": "gh:dnvriend/concat-glob-tool:plugins/concat-glob-tool", + "normalized": { + "repo": null, + "ref": "refs/tags/v20251128.0", + "commit": "c58f0ec353802d360ccff821e6910d800a509458", + "treeHash": "ecf6c6f8c6c3ba660bfc7be43403235996bf24b8d4b625f9d07e47673a07079e", + "generatedAt": "2025-11-28T10:16:34.790356Z", + "toolVersion": "publish_plugins.py@0.2.0" + }, + "origin": { + "remote": "git@github.com:zhongweili/42plugin-data.git", + "branch": "master", + "commit": "aa1497ed0949fd50e99e70d6324a29c5b34f9390", + "repoRoot": "/Users/zhongweili/projects/openmind/42plugin-data" + }, + "manifest": { + "name": "concat-glob-tool", + "description": "concat-glob-tool-plugin", + "version": "0.1.0" + }, + "content": { + "files": [ + { + "path": "README.md", + "sha256": "05ce2d28a144fe8e8eb8a84dba06e5bf8d49d3ee532eb39f4c6133fde0993cf6" + }, + { + "path": ".claude-plugin/plugin.json", + "sha256": "7d4b0b56568faf7a03e5bc43489014b2ecb407bae67962218b8fdcc3f5ffd795" + }, + { + "path": "commands/stdin.md", + "sha256": "9ffb3dc65e625be23e6b9043217d0a8e44ee22e5461fe7ed17c1266103023e5c" + }, + { + "path": "commands/help.md", + "sha256": "ae52d284cbb7ffbd42ae91e60fe53e7d3d6a7b65a0c361f757779a3c2fcf4106" + }, + { + "path": "commands/concat.md", + "sha256": "c3515b7e69b40e0d273ba4a4de2b0a159f65e77c9c7039179926c4ebd6e4116f" + }, + { + "path": "skills/concat-glob-tool/SKILL.md", + "sha256": "c9f75834f045e4c84ab09af8a3a597aca9119b478d55775fe7472bbcb4362977" + } + ], + "dirSha256": "ecf6c6f8c6c3ba660bfc7be43403235996bf24b8d4b625f9d07e47673a07079e" + }, + "security": { + "scannedAt": null, + "scannerVersion": null, + "flags": [] + } +} \ No newline at end of file diff --git a/skills/concat-glob-tool/SKILL.md b/skills/concat-glob-tool/SKILL.md new file mode 100644 index 0000000..553ad11 --- /dev/null +++ b/skills/concat-glob-tool/SKILL.md @@ -0,0 +1,494 @@ +--- +name: skill-concat-glob-tool +description: File concatenation with glob patterns guide +--- + +# When to use +- Need to concatenate multiple files matching patterns +- Creating LLM context files from source code +- Merging documentation or configuration files +- Building single-file distributions + +# concat-glob-tool Skill + +## Purpose + +Master file concatenation with glob patterns for efficient file merging, documentation bundling, and LLM context generation. This skill covers CLI usage, library integration, and advanced patterns. + +## When to Use This Skill + +**Use this skill when:** +- Concatenating files matching glob patterns (*.py, src/**/*.md) +- Creating context files for Large Language Models +- Merging multiple source files for documentation +- Building single-file distributions with separators +- Integrating file concatenation into Python scripts + +**Do NOT use this skill for:** +- Single file operations (use `cat` or `cp`) +- Binary file concatenation (use specialized tools) +- Real-time streaming (this is batch processing) + +## CLI Tool: concat-glob-tool + +A production-ready CLI utility that concatenates files matching glob patterns with intelligent separators. Features dry-run mode, stdin support, and both CLI and library modes. + +### Installation + +```bash +# From source +git clone https://github.com/dnvriend/concat-glob-tool.git +cd concat-glob-tool +uv tool install . + +# Verify +concat-glob-tool --version +``` + +### Prerequisites + +- Python 3.14+ +- uv package manager +- Access to file system + +### Quick Start + +```bash +# Preview (dry-run, default) +concat-glob-tool '*.py' -o output.txt + +# Execute +concat-glob-tool '*.py' '*.md' -o output.txt --no-dry-run + +# Stdin mode +find . -name '*.py' | concat-glob-tool --stdin -o output.txt --no-dry-run +``` + +## Progressive Disclosure + +
+📖 Core Commands (Click to expand) + +### concat - Concatenate Files with Glob Patterns + +Concatenate files matching glob patterns to a single output file with intelligent separators. + +**Usage:** +```bash +concat-glob-tool PATTERNS... --output-file FILE [OPTIONS] +``` + +**Arguments:** +- `PATTERNS...`: One or more glob patterns (e.g., `*.py`, `src/**/*.md`) +- `-o, --output-file FILE`: Output file path (required) +- `--separator TEXT`: Separator text between files (default: `---`) +- `-n, --dry-run`: Preview without writing (enabled by default) +- `--no-dry-run`: Actually execute the concatenation +- `-f, --force`: Overwrite existing output file +- `-v, --verbose`: Enable verbose output (repeatable: -v, -vv, -vvv) +- `--version`: Show version +- `--help`: Show help message with examples + +**Examples:** +```bash +# Basic concatenation - preview mode (dry-run) +concat-glob-tool '*.py' -o output.txt + +# Execute concatenation +concat-glob-tool '*.py' -o output.txt --no-dry-run + +# Multiple patterns +concat-glob-tool '*.py' '*.md' '*.txt' -o combined.txt --no-dry-run + +# Recursive patterns +concat-glob-tool 'src/**/*.py' -o all-code.txt --no-dry-run + +# Multiple recursive patterns +concat-glob-tool 'src/**/*.py' 'tests/**/*.py' -o codebase.txt --no-dry-run + +# Custom separator +concat-glob-tool '*.py' -o output.txt --separator '===' --no-dry-run + +# Force overwrite existing file +concat-glob-tool '*.py' -o existing.txt --force --no-dry-run + +# Verbose output +concat-glob-tool '*.py' -o output.txt -vv --no-dry-run + +# LLM context generation +concat-glob-tool 'src/**/*.py' 'tests/**/*.py' '*.md' \ + -o llm-context.txt \ + --separator '---' \ + --no-dry-run +``` + +**Output Format:** +Files are concatenated with this separator format between each file: +``` +--- +# /path/to/file.py +--- +[file contents] +``` + +The separator includes: +- Blank line before separator +- Separator line (configurable, default: `---`) +- Comment line with full file path +- Separator line +- Blank line before file contents + +**Dry-Run Mode (Default):** +By default, the tool runs in preview mode showing: +- Number of files that would be concatenated +- List of files to process +- Separator format +- Reminder to use `--no-dry-run` to execute + +This prevents accidental overwrites and lets you verify operations first. + +--- + +### stdin - Concatenate from Stdin + +Read file paths from stdin and concatenate them. Useful for integration with `find`, `fd`, `grep`, and other tools. + +**Usage:** +```bash +find ... | concat-glob-tool --stdin --output-file FILE [OPTIONS] +``` + +**Arguments:** +- `-s, --stdin`: Read file paths from stdin (one per line) +- `-o, --output-file FILE`: Output file path (required) +- `--separator TEXT`: Separator text (default: `---`) +- `--no-dry-run`: Execute the concatenation +- `-f, --force`: Overwrite existing output file +- `-v, --verbose`: Verbose output + +**Examples:** +```bash +# From find command +find . -name '*.py' -type f | concat-glob-tool --stdin -o output.txt --no-dry-run + +# From fd (faster find alternative) +fd -e py | concat-glob-tool --stdin -o output.txt --no-dry-run + +# Filter with grep +find . -name '*.py' | grep -v test | concat-glob-tool --stdin -o output.txt --no-dry-run + +# Complex pipeline +find . -name '*.py' -type f | \ + grep -v __pycache__ | \ + grep -v .venv | \ + concat-glob-tool --stdin -o filtered.txt --no-dry-run + +# With custom separator +find . -name '*.md' | concat-glob-tool --stdin -o docs.txt --separator '===' --no-dry-run +``` + +**Note:** +- Cannot use both `--stdin` and glob patterns in the same command +- Each line from stdin should be a valid file path +- Invalid paths will cause clear error messages with solutions + +
+ +
+⚙️ Advanced Features (Click to expand) + +### Environment Variable Expansion + +Glob patterns support environment variable expansion and home directory shortcuts. + +**Examples:** +```bash +# Home directory expansion +concat-glob-tool '~/projects/*.py' -o output.txt --no-dry-run + +# Environment variables +concat-glob-tool '$HOME/src/**/*.py' -o output.txt --no-dry-run + +# Mixed +concat-glob-tool '~/.config/*.conf' '$PROJECT_DIR/**/*.md' -o output.txt --no-dry-run +``` + +--- + +### Library Integration + +Use concat-glob-tool as a Python library for programmatic integration. + +**Installation:** +```bash +pip install concat-glob-tool +# or +uv add concat-glob-tool +``` + +**Core API:** +```python +from concat_glob_tool import ( + expand_glob_patterns, + concatenate_files, + format_separator, + ConcatError, + NoMatchesError, + OutputExistsError, +) +from pathlib import Path + +# Expand glob patterns +files = expand_glob_patterns(["*.py", "src/**/*.md"]) +print(f"Found {len(files)} files") + +# Concatenate files +result = concatenate_files( + files=files, + output_file=Path("output.txt"), + separator="---", + force=False, + dry_run=False, +) + +print(f"Concatenated {result['files_count']} files") +print(f"Wrote {result['bytes_written']} bytes to {result['output_file']}") +``` + +**Exception Handling:** +```python +from pathlib import Path +from concat_glob_tool import ( + concatenate_files, + expand_glob_patterns, + NoMatchesError, + OutputExistsError, + ConcatError, +) + +try: + files = expand_glob_patterns(["*.py"]) + result = concatenate_files( + files=files, + output_file=Path("output.txt"), + force=False, + dry_run=False, + ) + print(f"Success: {result}") + +except NoMatchesError as e: + print(f"No files found: {e}") +except OutputExistsError as e: + print(f"Output exists: {e}") +except ConcatError as e: + print(f"Error: {e}") +``` + +**Integration Example:** +```python +#!/usr/bin/env python3 +"""Example: Concatenate Python files with custom logic.""" + +from pathlib import Path +from concat_glob_tool import expand_glob_patterns, concatenate_files + +def main(): + # Find all Python files + files = expand_glob_patterns(["src/**/*.py", "tests/**/*.py"]) + + # Filter files (e.g., exclude __init__.py) + filtered_files = [f for f in files if f.name != "__init__.py"] + + # Concatenate + result = concatenate_files( + files=filtered_files, + output_file=Path("codebase.txt"), + separator="===", + force=True, + dry_run=False, + ) + + print(f"✅ Concatenated {result['files_count']} files") + print(f"📝 Output: {result['output_file']}") + print(f"💾 Size: {result['bytes_written']} bytes") + +if __name__ == "__main__": + main() +``` + +--- + +### Custom Separators + +Customize the separator text while maintaining the format structure. + +**Examples:** +```bash +# Simple separator +concat-glob-tool '*.py' -o output.txt --separator '---' --no-dry-run + +# Equals signs +concat-glob-tool '*.py' -o output.txt --separator '===' --no-dry-run + +# Hash separator +concat-glob-tool '*.py' -o output.txt --separator '###' --no-dry-run + +# Custom text +concat-glob-tool '*.py' -o output.txt --separator 'FILE' --no-dry-run +``` + +**Separator Format:** +The separator structure is always: `\n{separator}\n# {filename}\n{separator}\n` + +Example with `--separator '==='`: +``` +=== +# /path/to/file.py +=== +[file contents] +``` + +
+ +
+🔧 Troubleshooting (Click to expand) + +### Common Issues + +**Issue: No files matched the patterns** +```bash +$ concat-glob-tool '*.nonexistent' -o out.txt --no-dry-run +Error: No files matched the patterns: *.nonexistent + +Solution: Verify glob patterns are correct. Examples: + - '*.py' for Python files in current directory + - '**/*.py' for Python files recursively + - 'src/**/*.{py,md}' for multiple extensions +``` + +**Solution:** +- Check glob pattern syntax +- Verify files exist in specified locations +- Use `--verbose` to see detailed logging +- Test pattern with `ls` first: `ls *.py` + +--- + +**Issue: Output file already exists** +```bash +$ concat-glob-tool '*.py' -o existing.txt --no-dry-run +Error: Output file already exists: existing.txt + +Solution: Use --force to overwrite or choose a different output file. +``` + +**Solution:** +- Use `--force` flag to overwrite: `concat-glob-tool '*.py' -o existing.txt --force --no-dry-run` +- Choose different output filename +- Remove existing file first + +--- + +**Issue: Cannot use both --stdin and glob patterns** +```bash +$ concat-glob-tool '*.py' --stdin -o out.txt +Error: Cannot use both --stdin and glob patterns. + +Solution: Use either --stdin OR provide glob patterns, not both. +``` + +**Solution:** +- Use glob patterns: `concat-glob-tool '*.py' -o out.txt --no-dry-run` +- OR use stdin: `find . -name '*.py' | concat-glob-tool --stdin -o out.txt --no-dry-run` + +--- + +**Issue: Forgot --no-dry-run flag** +```bash +$ concat-glob-tool '*.py' -o output.txt +[DRY-RUN] Would concatenate 10 files to: output.txt +... +To execute, add --no-dry-run flag. +``` + +**Solution:** +- This is expected behavior (dry-run is default) +- Add `--no-dry-run` to actually execute +- Dry-run mode prevents accidental overwrites + +### Getting Help + +```bash +# Full help with examples +concat-glob-tool --help + +# Version information +concat-glob-tool --version + +# Verbose output for debugging +concat-glob-tool '*.py' -o output.txt -vv --no-dry-run +``` + +### Verbose Logging Levels + +- No flag (default): WARNING only +- `-v`: INFO level (operations and progress) +- `-vv`: DEBUG level (detailed information) +- `-vvv`: TRACE level (library internals) + +All logs go to stderr, keeping stdout clean for piping. + +
+ +## Exit Codes + +- `0`: Success +- `1`: Error (NoMatchesError, OutputExistsError, ConcatError, validation errors) + +## Output Formats + +**Separator Format:** +``` +\n---\n# /path/to/filename\n---\n +``` + +Components: +- Blank line before separator +- Separator line (customizable via `--separator`) +- Comment line with full file path +- Separator line +- Blank line before file contents + +**Dry-Run Output:** +- Number of files to concatenate +- List of file paths +- Separator format preview +- Instructions to execute + +**Execution Output:** +``` +Successfully concatenated 10 files to output.txt (52482 bytes) +``` + +## Best Practices + +1. **Always Preview First**: Use default dry-run mode to verify operations before executing with `--no-dry-run` + +2. **Use Descriptive Output Names**: Name output files clearly indicating contents (e.g., `llm-context.txt`, `all-python-code.txt`) + +3. **Leverage Verbose Mode**: Use `-v` or `-vv` when troubleshooting or learning the tool's behavior + +4. **Quote Glob Patterns**: Always quote patterns in shell to prevent premature expansion: `'*.py'` not `*.py` + +5. **Test Patterns with ls**: Verify glob patterns work with `ls` before using in concat-glob-tool + +6. **Use stdin for Complex Filtering**: Combine with `find`, `fd`, `grep` for advanced file selection + +7. **Custom Separators for Context**: Use meaningful separators when creating context files for specific purposes + +## Resources + +- **GitHub**: https://github.com/dnvriend/concat-glob-tool +- **Documentation**: README.md in repository +- **Python Package**: Available via pip/uv +- **License**: MIT