Initial commit
This commit is contained in:
Zhongwei Li
680
references/ast-grep-guide.md
Normal file
680
references/ast-grep-guide.md
Normal file
@@ -0,0 +1,680 @@
|
||||
# ast-grep - Code Structure Search Guide
|
||||
|
||||
## Overview
|
||||
|
||||
`ast-grep` searches code by Abstract Syntax Tree (AST) structure, not text. It understands code semantics and can find patterns that are impossible with regex.
|
||||
|
||||
## Core Concepts
|
||||
|
||||
### Pattern Syntax
|
||||
|
||||
- `$VAR` - Matches a single AST node (identifier, expression, etc.)
|
||||
- `$$$` - Matches zero or more nodes (variadic)
|
||||
- `$$` - Matches a sequence of statements
|
||||
|
||||
### Pattern Matching Philosophy
|
||||
|
||||
ast-grep matches **structure**, not **text**:
|
||||
|
||||
```bash
|
||||
# This finds ALL function calls to 'print', regardless of arguments
|
||||
ast-grep --pattern 'print($$$)'
|
||||
|
||||
# Matches:
|
||||
# print()
|
||||
# print("hello")
|
||||
# print("hello", "world")
|
||||
# print(x, y, z)
|
||||
```
|
||||
|
||||
## Language Support
|
||||
|
||||
ast-grep supports: Python, JavaScript, TypeScript, Rust, Go, Java, C, C++, C#, Ruby, Kotlin, Swift, and more.
|
||||
|
||||
---
|
||||
|
||||
## Python Patterns
|
||||
|
||||
### Function Definitions
|
||||
|
||||
```bash
|
||||
# Any function
|
||||
ast-grep --pattern 'def $FUNC($$$): $$$' -l py
|
||||
|
||||
# Functions with no parameters
|
||||
ast-grep --pattern 'def $FUNC(): $$$' -l py
|
||||
|
||||
# Functions with specific number of parameters
|
||||
ast-grep --pattern 'def $FUNC($A, $B): $$$' -l py
|
||||
|
||||
# Functions with type hints
|
||||
ast-grep --pattern 'def $FUNC($$$) -> $TYPE: $$$' -l py
|
||||
|
||||
# Async functions
|
||||
ast-grep --pattern 'async def $FUNC($$$): $$$' -l py
|
||||
|
||||
# Methods (functions with self)
|
||||
ast-grep --pattern 'def $METHOD(self, $$$): $$$' -l py
|
||||
|
||||
# Class methods
|
||||
ast-grep --pattern '@classmethod' -A 1 -l py
|
||||
|
||||
# Static methods
|
||||
ast-grep --pattern '@staticmethod' -A 1 -l py
|
||||
```
|
||||
|
||||
### Class Definitions
|
||||
|
||||
```bash
|
||||
# Any class
|
||||
ast-grep --pattern 'class $CLASS: $$$' -l py
|
||||
|
||||
# Class with inheritance
|
||||
ast-grep --pattern 'class $CLASS($BASE): $$$' -l py
|
||||
|
||||
# Class with multiple inheritance
|
||||
ast-grep --pattern 'class $CLASS($$$): $$$' -l py
|
||||
|
||||
# Dataclass
|
||||
ast-grep --pattern '@dataclass' -A 1 -l py
|
||||
```
|
||||
|
||||
### Imports
|
||||
|
||||
```bash
|
||||
# Import statements
|
||||
ast-grep --pattern 'import $MODULE' -l py
|
||||
|
||||
# From imports
|
||||
ast-grep --pattern 'from $MODULE import $$$' -l py
|
||||
|
||||
# Specific import
|
||||
ast-grep --pattern 'from $MODULE import $NAME' -l py
|
||||
|
||||
# Relative imports
|
||||
ast-grep --pattern 'from . import $$$' -l py
|
||||
ast-grep --pattern 'from .. import $$$' -l py
|
||||
|
||||
# Import aliases
|
||||
ast-grep --pattern 'import $MODULE as $ALIAS' -l py
|
||||
```
|
||||
|
||||
### Function Calls
|
||||
|
||||
```bash
|
||||
# Any call to a function
|
||||
ast-grep --pattern '$FUNC($$$)' -l py
|
||||
|
||||
# Method calls
|
||||
ast-grep --pattern '$OBJ.$METHOD($$$)' -l py
|
||||
|
||||
# Chained method calls
|
||||
ast-grep --pattern '$OBJ.$M1().$M2($$$)' -l py
|
||||
|
||||
# Specific function calls
|
||||
ast-grep --pattern 'print($$$)' -l py
|
||||
ast-grep --pattern 'open($$$)' -l py
|
||||
```
|
||||
|
||||
### Control Flow
|
||||
|
||||
```bash
|
||||
# If statements
|
||||
ast-grep --pattern 'if $COND: $$$' -l py
|
||||
|
||||
# For loops
|
||||
ast-grep --pattern 'for $VAR in $ITER: $$$' -l py
|
||||
|
||||
# While loops
|
||||
ast-grep --pattern 'while $COND: $$$' -l py
|
||||
|
||||
# Try/except
|
||||
ast-grep --pattern 'try: $$$ except $EXC: $$$' -l py
|
||||
|
||||
# Context managers
|
||||
ast-grep --pattern 'with $EXPR as $VAR: $$$' -l py
|
||||
```
|
||||
|
||||
### Decorators
|
||||
|
||||
```bash
|
||||
# Any decorator
|
||||
ast-grep --pattern '@$DECORATOR' -l py
|
||||
|
||||
# Specific decorator
|
||||
ast-grep --pattern '@property' -l py
|
||||
ast-grep --pattern '@staticmethod' -l py
|
||||
|
||||
# Decorator with arguments
|
||||
ast-grep --pattern '@$DECORATOR($$$)' -l py
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## JavaScript/TypeScript Patterns
|
||||
|
||||
### Function Definitions
|
||||
|
||||
```bash
|
||||
# Function declarations
|
||||
ast-grep --pattern 'function $FUNC($$$) { $$$ }' -l js
|
||||
|
||||
# Arrow functions
|
||||
ast-grep --pattern '($$$) => $BODY' -l js
|
||||
ast-grep --pattern '$VAR = ($$$) => { $$$ }' -l js
|
||||
|
||||
# Async functions
|
||||
ast-grep --pattern 'async function $FUNC($$$) { $$$ }' -l js
|
||||
ast-grep --pattern 'async ($$$) => $BODY' -l js
|
||||
|
||||
# Generator functions
|
||||
ast-grep --pattern 'function* $FUNC($$$) { $$$ }' -l js
|
||||
```
|
||||
|
||||
### Class and Components
|
||||
|
||||
```bash
|
||||
# Class declarations
|
||||
ast-grep --pattern 'class $CLASS { $$$ }' -l js
|
||||
|
||||
# Class with extends
|
||||
ast-grep --pattern 'class $CLASS extends $BASE { $$$ }' -l js
|
||||
|
||||
# React functional components
|
||||
ast-grep --pattern 'function $COMP() { $$$ }' -l tsx
|
||||
ast-grep --pattern 'const $COMP = () => { $$$ }' -l tsx
|
||||
|
||||
# React class components
|
||||
ast-grep --pattern 'class $COMP extends React.Component { $$$ }' -l tsx
|
||||
```
|
||||
|
||||
### Imports/Exports
|
||||
|
||||
```bash
|
||||
# Import statements
|
||||
ast-grep --pattern 'import $$ from "$MODULE"' -l js
|
||||
|
||||
# Named imports
|
||||
ast-grep --pattern 'import { $$ } from "$MODULE"' -l js
|
||||
|
||||
# Default import
|
||||
ast-grep --pattern 'import $DEFAULT from "$MODULE"' -l js
|
||||
|
||||
# Require (CommonJS)
|
||||
ast-grep --pattern 'require("$MODULE")' -l js
|
||||
|
||||
# Export default
|
||||
ast-grep --pattern 'export default $EXPR' -l js
|
||||
|
||||
# Named exports
|
||||
ast-grep --pattern 'export { $$ }' -l js
|
||||
```
|
||||
|
||||
### React Hooks
|
||||
|
||||
```bash
|
||||
# useState
|
||||
ast-grep --pattern 'const [$STATE, $SETTER] = useState($$$)' -l tsx
|
||||
|
||||
# useEffect
|
||||
ast-grep --pattern 'useEffect($$$)' -l tsx
|
||||
|
||||
# useCallback
|
||||
ast-grep --pattern 'useCallback($$$)' -l tsx
|
||||
|
||||
# useMemo
|
||||
ast-grep --pattern 'useMemo($$$)' -l tsx
|
||||
|
||||
# Custom hooks (by convention)
|
||||
ast-grep --pattern 'function use$HOOK($$$) { $$$ }' -l tsx
|
||||
```
|
||||
|
||||
### TypeScript Specifics
|
||||
|
||||
```bash
|
||||
# Type definitions
|
||||
ast-grep --pattern 'type $NAME = $TYPE' -l ts
|
||||
|
||||
# Interface definitions
|
||||
ast-grep --pattern 'interface $NAME { $$$ }' -l ts
|
||||
|
||||
# Type assertions
|
||||
ast-grep --pattern '$EXPR as $TYPE' -l ts
|
||||
|
||||
# Generic functions
|
||||
ast-grep --pattern 'function $FUNC<$$$>($$$) { $$$ }' -l ts
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Rust Patterns
|
||||
|
||||
### Function Definitions
|
||||
|
||||
```bash
|
||||
# Function declarations
|
||||
ast-grep --pattern 'fn $FUNC($$$) { $$$ }' -l rs
|
||||
|
||||
# Functions with return type
|
||||
ast-grep --pattern 'fn $FUNC($$$) -> $TYPE { $$$ }' -l rs
|
||||
|
||||
# Public functions
|
||||
ast-grep --pattern 'pub fn $FUNC($$$) { $$$ }' -l rs
|
||||
|
||||
# Async functions
|
||||
ast-grep --pattern 'async fn $FUNC($$$) { $$$ }' -l rs
|
||||
|
||||
# Unsafe functions
|
||||
ast-grep --pattern 'unsafe fn $FUNC($$$) { $$$ }' -l rs
|
||||
```
|
||||
|
||||
### Structs and Enums
|
||||
|
||||
```bash
|
||||
# Struct definitions
|
||||
ast-grep --pattern 'struct $NAME { $$$ }' -l rs
|
||||
|
||||
# Tuple structs
|
||||
ast-grep --pattern 'struct $NAME($$$);' -l rs
|
||||
|
||||
# Enum definitions
|
||||
ast-grep --pattern 'enum $NAME { $$$ }' -l rs
|
||||
|
||||
# Public structs
|
||||
ast-grep --pattern 'pub struct $NAME { $$$ }' -l rs
|
||||
```
|
||||
|
||||
### Impl Blocks
|
||||
|
||||
```bash
|
||||
# Implementation blocks
|
||||
ast-grep --pattern 'impl $TYPE { $$$ }' -l rs
|
||||
|
||||
# Trait implementations
|
||||
ast-grep --pattern 'impl $TRAIT for $TYPE { $$$ }' -l rs
|
||||
|
||||
# Generic implementations
|
||||
ast-grep --pattern 'impl<$$$> $TYPE { $$$ }' -l rs
|
||||
```
|
||||
|
||||
### Macros
|
||||
|
||||
```bash
|
||||
# Macro usage
|
||||
ast-grep --pattern '$MACRO!($$$)' -l rs
|
||||
|
||||
# Specific macros
|
||||
ast-grep --pattern 'println!($$$)' -l rs
|
||||
ast-grep --pattern 'vec![$$$]' -l rs
|
||||
|
||||
# Macro definitions
|
||||
ast-grep --pattern 'macro_rules! $NAME { $$$ }' -l rs
|
||||
```
|
||||
|
||||
### Error Handling
|
||||
|
||||
```bash
|
||||
# Result returns
|
||||
ast-grep --pattern 'fn $FUNC($$$) -> Result<$$$> { $$$ }' -l rs
|
||||
|
||||
# Unwrap calls
|
||||
ast-grep --pattern '$EXPR.unwrap()' -l rs
|
||||
|
||||
# Question mark operator
|
||||
ast-grep --pattern '$EXPR?' -l rs
|
||||
|
||||
# Match Result
|
||||
ast-grep --pattern 'match $EXPR { Ok($VAL) => $$$, Err($ERR) => $$$ }' -l rs
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Go Patterns
|
||||
|
||||
### Function Definitions
|
||||
|
||||
```bash
|
||||
# Function declarations
|
||||
ast-grep --pattern 'func $FUNC($$$) $$$ { $$$ }' -l go
|
||||
|
||||
# Methods
|
||||
ast-grep --pattern 'func ($RECV $TYPE) $METHOD($$$) $$$ { $$$ }' -l go
|
||||
|
||||
# Functions with multiple returns
|
||||
ast-grep --pattern 'func $FUNC($$$) ($$$, error) { $$$ }' -l go
|
||||
```
|
||||
|
||||
### Struct and Interface
|
||||
|
||||
```bash
|
||||
# Struct definitions
|
||||
ast-grep --pattern 'type $NAME struct { $$$ }' -l go
|
||||
|
||||
# Interface definitions
|
||||
ast-grep --pattern 'type $NAME interface { $$$ }' -l go
|
||||
|
||||
# Type aliases
|
||||
ast-grep --pattern 'type $NAME $TYPE' -l go
|
||||
```
|
||||
|
||||
### Concurrency
|
||||
|
||||
```bash
|
||||
# Goroutine launches
|
||||
ast-grep --pattern 'go $FUNC($$$)' -l go
|
||||
|
||||
# Channel operations
|
||||
ast-grep --pattern '$CHAN <- $VALUE' -l go
|
||||
ast-grep --pattern '$VAR := <-$CHAN' -l go
|
||||
|
||||
# Select statements
|
||||
ast-grep --pattern 'select { $$$ }' -l go
|
||||
```
|
||||
|
||||
### Error Handling
|
||||
|
||||
```bash
|
||||
# If err check
|
||||
ast-grep --pattern 'if err != nil { $$$ }' -l go
|
||||
|
||||
# Error returns
|
||||
ast-grep --pattern 'return $$$, err' -l go
|
||||
|
||||
# Defer statements
|
||||
ast-grep --pattern 'defer $FUNC($$$)' -l go
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Zig Patterns
|
||||
|
||||
### Function Definitions
|
||||
|
||||
```bash
|
||||
# Public functions
|
||||
ast-grep --pattern 'pub fn $FUNC($$$) $$$ { $$$ }' -l zig
|
||||
|
||||
# Private functions
|
||||
ast-grep --pattern 'fn $FUNC($$$) $$$ { $$$ }' -l zig
|
||||
|
||||
# Exported functions
|
||||
ast-grep --pattern 'export fn $FUNC($$$) $$$ { $$$ }' -l zig
|
||||
```
|
||||
|
||||
### Structs and Types
|
||||
|
||||
```bash
|
||||
# Struct definitions
|
||||
ast-grep --pattern 'const $NAME = struct { $$$ };' -l zig
|
||||
|
||||
# Packed structs
|
||||
ast-grep --pattern 'const $NAME = packed struct { $$$ };' -l zig
|
||||
|
||||
# Type definitions
|
||||
ast-grep --pattern 'const $NAME = $TYPE;' -l zig
|
||||
```
|
||||
|
||||
### Error Handling
|
||||
|
||||
```bash
|
||||
# Try expressions
|
||||
ast-grep --pattern 'try $EXPR' -l zig
|
||||
|
||||
# Catch
|
||||
ast-grep --pattern '$EXPR catch $$$' -l zig
|
||||
|
||||
# Error unions
|
||||
ast-grep --pattern 'fn $FUNC($$$) !$$$ { $$$ }' -l zig
|
||||
```
|
||||
|
||||
### Testing
|
||||
|
||||
```bash
|
||||
# Test blocks
|
||||
ast-grep --pattern 'test "$NAME" { $$$ }' -l zig
|
||||
|
||||
# Expect calls
|
||||
ast-grep --pattern 'try testing.expect($$$)' -l zig
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Ruby Patterns
|
||||
|
||||
### Method Definitions
|
||||
|
||||
```bash
|
||||
# Method definitions
|
||||
ast-grep --pattern 'def $METHOD($$$); $$$; end' -l rb
|
||||
|
||||
# Class methods
|
||||
ast-grep --pattern 'def self.$METHOD($$$); $$$; end' -l rb
|
||||
|
||||
# Private methods
|
||||
ast-grep --pattern 'private' -A 1 -l rb
|
||||
```
|
||||
|
||||
### Classes and Modules
|
||||
|
||||
```bash
|
||||
# Class definitions
|
||||
ast-grep --pattern 'class $CLASS; $$$; end' -l rb
|
||||
|
||||
# Class with inheritance
|
||||
ast-grep --pattern 'class $CLASS < $BASE; $$$; end' -l rb
|
||||
|
||||
# Module definitions
|
||||
ast-grep --pattern 'module $MODULE; $$$; end' -l rb
|
||||
```
|
||||
|
||||
### Blocks and Iterators
|
||||
|
||||
```bash
|
||||
# Blocks with do/end
|
||||
ast-grep --pattern '$EXPR do |$PARAM|; $$$; end' -l rb
|
||||
|
||||
# Blocks with braces
|
||||
ast-grep --pattern '$EXPR { |$PARAM| $$$ }' -l rb
|
||||
|
||||
# Each iterator
|
||||
ast-grep --pattern '$COLLECTION.each do |$ITEM|; $$$; end' -l rb
|
||||
|
||||
# Map
|
||||
ast-grep --pattern '$COLLECTION.map { |$ITEM| $$$ }' -l rb
|
||||
```
|
||||
|
||||
### Rails Patterns
|
||||
|
||||
```bash
|
||||
# Route definitions
|
||||
ast-grep --pattern 'get "$PATH", to: "$HANDLER"' -l rb
|
||||
ast-grep --pattern 'post "$PATH", to: "$HANDLER"' -l rb
|
||||
|
||||
# ActiveRecord queries
|
||||
ast-grep --pattern '$MODEL.where($$$)' -l rb
|
||||
ast-grep --pattern '$MODEL.find_by($$$)' -l rb
|
||||
|
||||
# Validations
|
||||
ast-grep --pattern 'validates $$$' -l rb
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Refactoring
|
||||
|
||||
```bash
|
||||
# Rename function
|
||||
ast-grep --pattern 'old_function($$$)' \
|
||||
--rewrite 'new_function($$$)' \
|
||||
-l py --interactive
|
||||
|
||||
# Update API version
|
||||
ast-grep --pattern 'api.v1.$METHOD($$$)' \
|
||||
--rewrite 'api.v2.$METHOD($$$)' \
|
||||
-l py
|
||||
|
||||
# Modernize var to const
|
||||
ast-grep --pattern 'var $VAR = $EXPR' \
|
||||
--rewrite 'const $VAR = $EXPR' \
|
||||
-l js --interactive
|
||||
|
||||
# Add type annotations
|
||||
ast-grep --pattern 'def $FUNC($PARAM):' \
|
||||
--rewrite 'def $FUNC($PARAM: Any):' \
|
||||
-l py
|
||||
```
|
||||
|
||||
### Multi-Pattern Search
|
||||
|
||||
```bash
|
||||
# Search with multiple patterns (OR)
|
||||
ast-grep --pattern 'print($$$)' -l py && \
|
||||
ast-grep --pattern 'logging.$METHOD($$$)' -l py
|
||||
|
||||
# Chain searches (AND)
|
||||
ast-grep --pattern 'def $FUNC($$$):' -l py | \
|
||||
grep -v test | \
|
||||
xargs -I {} ast-grep --pattern 'return $$$' {}
|
||||
```
|
||||
|
||||
### Complex Patterns
|
||||
|
||||
```bash
|
||||
# Find functions with many parameters (code smell)
|
||||
ast-grep --pattern 'def $FUNC($A, $B, $C, $D, $E, $$$):' -l py
|
||||
|
||||
# Find deeply nested function calls
|
||||
ast-grep --pattern '$A($B($C($$$)))' -l py
|
||||
|
||||
# Find unused variables (assigned but not used)
|
||||
ast-grep --pattern '$VAR = $$$' -l py # Then analyze
|
||||
|
||||
# Find mutable default arguments (Python anti-pattern)
|
||||
ast-grep --pattern 'def $FUNC($PARAM=[]):' -l py
|
||||
ast-grep --pattern 'def $FUNC($PARAM={}):' -l py
|
||||
```
|
||||
|
||||
### Debugging Patterns
|
||||
|
||||
```bash
|
||||
# Find console.log statements
|
||||
ast-grep --pattern 'console.log($$$)' -l js
|
||||
|
||||
# Find debugger statements
|
||||
ast-grep --pattern 'debugger' -l js
|
||||
|
||||
# Find print debugging
|
||||
ast-grep --pattern 'print($$$)' -l py
|
||||
|
||||
# Find TODO comments in code (not comments, but in strings)
|
||||
ast-grep --pattern '"TODO: $$$"' -l py
|
||||
```
|
||||
|
||||
## Rule Files
|
||||
|
||||
Create reusable rules in YAML:
|
||||
|
||||
```yaml
|
||||
# rules.yml
|
||||
id: no-mutable-default
|
||||
language: python
|
||||
rule:
|
||||
pattern: 'def $FUNC($$$, $PARAM=[], $$$):'
|
||||
message: Avoid mutable default arguments
|
||||
severity: warning
|
||||
```
|
||||
|
||||
Use with:
|
||||
```bash
|
||||
ast-grep scan --rule rules.yml
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
Create `.ast-grep.yml` in project root:
|
||||
|
||||
```yaml
|
||||
ruleDirs:
|
||||
- rules
|
||||
testDirs:
|
||||
- tests
|
||||
ignore:
|
||||
- node_modules
|
||||
- .venv
|
||||
- dist
|
||||
```
|
||||
|
||||
## Performance Tips
|
||||
|
||||
1. **Use language specification**: `-l py` is more accurate
|
||||
2. **Narrow scope**: Search specific directories
|
||||
3. **Use specific patterns**: More specific = faster
|
||||
4. **Cache results**: Output to file for repeated analysis
|
||||
|
||||
## Interactive Workflows
|
||||
|
||||
```bash
|
||||
# Find and select function to edit
|
||||
ast-grep --pattern 'def $FUNC($$$):' -l py | \
|
||||
fzf --preview 'bat --color=always {1}' | \
|
||||
cut -d: -f1 | \
|
||||
xargs $EDITOR
|
||||
|
||||
# Find usages and refactor
|
||||
ast-grep --pattern 'old_name($$$)' -l py | \
|
||||
fzf -m | \
|
||||
xargs ast-grep --pattern 'old_name($$$)' --rewrite 'new_name($$$)' --interactive
|
||||
```
|
||||
|
||||
## Tips and Tricks
|
||||
|
||||
### Finding Security Issues
|
||||
|
||||
```bash
|
||||
# SQL injection risks
|
||||
ast-grep --pattern 'execute("$$$" + $VAR)' -l py
|
||||
|
||||
# XSS risks
|
||||
ast-grep --pattern 'innerHTML = $$$' -l js
|
||||
|
||||
# Command injection
|
||||
ast-grep --pattern 'os.system($$$)' -l py
|
||||
```
|
||||
|
||||
### Code Quality Analysis
|
||||
|
||||
```bash
|
||||
# Find long functions (structural complexity)
|
||||
ast-grep --pattern 'def $FUNC($$$): $$$' -l py | \
|
||||
awk -F: '{print $1":"$2}' | \
|
||||
xargs -I {} sh -c 'echo {}; sed -n "{}p" | wc -l'
|
||||
|
||||
# Find classes with many methods
|
||||
ast-grep --pattern 'class $CLASS: $$$' -l py
|
||||
|
||||
# Find deeply nested code
|
||||
ast-grep --pattern 'if $A: if $B: if $C: $$$' -l py
|
||||
```
|
||||
|
||||
### Documentation
|
||||
|
||||
```bash
|
||||
# Find undocumented functions
|
||||
ast-grep --pattern 'def $FUNC($$$):' -l py | \
|
||||
xargs rg -L '"""' # Functions without docstrings
|
||||
|
||||
# Find public APIs
|
||||
ast-grep --pattern 'export function $FUNC($$$) { $$$ }' -l ts
|
||||
```
|
||||
|
||||
## Comparison with grep/rg
|
||||
|
||||
| Task | grep/rg | ast-grep |
|
||||
|------|---------|----------|
|
||||
| Find string "print" | `rg print` | N/A |
|
||||
| Find print function calls | `rg 'print\('` (fragile) | `ast-grep --pattern 'print($$$)'` |
|
||||
| Find function definitions | `rg '^def '` (limited) | `ast-grep --pattern 'def $F($$$):'` |
|
||||
| Rename function | Complex sed | `--rewrite` flag |
|
||||
| Find by structure | Not possible | Native |
|
||||
290
references/fd-patterns.md
Normal file
290
references/fd-patterns.md
Normal file
@@ -0,0 +1,290 @@
|
||||
# fd - File Finding Patterns
|
||||
|
||||
## Overview
|
||||
|
||||
`fd` is a fast, user-friendly alternative to `find`. It respects `.gitignore` by default and uses intuitive patterns.
|
||||
|
||||
## Basic Usage
|
||||
|
||||
```bash
|
||||
# Find files/directories by name (case-insensitive by default)
|
||||
fd pattern
|
||||
|
||||
# Case-sensitive search
|
||||
fd -s Pattern
|
||||
|
||||
# Case-insensitive explicitly
|
||||
fd -i pattern
|
||||
```
|
||||
|
||||
## File Type Filtering
|
||||
|
||||
```bash
|
||||
# By extension
|
||||
fd -e py # All .py files
|
||||
fd -e js -e ts # Multiple extensions
|
||||
|
||||
# By type
|
||||
fd -t f # Files only
|
||||
fd -t d # Directories only
|
||||
fd -t l # Symlinks only
|
||||
fd -t x # Executable files only
|
||||
|
||||
# Combine type and extension
|
||||
fd -t f -e py # Python files only
|
||||
```
|
||||
|
||||
## Search Scope
|
||||
|
||||
```bash
|
||||
# Search in specific directory
|
||||
fd pattern /path/to/dir
|
||||
|
||||
# Multiple directories
|
||||
fd pattern dir1/ dir2/
|
||||
|
||||
# Depth control
|
||||
fd -d 1 pattern # Current directory only
|
||||
fd -d 3 pattern # Up to 3 levels deep
|
||||
fd --max-depth 2 # Alternative syntax
|
||||
```
|
||||
|
||||
## Advanced Filtering
|
||||
|
||||
### By Size
|
||||
|
||||
```bash
|
||||
# Files larger than 100MB
|
||||
fd -t f --size +100m
|
||||
|
||||
# Files smaller than 1KB
|
||||
fd -t f --size -1k
|
||||
|
||||
# Files between 10KB and 1MB
|
||||
fd -t f --size +10k --size -1m
|
||||
```
|
||||
|
||||
### By Time
|
||||
|
||||
```bash
|
||||
# Modified in last 7 days
|
||||
fd --changed-within 7d
|
||||
|
||||
# Modified more than 30 days ago
|
||||
fd --changed-before 30d
|
||||
|
||||
# Specific date ranges
|
||||
fd --changed-within 2024-01-01..2024-12-31
|
||||
```
|
||||
|
||||
### Hidden and Ignored Files
|
||||
|
||||
```bash
|
||||
# Include hidden files
|
||||
fd -H pattern
|
||||
|
||||
# Ignore .gitignore rules
|
||||
fd -u pattern
|
||||
|
||||
# Both hidden and ignored
|
||||
fd -Hu pattern
|
||||
|
||||
# Search in .git directories
|
||||
fd -H -u -I pattern
|
||||
```
|
||||
|
||||
## Exclusion Patterns
|
||||
|
||||
```bash
|
||||
# Exclude specific patterns
|
||||
fd -E '*.pyc' -E '__pycache__' pattern
|
||||
|
||||
# Exclude directories
|
||||
fd -E 'node_modules' -E '.git' pattern
|
||||
|
||||
# Multiple exclusions
|
||||
fd -e py -E '*test*' -E '*_pb2.py'
|
||||
```
|
||||
|
||||
## Output Formatting
|
||||
|
||||
```bash
|
||||
# Full paths
|
||||
fd -a pattern
|
||||
|
||||
# Relative paths (default)
|
||||
fd pattern
|
||||
|
||||
# Print absolute paths
|
||||
fd -p pattern
|
||||
|
||||
# Null-separated output (for xargs -0)
|
||||
fd -0 pattern
|
||||
```
|
||||
|
||||
## Pattern Matching
|
||||
|
||||
```bash
|
||||
# Exact name match
|
||||
fd -g 'config.json'
|
||||
|
||||
# Wildcard patterns
|
||||
fd -g '*.test.js'
|
||||
fd -g 'test_*.py'
|
||||
|
||||
# Regex mode (default)
|
||||
fd '^[A-Z].*\.py$'
|
||||
|
||||
# Fixed string (faster)
|
||||
fd -F 'literal.string'
|
||||
```
|
||||
|
||||
## Execution
|
||||
|
||||
```bash
|
||||
# Execute command on each result
|
||||
fd -e py -x python -m py_compile {}
|
||||
|
||||
# Execute with multiple files
|
||||
fd -e txt -X cat
|
||||
|
||||
# Execute with placeholders
|
||||
fd -e md -x echo "File: {}" "Path: {/}" "Dir: {//}"
|
||||
|
||||
# Parallel execution
|
||||
fd -e py -x -j 4 pylint {}
|
||||
```
|
||||
|
||||
## Common Workflows
|
||||
|
||||
### Find and Edit
|
||||
|
||||
```bash
|
||||
# Find and edit config files
|
||||
fd config | fzf | xargs $EDITOR
|
||||
|
||||
# Find recent Python files and edit
|
||||
fd -e py --changed-within 1d | fzf | xargs $EDITOR
|
||||
```
|
||||
|
||||
### Find and Delete
|
||||
|
||||
```bash
|
||||
# Find and remove cache files
|
||||
fd -e pyc -X rm
|
||||
fd -t d __pycache__ -X rm -rf
|
||||
|
||||
# Interactive deletion
|
||||
fd '*.log' -x rm -i {}
|
||||
```
|
||||
|
||||
### Find and Copy
|
||||
|
||||
```bash
|
||||
# Copy all Python files to backup
|
||||
fd -e py -X cp -t backup/
|
||||
|
||||
# Copy with directory structure
|
||||
fd -e py -x cp --parents {} backup/
|
||||
```
|
||||
|
||||
### Statistics and Analysis
|
||||
|
||||
```bash
|
||||
# Count files by type
|
||||
fd -e py | wc -l
|
||||
|
||||
# List largest files
|
||||
fd -t f --size +1m -x ls -lh {} | sort -k5 -hr
|
||||
|
||||
# Find duplicate filenames
|
||||
fd | awk -F/ '{print $NF}' | sort | uniq -d
|
||||
```
|
||||
|
||||
## Performance Tips
|
||||
|
||||
1. **Narrow scope early**: Use `-d` to limit depth, `-t` to filter type
|
||||
2. **Use specific patterns**: More specific patterns are faster
|
||||
3. **Respect .gitignore**: Default behavior is already optimized
|
||||
4. **Use fixed strings**: `-F` is faster than regex when possible
|
||||
|
||||
## Common Use Cases
|
||||
|
||||
### Development
|
||||
|
||||
```bash
|
||||
# Find test files
|
||||
fd test -e py
|
||||
fd -g '*_test.go'
|
||||
fd -g '*.test.ts'
|
||||
|
||||
# Find configuration files
|
||||
fd -e json -e yaml -e toml config
|
||||
|
||||
# Find source files excluding tests
|
||||
fd -e py -E '*test*'
|
||||
|
||||
# Find files modified today
|
||||
fd --changed-within 1d
|
||||
```
|
||||
|
||||
### Documentation
|
||||
|
||||
```bash
|
||||
# Find all markdown files
|
||||
fd -e md
|
||||
|
||||
# Find README files
|
||||
fd -i readme
|
||||
|
||||
# Find documentation directories
|
||||
fd -t d docs
|
||||
```
|
||||
|
||||
### Build Artifacts
|
||||
|
||||
```bash
|
||||
# Find compiled files
|
||||
fd -e o -e so -e dylib
|
||||
|
||||
# Find build directories
|
||||
fd -t d -g 'build' -g 'dist' -g 'target'
|
||||
|
||||
# Find and clean
|
||||
fd -e pyc -E '.venv' -X rm
|
||||
```
|
||||
|
||||
## Comparison with find
|
||||
|
||||
| Task | find | fd |
|
||||
|------|------|-----|
|
||||
| Find by name | `find . -name '*.py'` | `fd -e py` |
|
||||
| Files only | `find . -type f` | `fd -t f` |
|
||||
| Max depth | `find . -maxdepth 2` | `fd -d 2` |
|
||||
| Exclude pattern | `find . ! -path '*/test/*'` | `fd -E '*test*'` |
|
||||
| Execute command | `find . -name '*.py' -exec python {} \;` | `fd -e py -x python {}` |
|
||||
|
||||
## Tips and Tricks
|
||||
|
||||
1. **Combine with other tools**:
|
||||
```bash
|
||||
fd -e py | xargs wc -l # Count lines
|
||||
fd -e md | xargs grip -b # Preview markdown
|
||||
fd -0 | xargs -0 tar czf # Create archive
|
||||
```
|
||||
|
||||
2. **Use shell aliases**:
|
||||
```bash
|
||||
alias fdd='fd -t d' # Directories only
|
||||
alias fdf='fd -t f' # Files only
|
||||
alias fdh='fd -H' # Include hidden
|
||||
```
|
||||
|
||||
3. **Create functions**:
|
||||
```bash
|
||||
# Find and cd into directory
|
||||
fcd() { cd $(fd -t d "$1" | fzf) }
|
||||
|
||||
# Find and open in editor
|
||||
fe() { $EDITOR $(fd "$1" | fzf) }
|
||||
```
|
||||
589
references/fzf-workflows.md
Normal file
589
references/fzf-workflows.md
Normal file
@@ -0,0 +1,589 @@
|
||||
# fzf - Interactive Selection Workflows
|
||||
|
||||
## Overview
|
||||
|
||||
`fzf` is a command-line fuzzy finder that enables interactive selection from lists. It's a powerful tool for building interactive workflows.
|
||||
|
||||
## Basic Usage
|
||||
|
||||
```bash
|
||||
# Pipe any list to fzf
|
||||
ls | fzf
|
||||
|
||||
# Multi-select with Tab
|
||||
ls | fzf -m
|
||||
|
||||
# Use selected result
|
||||
SELECTED=$(ls | fzf)
|
||||
echo "You selected: $SELECTED"
|
||||
```
|
||||
|
||||
## Preview Windows
|
||||
|
||||
### Basic Preview
|
||||
|
||||
```bash
|
||||
# Preview with cat
|
||||
fd | fzf --preview 'cat {}'
|
||||
|
||||
# Preview with bat (syntax highlighting)
|
||||
fd | fzf --preview 'bat --color=always {}'
|
||||
|
||||
# Preview with line numbers
|
||||
fd | fzf --preview 'bat --color=always --line-range :500 {}'
|
||||
```
|
||||
|
||||
### Preview Window Configuration
|
||||
|
||||
```bash
|
||||
# Position and size
|
||||
fzf --preview 'cat {}' --preview-window=right:50%
|
||||
fzf --preview 'cat {}' --preview-window=up:40%
|
||||
fzf --preview 'cat {}' --preview-window=down:30%
|
||||
fzf --preview 'cat {}' --preview-window=left:50%
|
||||
|
||||
# Hidden by default (toggle with ctrl-/)
|
||||
fzf --preview 'cat {}' --preview-window=hidden
|
||||
|
||||
# Wrap text
|
||||
fzf --preview 'cat {}' --preview-window=wrap
|
||||
|
||||
# No wrap (default)
|
||||
fzf --preview 'cat {}' --preview-window=nowrap
|
||||
```
|
||||
|
||||
### Advanced Previews
|
||||
|
||||
```bash
|
||||
# Preview with highlighted search term
|
||||
rg --line-number 'pattern' | \
|
||||
fzf --delimiter : \
|
||||
--preview 'bat --color=always {1} --highlight-line {2}'
|
||||
|
||||
# Preview JSON with jq
|
||||
fd -e json | fzf --preview 'jq --color-output . {}'
|
||||
|
||||
# Preview images (requires chafa/imgcat)
|
||||
fd -e png -e jpg | fzf --preview 'chafa {}'
|
||||
|
||||
# Preview directories
|
||||
fd -t d | fzf --preview 'ls -la {}'
|
||||
|
||||
# Preview with tree
|
||||
fd -t d | fzf --preview 'tree -L 2 {}'
|
||||
|
||||
# Multi-level preview (file type detection)
|
||||
fd | fzf --preview '[[ -f {} ]] && bat --color=always {} || tree -L 1 {}'
|
||||
```
|
||||
|
||||
## Keybindings
|
||||
|
||||
### Default Keybindings
|
||||
|
||||
```
|
||||
ctrl-j / ctrl-n / down : Move down
|
||||
ctrl-k / ctrl-p / up : Move up
|
||||
enter : Select and exit
|
||||
tab : Mark item (multi-select)
|
||||
shift-tab : Unmark item
|
||||
ctrl-c / esc : Exit without selection
|
||||
ctrl-/ : Toggle preview window
|
||||
```
|
||||
|
||||
### Custom Keybindings
|
||||
|
||||
```bash
|
||||
# Copy to clipboard (macOS)
|
||||
fzf --bind 'ctrl-y:execute-silent(echo {} | pbcopy)'
|
||||
|
||||
# Copy to clipboard (Linux)
|
||||
fzf --bind 'ctrl-y:execute-silent(echo {} | xclip -selection clipboard)'
|
||||
|
||||
# Open in editor
|
||||
fzf --bind 'ctrl-e:execute($EDITOR {})'
|
||||
|
||||
# Open in editor and exit
|
||||
fzf --bind 'ctrl-e:execute($EDITOR {})+abort'
|
||||
|
||||
# Delete file with confirmation
|
||||
fzf --bind 'ctrl-d:execute(rm -i {})'
|
||||
|
||||
# Multiple bindings
|
||||
fzf --bind 'ctrl-e:execute($EDITOR {})' \
|
||||
--bind 'ctrl-y:execute-silent(echo {} | pbcopy)' \
|
||||
--bind 'ctrl-d:execute(rm -i {})'
|
||||
```
|
||||
|
||||
### Advanced Keybinding Actions
|
||||
|
||||
```bash
|
||||
# Reload results
|
||||
fzf --bind 'ctrl-r:reload(fd)'
|
||||
|
||||
# Change preview
|
||||
fzf --preview 'bat {}' \
|
||||
--bind 'ctrl-/:change-preview-window(hidden|)'
|
||||
|
||||
# Toggle between preview styles
|
||||
fzf --preview 'bat --color=always {}' \
|
||||
--bind 'ctrl-t:change-preview(tree -L 2 {})'
|
||||
|
||||
# Accept and execute
|
||||
fzf --bind 'enter:execute(echo Selected: {})+accept'
|
||||
```
|
||||
|
||||
## Search Syntax
|
||||
|
||||
### Fuzzy Matching
|
||||
|
||||
```
|
||||
# Basic fuzzy search
|
||||
abc # Matches files containing 'a', 'b', 'c' in order
|
||||
# Examples: abc, aXXbXXc, axbxc
|
||||
|
||||
# Exact match (single quote)
|
||||
'abc # Matches files containing exact string "abc"
|
||||
|
||||
# Prefix match (caret)
|
||||
^abc # Matches files starting with "abc"
|
||||
|
||||
# Suffix match (dollar)
|
||||
abc$ # Matches files ending with "abc"
|
||||
|
||||
# Exact match (both)
|
||||
^abc$ # Matches exactly "abc"
|
||||
```
|
||||
|
||||
### Logical Operators
|
||||
|
||||
```
|
||||
# AND (space)
|
||||
abc def # Contains both "abc" AND "def"
|
||||
|
||||
# OR (pipe)
|
||||
abc|def # Contains "abc" OR "def"
|
||||
|
||||
# NOT (exclamation)
|
||||
abc !def # Contains "abc" but NOT "def"
|
||||
|
||||
# Combination
|
||||
abc def|ghi !jkl
|
||||
# Contains "abc" AND ("def" OR "ghi") but NOT "jkl"
|
||||
```
|
||||
|
||||
### Field Selection
|
||||
|
||||
```bash
|
||||
# Search specific field (delimiter-separated)
|
||||
# -d : Set delimiter
|
||||
# -n : Select field number (1-indexed)
|
||||
# --with-nth : Display specific fields
|
||||
|
||||
# Example: search only filenames in paths
|
||||
fd | fzf -d / --with-nth -1
|
||||
|
||||
# Example: search only line numbers
|
||||
rg --line-number 'pattern' | fzf -d : --with-nth 2
|
||||
```
|
||||
|
||||
## Common Workflows
|
||||
|
||||
### File Navigation
|
||||
|
||||
```bash
|
||||
# Find and edit file
|
||||
fe() {
|
||||
local file
|
||||
file=$(fd --type f | fzf --preview 'bat --color=always {}')
|
||||
[[ -n "$file" ]] && $EDITOR "$file"
|
||||
}
|
||||
|
||||
# Find and cd to directory
|
||||
fcd() {
|
||||
local dir
|
||||
dir=$(fd --type d | fzf --preview 'tree -L 1 {}')
|
||||
[[ -n "$dir" ]] && cd "$dir"
|
||||
}
|
||||
|
||||
# Find file with content preview
|
||||
ff() {
|
||||
local file
|
||||
file=$(rg --files-with-matches "${1:-.}" | \
|
||||
fzf --preview "rg --color=always --context 5 '$1' {}")
|
||||
[[ -n "$file" ]] && $EDITOR "$file"
|
||||
}
|
||||
```
|
||||
|
||||
### Code Search
|
||||
|
||||
```bash
|
||||
# Search content and edit
|
||||
fzf_grep() {
|
||||
local line
|
||||
line=$(rg --line-number "$1" | \
|
||||
fzf --delimiter : \
|
||||
--preview 'bat --color=always {1} --highlight-line {2}' \
|
||||
--preview-window +{2}-/2)
|
||||
|
||||
if [[ -n "$line" ]]; then
|
||||
local file=$(echo "$line" | cut -d: -f1)
|
||||
local lineno=$(echo "$line" | cut -d: -f2)
|
||||
$EDITOR "+$lineno" "$file"
|
||||
fi
|
||||
}
|
||||
|
||||
# Find function definition and edit
|
||||
find_function() {
|
||||
local result
|
||||
result=$(ast-grep --pattern "def $1(\$\$\$):" | \
|
||||
fzf --preview 'bat --color=always {1}')
|
||||
|
||||
if [[ -n "$result" ]]; then
|
||||
local file=$(echo "$result" | cut -d: -f1)
|
||||
$EDITOR "$file"
|
||||
fi
|
||||
}
|
||||
```
|
||||
|
||||
### Git Integration
|
||||
|
||||
```bash
|
||||
# Interactive git add
|
||||
fga() {
|
||||
git status --short | \
|
||||
fzf -m --preview 'git diff --color=always {2}' | \
|
||||
awk '{print $2}' | \
|
||||
xargs git add
|
||||
}
|
||||
|
||||
# Interactive git checkout
|
||||
fgco() {
|
||||
git branch --all | \
|
||||
grep -v HEAD | \
|
||||
sed 's/^[* ]*//' | \
|
||||
sed 's#remotes/origin/##' | \
|
||||
sort -u | \
|
||||
fzf --preview 'git log --oneline --color=always {}' | \
|
||||
xargs git checkout
|
||||
}
|
||||
|
||||
# Interactive git log
|
||||
fgl() {
|
||||
git log --oneline --color=always | \
|
||||
fzf --ansi --preview 'git show --color=always {1}' | \
|
||||
awk '{print $1}' | \
|
||||
xargs git show
|
||||
}
|
||||
|
||||
# Interactive git diff
|
||||
fgd() {
|
||||
git diff --name-only | \
|
||||
fzf --preview 'git diff --color=always {}' | \
|
||||
xargs git diff
|
||||
}
|
||||
```
|
||||
|
||||
### Process Management
|
||||
|
||||
```bash
|
||||
# Interactive kill
|
||||
fkill() {
|
||||
local pid
|
||||
pid=$(ps aux | \
|
||||
sed 1d | \
|
||||
fzf -m --header='Select process to kill' | \
|
||||
awk '{print $2}')
|
||||
|
||||
if [[ -n "$pid" ]]; then
|
||||
echo "$pid" | xargs kill -${1:-9}
|
||||
fi
|
||||
}
|
||||
|
||||
# Interactive lsof (find what's using a port)
|
||||
fport() {
|
||||
local port
|
||||
port=$(lsof -i -P -n | \
|
||||
sed 1d | \
|
||||
fzf --header='Select connection' | \
|
||||
awk '{print $2}')
|
||||
|
||||
[[ -n "$port" ]] && echo "PID: $port"
|
||||
}
|
||||
```
|
||||
|
||||
### Docker Integration
|
||||
|
||||
```bash
|
||||
# Interactive container selection
|
||||
fdocker() {
|
||||
docker ps -a | \
|
||||
sed 1d | \
|
||||
fzf --header='Select container' | \
|
||||
awk '{print $1}'
|
||||
}
|
||||
|
||||
# Interactive container logs
|
||||
fdlogs() {
|
||||
local container
|
||||
container=$(fdocker)
|
||||
[[ -n "$container" ]] && docker logs -f "$container"
|
||||
}
|
||||
|
||||
# Interactive container exec
|
||||
fdexec() {
|
||||
local container
|
||||
container=$(fdocker)
|
||||
[[ -n "$container" ]] && docker exec -it "$container" /bin/bash
|
||||
}
|
||||
```
|
||||
|
||||
### Package Management
|
||||
|
||||
```bash
|
||||
# Interactive npm script runner
|
||||
fnpm() {
|
||||
local script
|
||||
script=$(cat package.json | \
|
||||
jq -r '.scripts | keys[]' | \
|
||||
fzf --preview 'jq -r ".scripts.{}" package.json')
|
||||
|
||||
[[ -n "$script" ]] && npm run "$script"
|
||||
}
|
||||
|
||||
# Interactive pip package info
|
||||
fpip() {
|
||||
pip list | \
|
||||
fzf --preview 'pip show {1}'
|
||||
}
|
||||
```
|
||||
|
||||
### Configuration Files
|
||||
|
||||
```bash
|
||||
# Edit config file
|
||||
fconf() {
|
||||
local file
|
||||
file=$(fd -H -t f \
|
||||
-e conf -e config -e json -e yaml -e yml -e toml -e ini \
|
||||
. "$HOME/.config" | \
|
||||
fzf --preview 'bat --color=always {}')
|
||||
|
||||
[[ -n "$file" ]] && $EDITOR "$file"
|
||||
}
|
||||
|
||||
# Edit dotfile
|
||||
fdot() {
|
||||
local file
|
||||
file=$(fd -H -t f '^\..*' "$HOME" --max-depth 1 | \
|
||||
fzf --preview 'bat --color=always {}')
|
||||
|
||||
[[ -n "$file" ]] && $EDITOR "$file"
|
||||
}
|
||||
```
|
||||
|
||||
## Advanced Configurations
|
||||
|
||||
### Color Schemes
|
||||
|
||||
```bash
|
||||
# Monokai
|
||||
export FZF_DEFAULT_OPTS='
|
||||
--color=fg:#f8f8f2,bg:#272822,hl:#66d9ef
|
||||
--color=fg+:#f8f8f2,bg+:#44475a,hl+:#66d9ef
|
||||
--color=info:#a6e22e,prompt:#f92672,pointer:#f92672
|
||||
--color=marker:#f92672,spinner:#a6e22e,header:#6272a4
|
||||
'
|
||||
|
||||
# Nord
|
||||
export FZF_DEFAULT_OPTS='
|
||||
--color=fg:#e5e9f0,bg:#3b4252,hl:#81a1c1
|
||||
--color=fg+:#e5e9f0,bg+:#434c5e,hl+:#81a1c1
|
||||
--color=info:#eacb8a,prompt:#bf616a,pointer:#b48ead
|
||||
--color=marker:#a3be8b,spinner:#b48ead,header:#a3be8b
|
||||
'
|
||||
|
||||
# Tokyo Night
|
||||
export FZF_DEFAULT_OPTS='
|
||||
--color=fg:#c0caf5,bg:#1a1b26,hl:#bb9af7
|
||||
--color=fg+:#c0caf5,bg+:#292e42,hl+:#7dcfff
|
||||
--color=info:#7aa2f7,prompt:#7dcfff,pointer:#7dcfff
|
||||
--color=marker:#9ece6a,spinner:#9ece6a,header:#9ece6a
|
||||
'
|
||||
```
|
||||
|
||||
### Layout Options
|
||||
|
||||
```bash
|
||||
# Reverse layout (prompt at top)
|
||||
fzf --reverse
|
||||
|
||||
# Full screen
|
||||
fzf --height=100%
|
||||
|
||||
# 40% of screen
|
||||
fzf --height=40%
|
||||
|
||||
# Border
|
||||
fzf --border
|
||||
fzf --border=rounded
|
||||
fzf --border=sharp
|
||||
|
||||
# Inline info
|
||||
fzf --inline-info
|
||||
|
||||
# No info
|
||||
fzf --no-info
|
||||
```
|
||||
|
||||
### Performance Options
|
||||
|
||||
```bash
|
||||
# Limit results
|
||||
fzf --select-1 # Auto-select if only one match
|
||||
fzf --exit-0 # Exit if no match
|
||||
|
||||
# Algorithm
|
||||
fzf --algo=v1 # Faster, less accurate
|
||||
fzf --algo=v2 # Default, balanced
|
||||
|
||||
# History
|
||||
fzf --history=/tmp/fzf-history
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
```bash
|
||||
# Default command
|
||||
export FZF_DEFAULT_COMMAND='fd --type f --hidden --exclude .git'
|
||||
|
||||
# Default options
|
||||
export FZF_DEFAULT_OPTS='
|
||||
--height 40%
|
||||
--layout=reverse
|
||||
--border
|
||||
--inline-info
|
||||
--preview "bat --color=always --line-range :500 {}"
|
||||
--bind ctrl-/:toggle-preview
|
||||
'
|
||||
|
||||
# ctrl-t command (file widget)
|
||||
export FZF_CTRL_T_COMMAND="$FZF_DEFAULT_COMMAND"
|
||||
export FZF_CTRL_T_OPTS="
|
||||
--preview 'bat --color=always --line-range :500 {}'
|
||||
--bind 'ctrl-/:toggle-preview'
|
||||
"
|
||||
|
||||
# alt-c command (directory widget)
|
||||
export FZF_ALT_C_COMMAND='fd --type d --hidden --exclude .git'
|
||||
export FZF_ALT_C_OPTS="
|
||||
--preview 'tree -L 1 {}'
|
||||
"
|
||||
```
|
||||
|
||||
## Shell Integration
|
||||
|
||||
### Bash/Zsh Key Bindings
|
||||
|
||||
```bash
|
||||
# Add to ~/.bashrc or ~/.zshrc
|
||||
source /usr/share/doc/fzf/examples/key-bindings.bash # Bash
|
||||
source /usr/share/doc/fzf/examples/key-bindings.zsh # Zsh
|
||||
|
||||
# Or via package manager
|
||||
eval "$(fzf --bash)" # Bash
|
||||
eval "$(fzf --zsh)" # Zsh
|
||||
```
|
||||
|
||||
Default bindings:
|
||||
|
||||
- `ctrl-t`: Paste selected files
|
||||
- `ctrl-r`: Paste from command history
|
||||
- `alt-c`: cd into selected directory
|
||||
|
||||
### Custom Shell Functions
|
||||
|
||||
```bash
|
||||
# Add to ~/.bashrc or ~/.zshrc
|
||||
|
||||
# fh - repeat history
|
||||
fh() {
|
||||
print -z $( ([ -n "$ZSH_NAME" ] && fc -l 1 || history) | \
|
||||
fzf +s --tac | \
|
||||
sed -E 's/ *[0-9]*\*? *//' | \
|
||||
sed -E 's/\\/\\\\/g')
|
||||
}
|
||||
|
||||
# fbr - checkout git branch
|
||||
fbr() {
|
||||
local branches branch
|
||||
branches=$(git branch -vv) &&
|
||||
branch=$(echo "$branches" | fzf +m) &&
|
||||
git checkout $(echo "$branch" | awk '{print $1}' | sed "s/.* //")
|
||||
}
|
||||
|
||||
# fshow - git commit browser
|
||||
fshow() {
|
||||
git log --graph --color=always \
|
||||
--format="%C(auto)%h%d %s %C(black)%C(bold)%cr" "$@" |
|
||||
fzf --ansi --no-sort --reverse --tiebreak=index --bind=ctrl-s:toggle-sort \
|
||||
--bind "ctrl-m:execute:
|
||||
(grep -o '[a-f0-9]\{7\}' | head -1 |
|
||||
xargs -I % sh -c 'git show --color=always % | less -R') << 'FZF-EOF'
|
||||
{}
|
||||
FZF-EOF"
|
||||
}
|
||||
```
|
||||
|
||||
## Tips and Tricks
|
||||
|
||||
### Performance
|
||||
|
||||
1. Use fd instead of find for FZF_DEFAULT_COMMAND
|
||||
2. Limit preview size with --line-range
|
||||
3. Use --select-1 and --exit-0 for automation
|
||||
4. Cache results for repeated searches
|
||||
|
||||
### User Experience
|
||||
|
||||
1. Always provide preview when possible
|
||||
2. Use semantic keybindings (ctrl-e for edit)
|
||||
3. Provide helpful --header messages
|
||||
4. Use --multi for batch operations
|
||||
5. Toggle preview with ctrl-/ for large files
|
||||
|
||||
### Integration
|
||||
|
||||
```bash
|
||||
# Combine with xargs
|
||||
fd -e py | fzf -m | xargs wc -l
|
||||
|
||||
# Combine with while read
|
||||
fd | fzf -m | while read file; do
|
||||
echo "Processing: $file"
|
||||
# Process file
|
||||
done
|
||||
|
||||
# Combine with command substitution
|
||||
vim $(fzf)
|
||||
cd $(fd -t d | fzf)
|
||||
```
|
||||
|
||||
### Scripting
|
||||
|
||||
```bash
|
||||
# Exit code check
|
||||
if result=$(fd | fzf); then
|
||||
echo "Selected: $result"
|
||||
else
|
||||
echo "No selection or cancelled"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Multi-select results
|
||||
selected=$(fd | fzf -m)
|
||||
if [[ -n "$selected" ]]; then
|
||||
echo "$selected" | while read file; do
|
||||
process "$file"
|
||||
done
|
||||
fi
|
||||
```
|
||||
656
references/jq-cookbook.md
Normal file
656
references/jq-cookbook.md
Normal file
@@ -0,0 +1,656 @@
|
||||
# jq - JSON Processing Cookbook
|
||||
|
||||
## Overview
|
||||
|
||||
`jq` is a lightweight command-line JSON processor. It's like `sed` for JSON data.
|
||||
|
||||
## Basic Queries
|
||||
|
||||
### Identity and Pretty Print
|
||||
|
||||
```bash
|
||||
# Identity (pretty print)
|
||||
jq . file.json
|
||||
|
||||
# Compact output
|
||||
jq -c . file.json
|
||||
|
||||
# Raw output (no quotes for strings)
|
||||
jq -r . file.json
|
||||
|
||||
# Sort keys
|
||||
jq -S . file.json
|
||||
```
|
||||
|
||||
### Accessing Fields
|
||||
|
||||
```bash
|
||||
# Top-level field
|
||||
jq '.name' file.json
|
||||
|
||||
# Nested field
|
||||
jq '.user.email' file.json
|
||||
|
||||
# Deeply nested
|
||||
jq '.data.user.profile.settings.theme' file.json
|
||||
|
||||
# Optional fields (null if missing)
|
||||
jq '.user.address?' file.json
|
||||
```
|
||||
|
||||
### Array Access
|
||||
|
||||
```bash
|
||||
# First element
|
||||
jq '.[0]' file.json
|
||||
|
||||
# Last element
|
||||
jq '.[-1]' file.json
|
||||
|
||||
# Specific index
|
||||
jq '.[2]' file.json
|
||||
|
||||
# Slice (first 3 elements)
|
||||
jq '.[:3]' file.json
|
||||
|
||||
# Slice (elements 2-5)
|
||||
jq '.[2:5]' file.json
|
||||
|
||||
# All elements
|
||||
jq '.[]' file.json
|
||||
```
|
||||
|
||||
## Iterating
|
||||
|
||||
### Array Iteration
|
||||
|
||||
```bash
|
||||
# Extract field from each element
|
||||
jq '.users[].name' file.json
|
||||
|
||||
# Multiple fields
|
||||
jq '.users[] | .name, .email' file.json
|
||||
|
||||
# Nested iteration
|
||||
jq '.departments[].employees[].name' file.json
|
||||
```
|
||||
|
||||
### Object Iteration
|
||||
|
||||
```bash
|
||||
# All values
|
||||
jq '.[]' file.json
|
||||
|
||||
# All keys
|
||||
jq 'keys' file.json
|
||||
|
||||
# Key-value pairs
|
||||
jq 'to_entries' file.json
|
||||
|
||||
# Iterate key-value
|
||||
jq 'to_entries[] | .key + ": " + (.value | tostring)' file.json
|
||||
```
|
||||
|
||||
## Filtering
|
||||
|
||||
### Select
|
||||
|
||||
```bash
|
||||
# Filter by condition
|
||||
jq '.users[] | select(.active == true)' file.json
|
||||
|
||||
# Multiple conditions (AND)
|
||||
jq '.users[] | select(.active == true and .age > 18)' file.json
|
||||
|
||||
# Multiple conditions (OR)
|
||||
jq '.users[] | select(.active == true or .admin == true)' file.json
|
||||
|
||||
# Not
|
||||
jq '.users[] | select(.active != false)' file.json
|
||||
|
||||
# Check field exists
|
||||
jq '.users[] | select(.email != null)' file.json
|
||||
jq '.users[] | select(has("email"))' file.json
|
||||
```
|
||||
|
||||
### Comparison Operators
|
||||
|
||||
```bash
|
||||
# Equality
|
||||
jq '.items[] | select(.status == "active")' file.json
|
||||
|
||||
# Greater/Less than
|
||||
jq '.items[] | select(.price > 100)' file.json
|
||||
jq '.items[] | select(.price <= 50)' file.json
|
||||
|
||||
# String contains
|
||||
jq '.items[] | select(.name | contains("widget"))' file.json
|
||||
|
||||
# String starts with
|
||||
jq '.items[] | select(.name | startswith("A"))' file.json
|
||||
|
||||
# String ends with
|
||||
jq '.items[] | select(.name | endswith(".json"))' file.json
|
||||
|
||||
# Regex match
|
||||
jq '.items[] | select(.email | test("@gmail\\.com$"))' file.json
|
||||
```
|
||||
|
||||
## Transforming
|
||||
|
||||
### Map
|
||||
|
||||
```bash
|
||||
# Extract single field
|
||||
jq '.users | map(.name)' file.json
|
||||
|
||||
# Extract multiple fields
|
||||
jq '.users | map({name, email})' file.json
|
||||
|
||||
# Transform values
|
||||
jq '.prices | map(. * 1.1)' file.json
|
||||
|
||||
# Conditional transformation
|
||||
jq '.users | map(if .active then .name else empty end)' file.json
|
||||
```
|
||||
|
||||
### Reduce
|
||||
|
||||
```bash
|
||||
# Sum
|
||||
jq '[.items[].price] | add' file.json
|
||||
|
||||
# Average
|
||||
jq '[.items[].price] | add / length' file.json
|
||||
|
||||
# Min/Max
|
||||
jq '[.items[].price] | min' file.json
|
||||
jq '[.items[].price] | max' file.json
|
||||
|
||||
# Count
|
||||
jq '.items | length' file.json
|
||||
|
||||
# Custom reduce
|
||||
jq 'reduce .items[] as $item (0; . + $item.price)' file.json
|
||||
```
|
||||
|
||||
### Group By
|
||||
|
||||
```bash
|
||||
# Group by field
|
||||
jq 'group_by(.category)' file.json
|
||||
|
||||
# Group and count
|
||||
jq 'group_by(.category) | map({category: .[0].category, count: length})' file.json
|
||||
|
||||
# Group and aggregate
|
||||
jq 'group_by(.category) | map({
|
||||
category: .[0].category,
|
||||
total: map(.price) | add
|
||||
})' file.json
|
||||
```
|
||||
|
||||
## Sorting
|
||||
|
||||
```bash
|
||||
# Sort array
|
||||
jq 'sort' file.json
|
||||
|
||||
# Sort by field
|
||||
jq 'sort_by(.name)' file.json
|
||||
|
||||
# Reverse sort
|
||||
jq 'sort_by(.name) | reverse' file.json
|
||||
|
||||
# Sort by multiple fields
|
||||
jq 'sort_by(.category, .name)' file.json
|
||||
|
||||
# Sort numbers
|
||||
jq 'sort_by(.price | tonumber)' file.json
|
||||
```
|
||||
|
||||
## Constructing
|
||||
|
||||
### Objects
|
||||
|
||||
```bash
|
||||
# New object with specific fields
|
||||
jq '{name: .name, email: .email}' file.json
|
||||
|
||||
# Rename fields
|
||||
jq '{username: .name, mail: .email}' file.json
|
||||
|
||||
# Computed fields
|
||||
jq '{
|
||||
name: .name,
|
||||
fullName: .firstName + " " + .lastName,
|
||||
discountPrice: .price * 0.9
|
||||
}' file.json
|
||||
|
||||
# Nested objects
|
||||
jq '{
|
||||
user: {
|
||||
name: .name,
|
||||
contact: {email: .email}
|
||||
}
|
||||
}' file.json
|
||||
```
|
||||
|
||||
### Arrays
|
||||
|
||||
```bash
|
||||
# Create array from fields
|
||||
jq '[.name, .email, .age]' file.json
|
||||
|
||||
# Array of objects
|
||||
jq '[.users[] | {name, email}]' file.json
|
||||
|
||||
# Flatten nested arrays
|
||||
jq '.departments[].employees | flatten' file.json
|
||||
|
||||
# Unique elements
|
||||
jq '.tags | unique' file.json
|
||||
|
||||
# Array difference
|
||||
jq '.a - .b' file.json
|
||||
```
|
||||
|
||||
## String Operations
|
||||
|
||||
```bash
|
||||
# Concatenation
|
||||
jq '.firstName + " " + .lastName' file.json
|
||||
|
||||
# Interpolation
|
||||
jq '"Hello, \(.name)!"' file.json
|
||||
|
||||
# String length
|
||||
jq '.name | length' file.json
|
||||
|
||||
# Uppercase/Lowercase
|
||||
jq '.name | ascii_upcase' file.json
|
||||
jq '.name | ascii_downcase' file.json
|
||||
|
||||
# Split string
|
||||
jq '.path | split("/")' file.json
|
||||
|
||||
# Join array
|
||||
jq '.tags | join(", ")' file.json
|
||||
|
||||
# Trim whitespace
|
||||
jq '.text | gsub("^\\s+|\\s+$"; "")' file.json
|
||||
|
||||
# Replace
|
||||
jq '.text | gsub("old"; "new")' file.json
|
||||
```
|
||||
|
||||
## Type Conversions
|
||||
|
||||
```bash
|
||||
# To string
|
||||
jq '.age | tostring' file.json
|
||||
|
||||
# To number
|
||||
jq '.price | tonumber' file.json
|
||||
|
||||
# To array
|
||||
jq '[.]' file.json
|
||||
|
||||
# Type check
|
||||
jq '.value | type' file.json
|
||||
|
||||
# Type test
|
||||
jq 'if .value | type == "string" then "is string" else "not string" end' file.json
|
||||
```
|
||||
|
||||
## Conditionals
|
||||
|
||||
```bash
|
||||
# If-then-else
|
||||
jq 'if .active then "active" else "inactive" end' file.json
|
||||
|
||||
# Multiple conditions
|
||||
jq 'if .score >= 90 then "A"
|
||||
elif .score >= 80 then "B"
|
||||
elif .score >= 70 then "C"
|
||||
else "F"
|
||||
end' file.json
|
||||
|
||||
# Ternary-like
|
||||
jq '.users[] | {name, status: (if .active then "active" else "inactive" end)}' file.json
|
||||
|
||||
# Alternative operator
|
||||
jq '.value // "default"' file.json # Use "default" if .value is null/false
|
||||
```
|
||||
|
||||
## Combining Data
|
||||
|
||||
### Merging
|
||||
|
||||
```bash
|
||||
# Merge objects
|
||||
jq '. + {newField: "value"}' file.json
|
||||
|
||||
# Merge arrays
|
||||
jq '.a + .b' file.json
|
||||
|
||||
# Recursive merge
|
||||
jq '. * {user: {active: true}}' file.json
|
||||
|
||||
# Merge multiple files
|
||||
jq -s '.[0] + .[1]' file1.json file2.json
|
||||
```
|
||||
|
||||
### Joining
|
||||
|
||||
```bash
|
||||
# Array join
|
||||
jq '.array1 + .array2' file.json
|
||||
|
||||
# Unique elements from multiple arrays
|
||||
jq '(.array1 + .array2) | unique' file.json
|
||||
|
||||
# Intersection
|
||||
jq '.array1 - (.array1 - .array2)' file.json
|
||||
```
|
||||
|
||||
## Advanced Patterns
|
||||
|
||||
### Working with Nested Data
|
||||
|
||||
```bash
|
||||
# Recursive descent
|
||||
jq '.. | .name? // empty' file.json
|
||||
|
||||
# Find all values for a key
|
||||
jq '.. | .email? // empty' file.json
|
||||
|
||||
# Flatten deeply nested structure
|
||||
jq '[.. | .users? // empty] | flatten' file.json
|
||||
```
|
||||
|
||||
### Multiple Outputs
|
||||
|
||||
```bash
|
||||
# Multiple queries
|
||||
jq '.name, .email' file.json
|
||||
|
||||
# Conditional multiple outputs
|
||||
jq '.users[] | .name, (if .admin then "ADMIN" else empty end)' file.json
|
||||
```
|
||||
|
||||
### Error Handling
|
||||
|
||||
```bash
|
||||
# Try-catch
|
||||
jq 'try .field catch "not found"' file.json
|
||||
|
||||
# Optional field access
|
||||
jq '.user?.email?' file.json
|
||||
|
||||
# Default values
|
||||
jq '.count // 0' file.json
|
||||
```
|
||||
|
||||
### Custom Functions
|
||||
|
||||
```bash
|
||||
# Define and use function
|
||||
jq 'def double: . * 2; .values[] | double' file.json
|
||||
|
||||
# Function with parameters
|
||||
jq 'def multiply(n): . * n; .values[] | multiply(3)' file.json
|
||||
|
||||
# Recursive function
|
||||
jq 'def factorial: if . <= 1 then 1 else . * ((. - 1) | factorial) end; 5 | factorial'
|
||||
```
|
||||
|
||||
## Practical Examples
|
||||
|
||||
### API Response Processing
|
||||
|
||||
```bash
|
||||
# Extract IDs
|
||||
curl -s 'https://api.example.com/users' | jq '.data[].id'
|
||||
|
||||
# Format for CSV
|
||||
curl -s 'https://api.example.com/users' | \
|
||||
jq -r '.data[] | [.id, .name, .email] | @csv'
|
||||
|
||||
# Extract nested data
|
||||
curl -s 'https://api.example.com/posts' | \
|
||||
jq '.posts[] | {
|
||||
id,
|
||||
title,
|
||||
author: .user.name,
|
||||
comments: .comments | length
|
||||
}'
|
||||
```
|
||||
|
||||
### Configuration Management
|
||||
|
||||
```bash
|
||||
# Update config value
|
||||
jq '.database.host = "localhost"' config.json
|
||||
|
||||
# Add new field
|
||||
jq '. + {newFeature: true}' config.json
|
||||
|
||||
# Remove field
|
||||
jq 'del(.deprecated)' config.json
|
||||
|
||||
# Merge configs
|
||||
jq -s '.[0] * .[1]' base.json override.json > merged.json
|
||||
```
|
||||
|
||||
### Data Analysis
|
||||
|
||||
```bash
|
||||
# Count by category
|
||||
jq 'group_by(.category) | map({category: .[0].category, count: length})' data.json
|
||||
|
||||
# Top 10 by price
|
||||
jq 'sort_by(.price) | reverse | .[:10]' data.json
|
||||
|
||||
# Statistics
|
||||
jq '{
|
||||
total: length,
|
||||
avgPrice: (map(.price) | add / length),
|
||||
maxPrice: (map(.price) | max),
|
||||
minPrice: (map(.price) | min)
|
||||
}' data.json
|
||||
```
|
||||
|
||||
### Testing and Validation
|
||||
|
||||
```bash
|
||||
# Check required fields
|
||||
jq '.users[] | select(has("name") and has("email") | not)' data.json
|
||||
|
||||
# Validate email format
|
||||
jq '.users[] | select(.email | test("^[^@]+@[^@]+$") | not)' data.json
|
||||
|
||||
# Find duplicates
|
||||
jq 'group_by(.id) | map(select(length > 1))' data.json
|
||||
```
|
||||
|
||||
### Log Processing
|
||||
|
||||
```bash
|
||||
# Filter log level
|
||||
jq 'select(.level == "error")' logs.json
|
||||
|
||||
# Extract error messages
|
||||
jq 'select(.level == "error") | .message' logs.json
|
||||
|
||||
# Group errors by type
|
||||
jq -s 'group_by(.error_type) | map({type: .[0].error_type, count: length})' logs.json
|
||||
|
||||
# Time range filter
|
||||
jq 'select(.timestamp >= "2024-01-01" and .timestamp < "2024-02-01")' logs.json
|
||||
```
|
||||
|
||||
## Output Formats
|
||||
|
||||
```bash
|
||||
# Raw output (no quotes)
|
||||
jq -r '.name' file.json
|
||||
|
||||
# Compact output
|
||||
jq -c '.' file.json
|
||||
|
||||
# Tab-separated
|
||||
jq -r '.[] | [.name, .age] | @tsv' file.json
|
||||
|
||||
# CSV
|
||||
jq -r '.[] | [.name, .age, .email] | @csv' file.json
|
||||
|
||||
# URL encoding
|
||||
jq -r '.params | @uri' file.json
|
||||
|
||||
# Base64 encoding
|
||||
jq -r '.data | @base64' file.json
|
||||
|
||||
# HTML encoding
|
||||
jq -r '.content | @html' file.json
|
||||
|
||||
# JSON output (default)
|
||||
jq '.' file.json
|
||||
```
|
||||
|
||||
## Command-Line Options
|
||||
|
||||
```bash
|
||||
# Read from stdin
|
||||
echo '{"name":"test"}' | jq .
|
||||
|
||||
# Multiple files
|
||||
jq . file1.json file2.json
|
||||
|
||||
# Slurp (read all files into array)
|
||||
jq -s . file1.json file2.json
|
||||
|
||||
# Null input (generate data)
|
||||
jq -n '{name: "test", date: now}'
|
||||
|
||||
# Exit with status code
|
||||
jq -e 'select(.status == "ok")' file.json
|
||||
|
||||
# Tab indentation
|
||||
jq --tab . file.json
|
||||
|
||||
# No color
|
||||
jq -M . file.json
|
||||
|
||||
# Color (force)
|
||||
jq -C . file.json
|
||||
```
|
||||
|
||||
## Tips and Tricks
|
||||
|
||||
### Debugging
|
||||
|
||||
```bash
|
||||
# Pretty print
|
||||
jq . file.json | less
|
||||
|
||||
# Inspect structure
|
||||
jq 'paths' file.json
|
||||
|
||||
# Show all keys
|
||||
jq '[paths | select(length == 1)] | unique' file.json
|
||||
|
||||
# Debug filter
|
||||
jq 'debug | .name' file.json
|
||||
```
|
||||
|
||||
### Performance
|
||||
|
||||
```bash
|
||||
# Stream processing (memory efficient)
|
||||
jq --stream . large.json
|
||||
|
||||
# Compact input
|
||||
jq -c . file.json
|
||||
|
||||
# Limit output
|
||||
jq 'limit(10; .items[])' file.json
|
||||
```
|
||||
|
||||
### Combining with Other Tools
|
||||
|
||||
```bash
|
||||
# With curl
|
||||
curl -s 'https://api.github.com/users/github' | jq '.name'
|
||||
|
||||
# With fd/rg
|
||||
fd -e json | xargs jq '.version'
|
||||
|
||||
# With fzf
|
||||
jq -r '.users[].name' users.json | fzf
|
||||
|
||||
# With xargs
|
||||
jq -r '.files[]' manifest.json | xargs cat
|
||||
```
|
||||
|
||||
## Common Patterns
|
||||
|
||||
### Filter-Map-Reduce
|
||||
|
||||
```bash
|
||||
# Classic pattern
|
||||
jq '.items[] | # iterate
|
||||
select(.active) | # filter
|
||||
.price | # map
|
||||
add' # reduce
|
||||
```
|
||||
|
||||
### Pagination
|
||||
|
||||
```bash
|
||||
# Page 1 (items 0-9)
|
||||
jq '.items[0:10]' data.json
|
||||
|
||||
# Page 2 (items 10-19)
|
||||
jq '.items[10:20]' data.json
|
||||
|
||||
# Function for pagination
|
||||
page() {
|
||||
local page=$1
|
||||
local size=${2:-10}
|
||||
local start=$((page * size))
|
||||
local end=$((start + size))
|
||||
jq ".items[$start:$end]" data.json
|
||||
}
|
||||
```
|
||||
|
||||
### Data Migration
|
||||
|
||||
```bash
|
||||
# Old format to new format
|
||||
jq 'map({
|
||||
id: .user_id,
|
||||
profile: {
|
||||
name: .username,
|
||||
email: .user_email
|
||||
}
|
||||
})' old_format.json > new_format.json
|
||||
```
|
||||
|
||||
## Error Messages
|
||||
|
||||
```bash
|
||||
# Common errors and fixes
|
||||
|
||||
# "parse error: Invalid numeric literal"
|
||||
# → Check JSON syntax, trailing commas
|
||||
|
||||
# "Cannot iterate over null"
|
||||
# → Add optional operator: .field[]?
|
||||
|
||||
# "Cannot index string with string"
|
||||
# → Check data types, .field might be string not object
|
||||
|
||||
# "jq: error: syntax error"
|
||||
# → Check quote escaping in shell
|
||||
```
|
||||
443
references/rg-patterns.md
Normal file
443
references/rg-patterns.md
Normal file
@@ -0,0 +1,443 @@
|
||||
# rg - Ripgrep Patterns and Usage
|
||||
|
||||
## Overview
|
||||
|
||||
`ripgrep` (rg) is an extremely fast text search tool that respects `.gitignore` and supports regex patterns.
|
||||
|
||||
## Basic Search
|
||||
|
||||
```bash
|
||||
# Simple search
|
||||
rg 'pattern'
|
||||
|
||||
# Case-insensitive
|
||||
rg -i 'pattern'
|
||||
|
||||
# Case-sensitive (force)
|
||||
rg -s 'Pattern'
|
||||
|
||||
# Smart case (case-insensitive unless uppercase present)
|
||||
rg -S 'pattern'
|
||||
```
|
||||
|
||||
## File Type Filtering
|
||||
|
||||
```bash
|
||||
# Search in specific file types
|
||||
rg -t py 'import' # Python files
|
||||
rg -t rust 'fn main' # Rust files
|
||||
rg -t js 'function' # JavaScript files
|
||||
|
||||
# Multiple types
|
||||
rg -t py -t js 'class'
|
||||
|
||||
# Exclude types
|
||||
rg -T py 'pattern' # Everything except Python
|
||||
|
||||
# List available types
|
||||
rg --type-list
|
||||
```
|
||||
|
||||
## Context Control
|
||||
|
||||
```bash
|
||||
# Lines before match
|
||||
rg -B 2 'pattern'
|
||||
|
||||
# Lines after match
|
||||
rg -A 5 'pattern'
|
||||
|
||||
# Lines before and after
|
||||
rg -C 3 'pattern'
|
||||
|
||||
# Both (different values)
|
||||
rg -B 2 -A 5 'pattern'
|
||||
```
|
||||
|
||||
## Output Formatting
|
||||
|
||||
```bash
|
||||
# Show line numbers (default in terminal)
|
||||
rg -n 'pattern'
|
||||
|
||||
# Hide line numbers
|
||||
rg -N 'pattern'
|
||||
|
||||
# Show column numbers
|
||||
rg --column 'pattern'
|
||||
|
||||
# Show file paths only
|
||||
rg -l 'pattern'
|
||||
|
||||
# Show files without matches
|
||||
rg --files-without-match 'pattern'
|
||||
|
||||
# Count matches per file
|
||||
rg -c 'pattern'
|
||||
|
||||
# Count total matches
|
||||
rg --count-matches 'pattern'
|
||||
```
|
||||
|
||||
## Advanced Search
|
||||
|
||||
### Word Boundaries
|
||||
|
||||
```bash
|
||||
# Match whole words only
|
||||
rg -w 'word'
|
||||
|
||||
# Example: find 'test' but not 'testing'
|
||||
rg -w 'test'
|
||||
|
||||
# Regex word boundaries
|
||||
rg '\bword\b'
|
||||
```
|
||||
|
||||
### Multiline Search
|
||||
|
||||
```bash
|
||||
# Enable multiline mode
|
||||
rg -U 'pattern.*spanning.*lines'
|
||||
|
||||
# Example: find function definitions
|
||||
rg -U 'def \w+\([^)]*\):\s*"""[^"]*"""'
|
||||
|
||||
# Multiline with context
|
||||
rg -U -C 2 'pattern.*multiline'
|
||||
```
|
||||
|
||||
### Fixed Strings
|
||||
|
||||
```bash
|
||||
# Literal string search (no regex, faster)
|
||||
rg -F 'literal.string[with]special.chars'
|
||||
|
||||
# Case-insensitive literal
|
||||
rg -F -i 'Literal String'
|
||||
```
|
||||
|
||||
## File Filtering
|
||||
|
||||
### By Path
|
||||
|
||||
```bash
|
||||
# Search in specific directory
|
||||
rg 'pattern' src/
|
||||
|
||||
# Multiple directories
|
||||
rg 'pattern' src/ tests/
|
||||
|
||||
# Glob patterns
|
||||
rg -g '*.py' 'pattern'
|
||||
rg -g '**/*test*.py' 'pattern'
|
||||
|
||||
# Multiple globs
|
||||
rg -g '*.{js,ts}' 'pattern'
|
||||
|
||||
# Negative globs (exclude)
|
||||
rg -g '!*test*' 'pattern'
|
||||
rg -g '!*.min.js' 'pattern'
|
||||
```
|
||||
|
||||
### Hidden and Ignored Files
|
||||
|
||||
```bash
|
||||
# Search hidden files
|
||||
rg --hidden 'pattern'
|
||||
|
||||
# Ignore .gitignore rules
|
||||
rg --no-ignore 'pattern'
|
||||
|
||||
# Both
|
||||
rg --hidden --no-ignore 'pattern'
|
||||
|
||||
# Search specific ignored directories
|
||||
rg --no-ignore -g 'node_modules/**' 'pattern'
|
||||
```
|
||||
|
||||
## Regex Patterns
|
||||
|
||||
### Basic Regex
|
||||
|
||||
```bash
|
||||
# Any character
|
||||
rg 'a.c' # matches 'abc', 'a2c', etc.
|
||||
|
||||
# Character classes
|
||||
rg '[0-9]+' # one or more digits
|
||||
rg '[a-zA-Z]+' # letters only
|
||||
rg '[^0-9]' # anything except digits
|
||||
|
||||
# Anchors
|
||||
rg '^pattern' # start of line
|
||||
rg 'pattern$' # end of line
|
||||
rg '^pattern$' # entire line matches
|
||||
```
|
||||
|
||||
### Quantifiers
|
||||
|
||||
```bash
|
||||
# Zero or more
|
||||
rg 'ab*c' # ac, abc, abbc, etc.
|
||||
|
||||
# One or more
|
||||
rg 'ab+c' # abc, abbc, etc.
|
||||
|
||||
# Zero or one
|
||||
rg 'ab?c' # ac, abc
|
||||
|
||||
# Exact count
|
||||
rg 'a{3}' # aaa
|
||||
rg 'a{2,4}' # aa, aaa, aaaa
|
||||
rg 'a{2,}' # aa, aaa, aaaa, etc.
|
||||
```
|
||||
|
||||
### Groups and Alternation
|
||||
|
||||
```bash
|
||||
# Groups
|
||||
rg '(ab)+' # ab, abab, ababab, etc.
|
||||
|
||||
# Alternation
|
||||
rg 'cat|dog' # cat or dog
|
||||
rg '(import|from) numpy'
|
||||
|
||||
# Non-capturing groups
|
||||
rg '(?:http|https)://[^\s]+'
|
||||
```
|
||||
|
||||
### Lookahead/Lookbehind
|
||||
|
||||
```bash
|
||||
# Positive lookahead
|
||||
rg 'test(?=_unit)' # 'test' followed by '_unit'
|
||||
|
||||
# Negative lookahead
|
||||
rg 'test(?!_integration)' # 'test' not followed by '_integration'
|
||||
|
||||
# Positive lookbehind
|
||||
rg '(?<=def )\\w+' # word after 'def '
|
||||
|
||||
# Negative lookbehind
|
||||
rg '(?<!@)property' # 'property' not preceded by '@'
|
||||
```
|
||||
|
||||
## Replacement (with sed/perl)
|
||||
|
||||
```bash
|
||||
# Find and replace (dry run)
|
||||
rg -l 'old_pattern' | xargs sed -n 's/old_pattern/new_pattern/gp'
|
||||
|
||||
# Find and replace (in-place)
|
||||
rg -l 'old_pattern' | xargs sed -i '' 's/old_pattern/new_pattern/g'
|
||||
|
||||
# Using perl for complex replacements
|
||||
rg -l 'pattern' | xargs perl -pi -e 's/old/new/g'
|
||||
```
|
||||
|
||||
## JSON Output
|
||||
|
||||
```bash
|
||||
# JSON output for scripting
|
||||
rg --json 'pattern'
|
||||
|
||||
# Parse with jq
|
||||
rg --json 'pattern' | jq -r 'select(.type == "match") | .data.path.text'
|
||||
|
||||
# Get line numbers
|
||||
rg --json 'pattern' | jq -r 'select(.type == "match") | "\(.data.path.text):\(.data.line_number)"'
|
||||
```
|
||||
|
||||
## Statistics and Analysis
|
||||
|
||||
```bash
|
||||
# Show statistics
|
||||
rg --stats 'pattern'
|
||||
|
||||
# Count matches per file
|
||||
rg -c 'pattern' | sort -t: -k2 -nr
|
||||
|
||||
# Find most common matches
|
||||
rg -o 'pattern' | sort | uniq -c | sort -nr
|
||||
|
||||
# Files with most matches
|
||||
rg -c 'TODO' | awk -F: '$2 > 0' | sort -t: -k2 -nr | head
|
||||
```
|
||||
|
||||
## Common Workflows
|
||||
|
||||
### Code Search
|
||||
|
||||
```bash
|
||||
# Find function definitions (Python)
|
||||
rg 'def \w+\(' -t py
|
||||
|
||||
# Find class definitions
|
||||
rg 'class \w+' -t py
|
||||
|
||||
# Find imports
|
||||
rg 'from .* import' -t py
|
||||
|
||||
# Find TODO/FIXME comments
|
||||
rg '(TODO|FIXME|XXX|HACK):'
|
||||
|
||||
# Find console.log statements
|
||||
rg 'console\.(log|error|warn)' -t js
|
||||
```
|
||||
|
||||
### Configuration Search
|
||||
|
||||
```bash
|
||||
# Find API keys (be careful!)
|
||||
rg -i 'api[_-]?key' -g '!*.md'
|
||||
|
||||
# Find environment variables
|
||||
rg 'process\.env\.' -t js
|
||||
rg 'os\.getenv' -t py
|
||||
|
||||
# Find hardcoded IPs
|
||||
rg '\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b'
|
||||
|
||||
# Find URLs
|
||||
rg 'https?://[^\s<>"{}|\\^`\[\]]+'
|
||||
```
|
||||
|
||||
### Dependency Analysis
|
||||
|
||||
```bash
|
||||
# Find all imports of a module
|
||||
rg 'from mymodule import' -t py
|
||||
|
||||
# Find all requires
|
||||
rg "require\(['\"].*['\"]\\)" -t js
|
||||
|
||||
# Find external dependencies
|
||||
rg 'import.*from ["\'](?!\.)[^"\']+' -t py
|
||||
```
|
||||
|
||||
### Testing
|
||||
|
||||
```bash
|
||||
# Find test functions
|
||||
rg 'def test_\w+' -t py
|
||||
rg 'it\(["\']' -t js
|
||||
|
||||
# Find assertions
|
||||
rg 'assert\w*\(' -t py
|
||||
rg 'expect\(' -t js
|
||||
|
||||
# Find skipped tests
|
||||
rg '@skip|@pytest\.mark\.skip' -t py
|
||||
rg '\.skip\(' -t js
|
||||
```
|
||||
|
||||
## Performance Tips
|
||||
|
||||
1. **Use file type filters**: `-t` is faster than glob patterns
|
||||
2. **Use fixed strings when possible**: `-F` bypasses regex engine
|
||||
3. **Limit search scope**: Specify directories explicitly
|
||||
4. **Use word boundaries**: `-w` is faster than `\b` regex
|
||||
5. **Respect .gitignore**: Default behavior is already optimized
|
||||
|
||||
## Interactive Workflows
|
||||
|
||||
```bash
|
||||
# Search and edit
|
||||
rg -l 'TODO' | fzf | xargs $EDITOR
|
||||
|
||||
# Search with preview
|
||||
rg --line-number 'pattern' | fzf --delimiter : --preview 'bat --color=always {1} --highlight-line {2}'
|
||||
|
||||
# Multi-select and edit
|
||||
rg -l 'pattern' | fzf -m | xargs $EDITOR
|
||||
|
||||
# Search, select, and replace
|
||||
FILE=$(rg -l 'pattern' | fzf)
|
||||
rg 'pattern' "$FILE" | fzf
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
Create `~/.ripgreprc` for default options:
|
||||
|
||||
```bash
|
||||
# Always show line numbers
|
||||
--line-number
|
||||
|
||||
# Smart case search
|
||||
--smart-case
|
||||
|
||||
# Follow symlinks
|
||||
--follow
|
||||
|
||||
# Custom type definitions
|
||||
--type-add=web:*.{html,css,js}*
|
||||
```
|
||||
|
||||
Use with: `export RIPGREP_CONFIG_PATH=~/.ripgreprc`
|
||||
|
||||
## Comparison with grep
|
||||
|
||||
| Task | grep | rg |
|
||||
|------|------|-----|
|
||||
| Case-insensitive | `grep -i` | `rg -i` |
|
||||
| Recursive | `grep -r` | `rg` (default) |
|
||||
| Show line numbers | `grep -n` | `rg -n` (default) |
|
||||
| Context | `grep -C 3` | `rg -C 3` |
|
||||
| File type | `grep --include='*.py'` | `rg -t py` |
|
||||
| Exclude pattern | `grep --exclude='test*'` | `rg -g '!test*'` |
|
||||
|
||||
## Tips and Tricks
|
||||
|
||||
### Shell Aliases
|
||||
|
||||
```bash
|
||||
alias rgi='rg -i' # Case-insensitive
|
||||
alias rga='rg --hidden --no-ignore' # Search everything
|
||||
alias rgl='rg -l' # Files only
|
||||
alias rgc='rg -c' # Count per file
|
||||
```
|
||||
|
||||
### Functions
|
||||
|
||||
```bash
|
||||
# Search and edit function
|
||||
rge() {
|
||||
rg -l "$1" | fzf --preview "rg --color=always -C 5 '$1' {}" | xargs $EDITOR
|
||||
}
|
||||
|
||||
# Search in specific file type
|
||||
rgt() {
|
||||
TYPE=$1
|
||||
shift
|
||||
rg -t "$TYPE" "$@"
|
||||
}
|
||||
|
||||
# Search with context and open
|
||||
rgf() {
|
||||
rg --line-number "$1" | fzf --delimiter : --preview 'bat --color=always {1} --highlight-line {2}' | awk -F: '{print $1}' | xargs $EDITOR
|
||||
}
|
||||
```
|
||||
|
||||
### Complex Patterns
|
||||
|
||||
```bash
|
||||
# Email addresses
|
||||
rg '\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b'
|
||||
|
||||
# Phone numbers (US)
|
||||
rg '\b\d{3}[-.]?\d{3}[-.]?\d{4}\b'
|
||||
|
||||
# Hex colors
|
||||
rg '#[0-9a-fA-F]{6}\b'
|
||||
|
||||
# Semantic versions
|
||||
rg '\b\d+\.\d+\.\d+\b'
|
||||
|
||||
# Python function with type hints
|
||||
rg 'def \w+\([^)]*\) -> \w+:'
|
||||
|
||||
# SQL queries in code
|
||||
rg -U 'SELECT.*FROM.*WHERE'
|
||||
```
|
||||
677
references/yq-examples.md
Normal file
677
references/yq-examples.md
Normal file
@@ -0,0 +1,677 @@
|
||||
# yq - YAML/XML Processing Examples
|
||||
|
||||
## Overview
|
||||
|
||||
`yq` is a lightweight command-line YAML/XML processor. It's like `jq` but for YAML and XML.
|
||||
|
||||
## YAML Processing
|
||||
|
||||
### Basic Queries
|
||||
|
||||
```bash
|
||||
# Pretty print
|
||||
yq . file.yaml
|
||||
|
||||
# Access field
|
||||
yq '.name' file.yaml
|
||||
|
||||
# Nested access
|
||||
yq '.database.host' file.yaml
|
||||
|
||||
# Array access
|
||||
yq '.items[0]' file.yaml
|
||||
|
||||
# All array elements
|
||||
yq '.items[]' file.yaml
|
||||
```
|
||||
|
||||
### Filtering
|
||||
|
||||
```bash
|
||||
# Filter by condition
|
||||
yq '.users[] | select(.active == true)' file.yaml
|
||||
|
||||
# Multiple conditions
|
||||
yq '.users[] | select(.age > 18 and .active == true)' file.yaml
|
||||
|
||||
# String contains
|
||||
yq '.items[] | select(.name | contains("widget"))' file.yaml
|
||||
```
|
||||
|
||||
### Transforming
|
||||
|
||||
```bash
|
||||
# Extract specific fields
|
||||
yq '.users[] | {name, email}' file.yaml
|
||||
|
||||
# Map array
|
||||
yq '.items | map(.price * 1.1)' file.yaml
|
||||
|
||||
# Sort
|
||||
yq 'sort_by(.name)' file.yaml
|
||||
|
||||
# Group by
|
||||
yq 'group_by(.category)' file.yaml
|
||||
```
|
||||
|
||||
### Updating Values
|
||||
|
||||
```bash
|
||||
# Update single field
|
||||
yq '.database.host = "localhost"' file.yaml
|
||||
|
||||
# Update nested field
|
||||
yq '.server.config.port = 8080' file.yaml
|
||||
|
||||
# Update array element
|
||||
yq '.items[0].price = 99.99' file.yaml
|
||||
|
||||
# Add new field
|
||||
yq '. + {"newField": "value"}' file.yaml
|
||||
|
||||
# Delete field
|
||||
yq 'del(.deprecated)' file.yaml
|
||||
```
|
||||
|
||||
### In-Place Editing
|
||||
|
||||
```bash
|
||||
# Edit file in-place
|
||||
yq -i '.version = "2.0"' file.yaml
|
||||
|
||||
# Multiple updates
|
||||
yq -i '
|
||||
.version = "2.0" |
|
||||
.updated = now |
|
||||
.deprecated = null
|
||||
' file.yaml
|
||||
```
|
||||
|
||||
### Merging Files
|
||||
|
||||
```bash
|
||||
# Merge two YAML files
|
||||
yq '. *= load("other.yaml")' base.yaml
|
||||
|
||||
# Merge with override
|
||||
yq 'load("base.yaml") * load("override.yaml")'
|
||||
|
||||
# Merge multiple files
|
||||
yq eval-all '. as $item ireduce ({}; . * $item)' file1.yaml file2.yaml file3.yaml
|
||||
```
|
||||
|
||||
## Format Conversion
|
||||
|
||||
### YAML to JSON
|
||||
|
||||
```bash
|
||||
# Convert YAML to JSON
|
||||
yq -o json . file.yaml
|
||||
|
||||
# Compact JSON
|
||||
yq -o json -I 0 . file.yaml
|
||||
|
||||
# Pretty JSON
|
||||
yq -o json . file.yaml | jq .
|
||||
```
|
||||
|
||||
### JSON to YAML
|
||||
|
||||
```bash
|
||||
# Convert JSON to YAML
|
||||
yq -P . file.json
|
||||
|
||||
# From stdin
|
||||
echo '{"name":"test"}' | yq -P .
|
||||
|
||||
# Pipeline
|
||||
cat file.json | yq -P . > file.yaml
|
||||
```
|
||||
|
||||
### YAML to TOML
|
||||
|
||||
```bash
|
||||
# Convert YAML to TOML
|
||||
yq -o toml . file.yaml
|
||||
```
|
||||
|
||||
### YAML to XML
|
||||
|
||||
```bash
|
||||
# Convert YAML to XML
|
||||
yq -o xml . file.yaml
|
||||
```
|
||||
|
||||
## XML Processing
|
||||
|
||||
### Reading XML
|
||||
|
||||
```bash
|
||||
# Parse XML
|
||||
yq -p xml . file.xml
|
||||
|
||||
# Access element
|
||||
yq -p xml '.root.element' file.xml
|
||||
|
||||
# Access attribute
|
||||
yq -p xml '.root.+@attr' file.xml
|
||||
|
||||
# All elements
|
||||
yq -p xml '.root.items[]' file.xml
|
||||
```
|
||||
|
||||
### Filtering XML
|
||||
|
||||
```bash
|
||||
# Filter by attribute
|
||||
yq -p xml '.root.item[] | select(.+@id == "123")' file.xml
|
||||
|
||||
# Filter by element value
|
||||
yq -p xml '.root.item[] | select(.name == "test")' file.xml
|
||||
```
|
||||
|
||||
### XML to JSON
|
||||
|
||||
```bash
|
||||
# Convert XML to JSON
|
||||
yq -p xml -o json . file.xml
|
||||
|
||||
# Extract and convert
|
||||
yq -p xml -o json '.root.items' file.xml
|
||||
```
|
||||
|
||||
### XML to YAML
|
||||
|
||||
```bash
|
||||
# Convert XML to YAML
|
||||
yq -p xml . file.xml
|
||||
|
||||
# Pretty print
|
||||
yq -p xml -P . file.xml
|
||||
```
|
||||
|
||||
## Common Use Cases
|
||||
|
||||
### Docker Compose
|
||||
|
||||
```bash
|
||||
# Get service names
|
||||
yq '.services | keys' docker-compose.yml
|
||||
|
||||
# Get image for service
|
||||
yq '.services.web.image' docker-compose.yml
|
||||
|
||||
# Update service port
|
||||
yq -i '.services.web.ports[0] = "8080:80"' docker-compose.yml
|
||||
|
||||
# Add environment variable
|
||||
yq -i '.services.web.environment.DEBUG = "true"' docker-compose.yml
|
||||
|
||||
# Get all exposed ports
|
||||
yq '.services[].ports[]' docker-compose.yml
|
||||
```
|
||||
|
||||
### Kubernetes
|
||||
|
||||
```bash
|
||||
# Get pod name
|
||||
yq '.metadata.name' pod.yaml
|
||||
|
||||
# Get container image
|
||||
yq '.spec.containers[0].image' pod.yaml
|
||||
|
||||
# Update replicas
|
||||
yq -i '.spec.replicas = 3' deployment.yaml
|
||||
|
||||
# Get all container names
|
||||
yq '.spec.template.spec.containers[].name' deployment.yaml
|
||||
|
||||
# Get resource limits
|
||||
yq '.spec.template.spec.containers[0].resources.limits' deployment.yaml
|
||||
```
|
||||
|
||||
### CI/CD (GitHub Actions, GitLab CI)
|
||||
|
||||
```bash
|
||||
# Get job names
|
||||
yq '.jobs | keys' .github/workflows/ci.yml
|
||||
|
||||
# Get steps for job
|
||||
yq '.jobs.build.steps[].name' .github/workflows/ci.yml
|
||||
|
||||
# Update trigger
|
||||
yq -i '.on.push.branches = ["main", "develop"]' .github/workflows/ci.yml
|
||||
|
||||
# Get all used actions
|
||||
yq '.jobs[].steps[].uses' .github/workflows/ci.yml
|
||||
```
|
||||
|
||||
### Configuration Files
|
||||
|
||||
```bash
|
||||
# Application config
|
||||
yq '.app.port' config.yaml
|
||||
yq '.database.connection.host' config.yaml
|
||||
|
||||
# Update config
|
||||
yq -i '.app.debug = true' config.yaml
|
||||
yq -i '.cache.enabled = false' config.yaml
|
||||
|
||||
# Merge configs
|
||||
yq '. *= load("config.local.yaml")' config.yaml
|
||||
```
|
||||
|
||||
### Ansible
|
||||
|
||||
```bash
|
||||
# Get hosts
|
||||
yq '.all.hosts | keys' inventory.yaml
|
||||
|
||||
# Get variables
|
||||
yq '.all.vars' inventory.yaml
|
||||
|
||||
# Update host variable
|
||||
yq -i '.all.hosts.server1.ansible_host = "192.168.1.10"' inventory.yaml
|
||||
|
||||
# Get all playbook tasks
|
||||
yq '.[] | select(.tasks) | .tasks[].name' playbook.yaml
|
||||
```
|
||||
|
||||
## Advanced Patterns
|
||||
|
||||
### Conditional Updates
|
||||
|
||||
```bash
|
||||
# Update if exists
|
||||
yq '(select(.field) | .field) = "new-value"' file.yaml
|
||||
|
||||
# Update based on condition
|
||||
yq '(.items[] | select(.price > 100) | .discount) = 0.2' file.yaml
|
||||
|
||||
# Add field if missing
|
||||
yq '. + (if has("field") then {} else {"field": "default"} end)' file.yaml
|
||||
```
|
||||
|
||||
### Complex Transformations
|
||||
|
||||
```bash
|
||||
# Restructure data
|
||||
yq '.users | map({
|
||||
username: .name,
|
||||
contact: {
|
||||
email: .email,
|
||||
phone: .phone
|
||||
}
|
||||
})' file.yaml
|
||||
|
||||
# Aggregate data
|
||||
yq '.items | group_by(.category) | map({
|
||||
category: .[0].category,
|
||||
total: map(.price) | add
|
||||
})' file.yaml
|
||||
|
||||
# Pivot data
|
||||
yq 'reduce .items[] as $item ({}; .[$item.name] = $item.value)' file.yaml
|
||||
```
|
||||
|
||||
### Working with Arrays
|
||||
|
||||
```bash
|
||||
# Append to array
|
||||
yq '.items += ["new-item"]' file.yaml
|
||||
|
||||
# Prepend to array
|
||||
yq '.items = ["new-item"] + .items' file.yaml
|
||||
|
||||
# Remove from array
|
||||
yq 'del(.items[2])' file.yaml
|
||||
|
||||
# Filter array
|
||||
yq '.items = [.items[] | select(.active == true)]' file.yaml
|
||||
|
||||
# Unique array
|
||||
yq '.items | unique' file.yaml
|
||||
|
||||
# Flatten array
|
||||
yq '.items | flatten' file.yaml
|
||||
```
|
||||
|
||||
### Multi-Document YAML
|
||||
|
||||
```bash
|
||||
# Process all documents
|
||||
yq '.' multi-doc.yaml
|
||||
|
||||
# Select specific document
|
||||
yq 'select(document_index == 0)' multi-doc.yaml
|
||||
|
||||
# Filter documents
|
||||
yq 'select(.kind == "Deployment")' multi-doc.yaml
|
||||
|
||||
# Update all documents
|
||||
yq -i '.metadata.labels.app = "myapp"' multi-doc.yaml
|
||||
```
|
||||
|
||||
## String Operations
|
||||
|
||||
```bash
|
||||
# Concatenation
|
||||
yq '.fullName = .firstName + " " + .lastName' file.yaml
|
||||
|
||||
# Interpolation
|
||||
yq '.message = "Hello, " + .name + "!"' file.yaml
|
||||
|
||||
# Uppercase
|
||||
yq '.name | upcase' file.yaml
|
||||
|
||||
# Lowercase
|
||||
yq '.name | downcase' file.yaml
|
||||
|
||||
# Split
|
||||
yq '.path | split("/")' file.yaml
|
||||
|
||||
# Join
|
||||
yq '.items | join(", ")' file.yaml
|
||||
|
||||
# Replace
|
||||
yq '.text | sub("old", "new")' file.yaml
|
||||
|
||||
# Trim
|
||||
yq '.text | trim' file.yaml
|
||||
```
|
||||
|
||||
## Type Checking and Conversion
|
||||
|
||||
```bash
|
||||
# Check type
|
||||
yq '.field | type' file.yaml
|
||||
|
||||
# Convert to string
|
||||
yq '.number | tostring' file.yaml
|
||||
|
||||
# Convert to number
|
||||
yq '.string | tonumber' file.yaml
|
||||
|
||||
# Check if exists
|
||||
yq 'has("field")' file.yaml
|
||||
|
||||
# Length
|
||||
yq '.items | length' file.yaml
|
||||
```
|
||||
|
||||
## Comments
|
||||
|
||||
```bash
|
||||
# Preserve comments (default behavior)
|
||||
yq '.field = "new-value"' file.yaml
|
||||
|
||||
# Add headComment
|
||||
yq '.field headComment="This is a comment"' file.yaml
|
||||
|
||||
# Add lineComment
|
||||
yq '.field lineComment="inline comment"' file.yaml
|
||||
|
||||
# Add footComment
|
||||
yq '.field footComment="bottom comment"' file.yaml
|
||||
```
|
||||
|
||||
## Anchors and Aliases
|
||||
|
||||
```bash
|
||||
# Create anchor
|
||||
yq '.base.&anchor = {"key": "value"}' file.yaml
|
||||
|
||||
# Reference anchor
|
||||
yq '.other = .base.*anchor' file.yaml
|
||||
|
||||
# Merge anchors
|
||||
yq '.combined = .base.*anchor * .override' file.yaml
|
||||
```
|
||||
|
||||
## Validation
|
||||
|
||||
```bash
|
||||
# Check if valid YAML
|
||||
yq . file.yaml > /dev/null && echo "Valid" || echo "Invalid"
|
||||
|
||||
# Validate schema (requires schema file)
|
||||
yq 'load("schema.yaml") | . as $schema | load("data.yaml") | validate($schema)'
|
||||
|
||||
# Check required fields
|
||||
yq 'select(has("requiredField") | not)' file.yaml
|
||||
|
||||
# Validate format
|
||||
yq 'select(.email | test("^[^@]+@[^@]+$") | not)' file.yaml
|
||||
```
|
||||
|
||||
## Output Formatting
|
||||
|
||||
```bash
|
||||
# Default YAML output
|
||||
yq . file.yaml
|
||||
|
||||
# Compact (minimal whitespace)
|
||||
yq -I 0 . file.yaml
|
||||
|
||||
# Custom indentation (4 spaces)
|
||||
yq -I 4 . file.yaml
|
||||
|
||||
# No colors
|
||||
yq -C 0 . file.yaml
|
||||
|
||||
# Force colors
|
||||
yq -C 1 . file.yaml
|
||||
|
||||
# Raw output (no quotes)
|
||||
yq -r '.name' file.yaml
|
||||
|
||||
# One line per document
|
||||
yq -o json . multi-doc.yaml
|
||||
```
|
||||
|
||||
## Scripting Examples
|
||||
|
||||
### Environment-Specific Configs
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
ENV=${1:-dev}
|
||||
|
||||
yq ". *= load(\"config.$ENV.yaml\")" config.base.yaml > config.yaml
|
||||
```
|
||||
|
||||
### Bulk Updates
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
for file in *.yaml; do
|
||||
yq -i '.metadata.labels.version = "2.0"' "$file"
|
||||
done
|
||||
```
|
||||
|
||||
### Config Validation
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
REQUIRED_FIELDS=("name" "version" "description")
|
||||
|
||||
for field in "${REQUIRED_FIELDS[@]}"; do
|
||||
if ! yq -e "has(\"$field\")" config.yaml > /dev/null; then
|
||||
echo "Missing required field: $field"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
```
|
||||
|
||||
### Secret Management
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# Extract secrets from YAML
|
||||
yq '.secrets' config.yaml > secrets.yaml
|
||||
|
||||
# Encrypt secrets (example with age)
|
||||
yq '.secrets' config.yaml | age -e -o secrets.age
|
||||
|
||||
# Decrypt and merge
|
||||
age -d secrets.age | yq -P . > secrets.yaml
|
||||
yq '. *= load("secrets.yaml")' config.yaml
|
||||
```
|
||||
|
||||
## Integration Examples
|
||||
|
||||
### With Git
|
||||
|
||||
```bash
|
||||
# Pre-commit hook to sort keys
|
||||
git diff --cached --name-only | \
|
||||
grep '\.yaml$' | \
|
||||
xargs -I {} yq -i -P 'sort_keys(..)' {}
|
||||
```
|
||||
|
||||
### With Docker
|
||||
|
||||
```bash
|
||||
# Extract image tags
|
||||
yq '.services[].image' docker-compose.yml | \
|
||||
cut -d: -f2 | \
|
||||
sort -u
|
||||
|
||||
# Update all images to latest
|
||||
yq -i '(.services[].image) |= sub(":.*", ":latest")' docker-compose.yml
|
||||
```
|
||||
|
||||
### With Kubernetes
|
||||
|
||||
```bash
|
||||
# Apply with modifications
|
||||
yq '.spec.replicas = 3' deployment.yaml | kubectl apply -f -
|
||||
|
||||
# Extract all images
|
||||
kubectl get pods -o yaml | \
|
||||
yq '.items[].spec.containers[].image' | \
|
||||
sort -u
|
||||
|
||||
# Generate manifests
|
||||
yq eval-all '. as $item ireduce ({}; . * $item)' \
|
||||
base/*.yaml overlays/prod/*.yaml | \
|
||||
kubectl apply -f -
|
||||
```
|
||||
|
||||
### With CI/CD
|
||||
|
||||
```bash
|
||||
# Update version in workflow
|
||||
VERSION=$(git describe --tags)
|
||||
yq -i ".env.VERSION = \"$VERSION\"" .github/workflows/deploy.yml
|
||||
|
||||
# Extract job matrix
|
||||
MATRIX=$(yq -o json '.jobs.test.strategy.matrix' .github/workflows/ci.yml)
|
||||
echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
|
||||
```
|
||||
|
||||
## Performance Tips
|
||||
|
||||
1. **Use in-place editing**: `-i` avoids reading/writing twice
|
||||
2. **Pipe instead of temp files**: More efficient for transformations
|
||||
3. **Specific paths**: Access `.field.subfield` instead of iterating
|
||||
4. **Limit output**: Use `select` early to filter data
|
||||
5. **Batch operations**: Combine multiple updates in one command
|
||||
|
||||
## Common Patterns
|
||||
|
||||
### Configuration Merging
|
||||
|
||||
```bash
|
||||
# Base + environment + local
|
||||
yq eval-all '. as $item ireduce ({}; . * $item)' \
|
||||
config.base.yaml \
|
||||
config.$ENV.yaml \
|
||||
config.local.yaml
|
||||
```
|
||||
|
||||
### Secret Injection
|
||||
|
||||
```bash
|
||||
# Replace placeholders with actual values
|
||||
yq '(.. | select(tag == "!!str")) |= envsubst' config.yaml
|
||||
```
|
||||
|
||||
### Manifest Generation
|
||||
|
||||
```bash
|
||||
# Generate K8s manifests from template
|
||||
yq '(.metadata.name) = "app-" + env(ENV) |
|
||||
(.spec.replicas) = env(REPLICAS) |
|
||||
(.spec.template.spec.containers[0].image) = env(IMAGE)' \
|
||||
template.yaml
|
||||
```
|
||||
|
||||
## Tips and Tricks
|
||||
|
||||
### Debugging
|
||||
|
||||
```bash
|
||||
# Pretty print to see structure
|
||||
yq -P . file.yaml
|
||||
|
||||
# Show paths
|
||||
yq '.. | path | join(".")' file.yaml
|
||||
|
||||
# Find all keys
|
||||
yq '.. | select(tag == "!!map") | keys' file.yaml
|
||||
```
|
||||
|
||||
### Working with Large Files
|
||||
|
||||
```bash
|
||||
# Stream processing
|
||||
yq -N . large.yaml # Process without loading entirely
|
||||
|
||||
# Extract specific document
|
||||
yq 'select(document_index == 5)' large-multi-doc.yaml
|
||||
|
||||
# Count documents
|
||||
yq -N '.' multi-doc.yaml | grep -c '^---$'
|
||||
```
|
||||
|
||||
### Shell Aliases
|
||||
|
||||
```bash
|
||||
alias yqp='yq -P' # Pretty print
|
||||
alias yqj='yq -o json' # To JSON
|
||||
alias yqs='yq -I 2' # 2-space indent
|
||||
alias yqc='yq -C 1' # Force colors
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
```bash
|
||||
# Check if file exists and is valid
|
||||
if yq . file.yaml &>/dev/null; then
|
||||
echo "Valid YAML"
|
||||
else
|
||||
echo "Invalid YAML or file not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Try with fallback
|
||||
VALUE=$(yq '.field // "default"' file.yaml)
|
||||
|
||||
# Conditional processing
|
||||
if yq -e '.enabled == true' config.yaml > /dev/null; then
|
||||
# Process when enabled
|
||||
yq '.items[]' config.yaml
|
||||
fi
|
||||
```
|
||||
|
||||
## Comparison with Other Tools
|
||||
|
||||
| Task | yq | jq (JSON) | sed/awk |
|
||||
|------|----|-----------| --------|
|
||||
| Parse YAML | Native | Need conversion | Complex |
|
||||
| Parse JSON | Can do | Native | Possible |
|
||||
| Update in-place | `-i` flag | Needs sponge | `-i` flag |
|
||||
| Preserve comments | Yes | N/A | Yes |
|
||||
| Type safety | Yes | Yes | No |
|
||||
| Learning curve | Medium | Medium | Low/High |
|
||||
Reference in New Issue
Block a user