Initial commit
This commit is contained in:
Zhongwei Li
406
skills/cli-testing-patterns/examples/exit-code-testing/README.md
Normal file
406
skills/cli-testing-patterns/examples/exit-code-testing/README.md
Normal file
@@ -0,0 +1,406 @@
|
||||
# Exit Code Testing Patterns
|
||||
|
||||
Comprehensive guide to testing CLI exit codes correctly.
|
||||
|
||||
## Standard Exit Codes
|
||||
|
||||
### POSIX Standard Exit Codes
|
||||
|
||||
| Code | Meaning | When to Use |
|
||||
|------|---------|-------------|
|
||||
| 0 | Success | Command completed successfully |
|
||||
| 1 | General Error | Catchall for general errors |
|
||||
| 2 | Misuse of Command | Invalid arguments or options |
|
||||
| 126 | Command Cannot Execute | Permission problem or not executable |
|
||||
| 127 | Command Not Found | Command not found in PATH |
|
||||
| 128+N | Fatal Error Signal N | Process terminated by signal N |
|
||||
| 130 | Ctrl+C Termination | Process terminated by SIGINT |
|
||||
|
||||
### Custom Application Exit Codes
|
||||
|
||||
```typescript
|
||||
// Define custom exit codes
|
||||
enum ExitCode {
|
||||
SUCCESS = 0,
|
||||
GENERAL_ERROR = 1,
|
||||
INVALID_ARGUMENT = 2,
|
||||
CONFIG_ERROR = 3,
|
||||
NETWORK_ERROR = 4,
|
||||
AUTH_ERROR = 5,
|
||||
NOT_FOUND = 6,
|
||||
ALREADY_EXISTS = 7,
|
||||
PERMISSION_DENIED = 8,
|
||||
}
|
||||
```
|
||||
|
||||
## Node.js Exit Code Testing
|
||||
|
||||
### Basic Exit Code Testing
|
||||
|
||||
```typescript
|
||||
describe('Exit Code Tests', () => {
|
||||
test('success returns 0', () => {
|
||||
const { code } = runCLI('status');
|
||||
expect(code).toBe(0);
|
||||
});
|
||||
|
||||
test('general error returns 1', () => {
|
||||
const { code } = runCLI('fail-command');
|
||||
expect(code).toBe(1);
|
||||
});
|
||||
|
||||
test('invalid argument returns 2', () => {
|
||||
const { code } = runCLI('deploy --invalid-env unknown');
|
||||
expect(code).toBe(2);
|
||||
});
|
||||
|
||||
test('command not found returns 127', () => {
|
||||
const { code } = runCLI('nonexistent-command');
|
||||
expect(code).toBe(127);
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### Specific Error Conditions
|
||||
|
||||
```typescript
|
||||
describe('Specific Exit Codes', () => {
|
||||
test('configuration error', () => {
|
||||
const { code, stderr } = runCLI('deploy production');
|
||||
expect(code).toBe(3); // CONFIG_ERROR
|
||||
expect(stderr).toContain('configuration');
|
||||
});
|
||||
|
||||
test('network error', () => {
|
||||
// Mock network failure
|
||||
const { code, stderr } = runCLI('fetch --url https://unreachable.example.com');
|
||||
expect(code).toBe(4); // NETWORK_ERROR
|
||||
expect(stderr).toContain('network');
|
||||
});
|
||||
|
||||
test('authentication error', () => {
|
||||
const { code, stderr } = runCLI('login --token invalid');
|
||||
expect(code).toBe(5); // AUTH_ERROR
|
||||
expect(stderr).toContain('authentication');
|
||||
});
|
||||
|
||||
test('resource not found', () => {
|
||||
const { code, stderr } = runCLI('get resource-123');
|
||||
expect(code).toBe(6); // NOT_FOUND
|
||||
expect(stderr).toContain('not found');
|
||||
});
|
||||
|
||||
test('resource already exists', () => {
|
||||
runCLI('create my-resource');
|
||||
const { code, stderr } = runCLI('create my-resource');
|
||||
expect(code).toBe(7); // ALREADY_EXISTS
|
||||
expect(stderr).toContain('already exists');
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### Testing Exit Code Consistency
|
||||
|
||||
```typescript
|
||||
describe('Exit Code Consistency', () => {
|
||||
const errorScenarios = [
|
||||
{ args: 'deploy', expectedCode: 2, reason: 'missing required argument' },
|
||||
{ args: 'deploy --env invalid', expectedCode: 2, reason: 'invalid environment' },
|
||||
{ args: 'config get missing', expectedCode: 6, reason: 'config key not found' },
|
||||
{ args: 'unknown-cmd', expectedCode: 127, reason: 'command not found' },
|
||||
];
|
||||
|
||||
test.each(errorScenarios)(
|
||||
'should return exit code $expectedCode for $reason',
|
||||
({ args, expectedCode }) => {
|
||||
const { code } = runCLI(args);
|
||||
expect(code).toBe(expectedCode);
|
||||
}
|
||||
);
|
||||
});
|
||||
```
|
||||
|
||||
## Python Exit Code Testing
|
||||
|
||||
### Basic Exit Code Testing
|
||||
|
||||
```python
|
||||
class TestExitCodes:
|
||||
"""Test CLI exit codes"""
|
||||
|
||||
def test_success_exit_code(self, runner):
|
||||
"""Success should return 0"""
|
||||
result = runner.invoke(cli, ['status'])
|
||||
assert result.exit_code == 0
|
||||
|
||||
def test_general_error_exit_code(self, runner):
|
||||
"""General error should return 1"""
|
||||
result = runner.invoke(cli, ['fail-command'])
|
||||
assert result.exit_code == 1
|
||||
|
||||
def test_usage_error_exit_code(self, runner):
|
||||
"""Usage error should return 2"""
|
||||
result = runner.invoke(cli, ['deploy']) # Missing required arg
|
||||
assert result.exit_code == 2
|
||||
|
||||
def test_unknown_command_exit_code(self, runner):
|
||||
"""Unknown command handling"""
|
||||
result = runner.invoke(cli, ['nonexistent'])
|
||||
assert result.exit_code != 0
|
||||
```
|
||||
|
||||
### Custom Exit Codes with Click
|
||||
|
||||
```python
|
||||
import click
|
||||
import sys
|
||||
|
||||
# Define custom exit codes
|
||||
class ExitCode:
|
||||
SUCCESS = 0
|
||||
GENERAL_ERROR = 1
|
||||
INVALID_ARGUMENT = 2
|
||||
CONFIG_ERROR = 3
|
||||
NETWORK_ERROR = 4
|
||||
AUTH_ERROR = 5
|
||||
|
||||
|
||||
@click.command()
|
||||
def deploy():
|
||||
"""Deploy command with custom exit codes"""
|
||||
try:
|
||||
# Check configuration
|
||||
if not has_valid_config():
|
||||
click.echo("Configuration error", err=True)
|
||||
sys.exit(ExitCode.CONFIG_ERROR)
|
||||
|
||||
# Check authentication
|
||||
if not is_authenticated():
|
||||
click.echo("Authentication failed", err=True)
|
||||
sys.exit(ExitCode.AUTH_ERROR)
|
||||
|
||||
# Deploy
|
||||
deploy_application()
|
||||
click.echo("Deployment successful")
|
||||
sys.exit(ExitCode.SUCCESS)
|
||||
|
||||
except NetworkError:
|
||||
click.echo("Network error", err=True)
|
||||
sys.exit(ExitCode.NETWORK_ERROR)
|
||||
except Exception as e:
|
||||
click.echo(f"Error: {e}", err=True)
|
||||
sys.exit(ExitCode.GENERAL_ERROR)
|
||||
```
|
||||
|
||||
### Testing Custom Exit Codes
|
||||
|
||||
```python
|
||||
class TestCustomExitCodes:
|
||||
"""Test custom exit codes"""
|
||||
|
||||
def test_config_error_exit_code(self, runner, tmp_path):
|
||||
"""Configuration error should return 3"""
|
||||
# Remove config file
|
||||
result = runner.invoke(cli, ['deploy', 'production'])
|
||||
assert result.exit_code == 3
|
||||
assert 'configuration' in result.output.lower()
|
||||
|
||||
def test_network_error_exit_code(self, runner, monkeypatch):
|
||||
"""Network error should return 4"""
|
||||
def mock_request(*args, **kwargs):
|
||||
raise NetworkError("Connection failed")
|
||||
|
||||
monkeypatch.setattr('requests.post', mock_request)
|
||||
result = runner.invoke(cli, ['deploy', 'production'])
|
||||
assert result.exit_code == 4
|
||||
assert 'network' in result.output.lower()
|
||||
|
||||
def test_auth_error_exit_code(self, runner):
|
||||
"""Authentication error should return 5"""
|
||||
result = runner.invoke(cli, ['deploy', 'production', '--token', 'invalid'])
|
||||
assert result.exit_code == 5
|
||||
assert 'authentication' in result.output.lower()
|
||||
```
|
||||
|
||||
## Testing Exit Codes in Scripts
|
||||
|
||||
### Bash Script Exit Code Testing
|
||||
|
||||
```typescript
|
||||
describe('Script Exit Codes', () => {
|
||||
test('should respect shell exit codes', () => {
|
||||
// Test that CLI properly exits with script error codes
|
||||
const script = `
|
||||
#!/bin/bash
|
||||
${CLI_PATH} deploy staging
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Deployment failed"
|
||||
exit 1
|
||||
fi
|
||||
echo "Deployment succeeded"
|
||||
`;
|
||||
|
||||
const { code, stdout } = execSync(script, { encoding: 'utf8' });
|
||||
expect(code).toBe(0);
|
||||
expect(stdout).toContain('Deployment succeeded');
|
||||
});
|
||||
|
||||
test('should propagate errors in pipelines', () => {
|
||||
const { code } = execSync(`${CLI_PATH} invalid | tee output.log`, {
|
||||
encoding: 'utf8',
|
||||
});
|
||||
expect(code).not.toBe(0);
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
## Exit Code Best Practices
|
||||
|
||||
### 1. Document Exit Codes
|
||||
|
||||
```typescript
|
||||
/**
|
||||
* CLI Exit Codes
|
||||
*
|
||||
* 0 - Success
|
||||
* 1 - General error
|
||||
* 2 - Invalid arguments
|
||||
* 3 - Configuration error
|
||||
* 4 - Network error
|
||||
* 5 - Authentication error
|
||||
* 6 - Resource not found
|
||||
* 7 - Resource already exists
|
||||
* 8 - Permission denied
|
||||
*/
|
||||
```
|
||||
|
||||
### 2. Consistent Error Handling
|
||||
|
||||
```python
|
||||
def handle_error(error: Exception) -> int:
|
||||
"""
|
||||
Handle errors and return appropriate exit code
|
||||
|
||||
Returns:
|
||||
Appropriate exit code for the error type
|
||||
"""
|
||||
if isinstance(error, ConfigurationError):
|
||||
click.echo(f"Configuration error: {error}", err=True)
|
||||
return ExitCode.CONFIG_ERROR
|
||||
elif isinstance(error, NetworkError):
|
||||
click.echo(f"Network error: {error}", err=True)
|
||||
return ExitCode.NETWORK_ERROR
|
||||
elif isinstance(error, AuthenticationError):
|
||||
click.echo(f"Authentication failed: {error}", err=True)
|
||||
return ExitCode.AUTH_ERROR
|
||||
else:
|
||||
click.echo(f"Error: {error}", err=True)
|
||||
return ExitCode.GENERAL_ERROR
|
||||
```
|
||||
|
||||
### 3. Test Exit Codes with Error Messages
|
||||
|
||||
```typescript
|
||||
test('exit code matches error type', () => {
|
||||
const errorCases = [
|
||||
{ args: 'deploy', expectedCode: 2, expectedMsg: 'missing required argument' },
|
||||
{ args: 'login --token bad', expectedCode: 5, expectedMsg: 'authentication failed' },
|
||||
{ args: 'get missing-id', expectedCode: 6, expectedMsg: 'not found' },
|
||||
];
|
||||
|
||||
errorCases.forEach(({ args, expectedCode, expectedMsg }) => {
|
||||
const { code, stderr } = runCLI(args);
|
||||
expect(code).toBe(expectedCode);
|
||||
expect(stderr.toLowerCase()).toContain(expectedMsg);
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### 4. Test Help and Version Return 0
|
||||
|
||||
```python
|
||||
def test_help_returns_success(runner):
|
||||
"""Help should return 0"""
|
||||
result = runner.invoke(cli, ['--help'])
|
||||
assert result.exit_code == 0
|
||||
|
||||
def test_version_returns_success(runner):
|
||||
"""Version should return 0"""
|
||||
result = runner.invoke(cli, ['--version'])
|
||||
assert result.exit_code == 0
|
||||
```
|
||||
|
||||
## Common Pitfalls
|
||||
|
||||
### 1. Don't Use Exit Code 0 for Errors
|
||||
|
||||
```typescript
|
||||
// ❌ Wrong - using 0 for errors
|
||||
if (error) {
|
||||
console.error('Error occurred');
|
||||
process.exit(0); // Should be non-zero!
|
||||
}
|
||||
|
||||
// ✅ Correct - using non-zero for errors
|
||||
if (error) {
|
||||
console.error('Error occurred');
|
||||
process.exit(1);
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Don't Ignore Exit Codes in Tests
|
||||
|
||||
```python
|
||||
# ❌ Wrong - not checking exit code
|
||||
def test_deploy(runner):
|
||||
result = runner.invoke(cli, ['deploy', 'production'])
|
||||
assert 'deployed' in result.output # What if it failed?
|
||||
|
||||
# ✅ Correct - always check exit code
|
||||
def test_deploy(runner):
|
||||
result = runner.invoke(cli, ['deploy', 'production'])
|
||||
assert result.exit_code == 0
|
||||
assert 'deployed' in result.output
|
||||
```
|
||||
|
||||
### 3. Use Specific Exit Codes
|
||||
|
||||
```typescript
|
||||
// ❌ Wrong - using 1 for everything
|
||||
if (configError) process.exit(1);
|
||||
if (networkError) process.exit(1);
|
||||
if (authError) process.exit(1);
|
||||
|
||||
// ✅ Correct - using specific codes
|
||||
if (configError) process.exit(ExitCode.CONFIG_ERROR);
|
||||
if (networkError) process.exit(ExitCode.NETWORK_ERROR);
|
||||
if (authError) process.exit(ExitCode.AUTH_ERROR);
|
||||
```
|
||||
|
||||
## Testing Exit Codes in CI/CD
|
||||
|
||||
```yaml
|
||||
# GitHub Actions example
|
||||
- name: Test CLI Exit Codes
|
||||
run: |
|
||||
# Should succeed
|
||||
./cli status && echo "Status check passed" || exit 1
|
||||
|
||||
# Should fail
|
||||
./cli invalid-command && exit 1 || echo "Error handling works"
|
||||
|
||||
# Check specific exit code
|
||||
./cli deploy --missing-arg
|
||||
if [ $? -eq 2 ]; then
|
||||
echo "Correct exit code for invalid argument"
|
||||
else
|
||||
echo "Wrong exit code"
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
## Resources
|
||||
|
||||
- [Exit Codes on Linux](https://tldp.org/LDP/abs/html/exitcodes.html)
|
||||
- [POSIX Exit Codes](https://pubs.opengroup.org/onlinepubs/9699919799/)
|
||||
- [GNU Exit Codes](https://www.gnu.org/software/libc/manual/html_node/Exit-Status.html)
|
||||
Reference in New Issue
Block a user