407 lines
11 KiB
Markdown
407 lines
11 KiB
Markdown
# Exit Code Testing Patterns
|
|
|
|
Comprehensive guide to testing CLI exit codes correctly.
|
|
|
|
## Standard Exit Codes
|
|
|
|
### POSIX Standard Exit Codes
|
|
|
|
| Code | Meaning | When to Use |
|
|
|------|---------|-------------|
|
|
| 0 | Success | Command completed successfully |
|
|
| 1 | General Error | Catchall for general errors |
|
|
| 2 | Misuse of Command | Invalid arguments or options |
|
|
| 126 | Command Cannot Execute | Permission problem or not executable |
|
|
| 127 | Command Not Found | Command not found in PATH |
|
|
| 128+N | Fatal Error Signal N | Process terminated by signal N |
|
|
| 130 | Ctrl+C Termination | Process terminated by SIGINT |
|
|
|
|
### Custom Application Exit Codes
|
|
|
|
```typescript
|
|
// Define custom exit codes
|
|
enum ExitCode {
|
|
SUCCESS = 0,
|
|
GENERAL_ERROR = 1,
|
|
INVALID_ARGUMENT = 2,
|
|
CONFIG_ERROR = 3,
|
|
NETWORK_ERROR = 4,
|
|
AUTH_ERROR = 5,
|
|
NOT_FOUND = 6,
|
|
ALREADY_EXISTS = 7,
|
|
PERMISSION_DENIED = 8,
|
|
}
|
|
```
|
|
|
|
## Node.js Exit Code Testing
|
|
|
|
### Basic Exit Code Testing
|
|
|
|
```typescript
|
|
describe('Exit Code Tests', () => {
|
|
test('success returns 0', () => {
|
|
const { code } = runCLI('status');
|
|
expect(code).toBe(0);
|
|
});
|
|
|
|
test('general error returns 1', () => {
|
|
const { code } = runCLI('fail-command');
|
|
expect(code).toBe(1);
|
|
});
|
|
|
|
test('invalid argument returns 2', () => {
|
|
const { code } = runCLI('deploy --invalid-env unknown');
|
|
expect(code).toBe(2);
|
|
});
|
|
|
|
test('command not found returns 127', () => {
|
|
const { code } = runCLI('nonexistent-command');
|
|
expect(code).toBe(127);
|
|
});
|
|
});
|
|
```
|
|
|
|
### Specific Error Conditions
|
|
|
|
```typescript
|
|
describe('Specific Exit Codes', () => {
|
|
test('configuration error', () => {
|
|
const { code, stderr } = runCLI('deploy production');
|
|
expect(code).toBe(3); // CONFIG_ERROR
|
|
expect(stderr).toContain('configuration');
|
|
});
|
|
|
|
test('network error', () => {
|
|
// Mock network failure
|
|
const { code, stderr } = runCLI('fetch --url https://unreachable.example.com');
|
|
expect(code).toBe(4); // NETWORK_ERROR
|
|
expect(stderr).toContain('network');
|
|
});
|
|
|
|
test('authentication error', () => {
|
|
const { code, stderr } = runCLI('login --token invalid');
|
|
expect(code).toBe(5); // AUTH_ERROR
|
|
expect(stderr).toContain('authentication');
|
|
});
|
|
|
|
test('resource not found', () => {
|
|
const { code, stderr } = runCLI('get resource-123');
|
|
expect(code).toBe(6); // NOT_FOUND
|
|
expect(stderr).toContain('not found');
|
|
});
|
|
|
|
test('resource already exists', () => {
|
|
runCLI('create my-resource');
|
|
const { code, stderr } = runCLI('create my-resource');
|
|
expect(code).toBe(7); // ALREADY_EXISTS
|
|
expect(stderr).toContain('already exists');
|
|
});
|
|
});
|
|
```
|
|
|
|
### Testing Exit Code Consistency
|
|
|
|
```typescript
|
|
describe('Exit Code Consistency', () => {
|
|
const errorScenarios = [
|
|
{ args: 'deploy', expectedCode: 2, reason: 'missing required argument' },
|
|
{ args: 'deploy --env invalid', expectedCode: 2, reason: 'invalid environment' },
|
|
{ args: 'config get missing', expectedCode: 6, reason: 'config key not found' },
|
|
{ args: 'unknown-cmd', expectedCode: 127, reason: 'command not found' },
|
|
];
|
|
|
|
test.each(errorScenarios)(
|
|
'should return exit code $expectedCode for $reason',
|
|
({ args, expectedCode }) => {
|
|
const { code } = runCLI(args);
|
|
expect(code).toBe(expectedCode);
|
|
}
|
|
);
|
|
});
|
|
```
|
|
|
|
## Python Exit Code Testing
|
|
|
|
### Basic Exit Code Testing
|
|
|
|
```python
|
|
class TestExitCodes:
|
|
"""Test CLI exit codes"""
|
|
|
|
def test_success_exit_code(self, runner):
|
|
"""Success should return 0"""
|
|
result = runner.invoke(cli, ['status'])
|
|
assert result.exit_code == 0
|
|
|
|
def test_general_error_exit_code(self, runner):
|
|
"""General error should return 1"""
|
|
result = runner.invoke(cli, ['fail-command'])
|
|
assert result.exit_code == 1
|
|
|
|
def test_usage_error_exit_code(self, runner):
|
|
"""Usage error should return 2"""
|
|
result = runner.invoke(cli, ['deploy']) # Missing required arg
|
|
assert result.exit_code == 2
|
|
|
|
def test_unknown_command_exit_code(self, runner):
|
|
"""Unknown command handling"""
|
|
result = runner.invoke(cli, ['nonexistent'])
|
|
assert result.exit_code != 0
|
|
```
|
|
|
|
### Custom Exit Codes with Click
|
|
|
|
```python
|
|
import click
|
|
import sys
|
|
|
|
# Define custom exit codes
|
|
class ExitCode:
|
|
SUCCESS = 0
|
|
GENERAL_ERROR = 1
|
|
INVALID_ARGUMENT = 2
|
|
CONFIG_ERROR = 3
|
|
NETWORK_ERROR = 4
|
|
AUTH_ERROR = 5
|
|
|
|
|
|
@click.command()
|
|
def deploy():
|
|
"""Deploy command with custom exit codes"""
|
|
try:
|
|
# Check configuration
|
|
if not has_valid_config():
|
|
click.echo("Configuration error", err=True)
|
|
sys.exit(ExitCode.CONFIG_ERROR)
|
|
|
|
# Check authentication
|
|
if not is_authenticated():
|
|
click.echo("Authentication failed", err=True)
|
|
sys.exit(ExitCode.AUTH_ERROR)
|
|
|
|
# Deploy
|
|
deploy_application()
|
|
click.echo("Deployment successful")
|
|
sys.exit(ExitCode.SUCCESS)
|
|
|
|
except NetworkError:
|
|
click.echo("Network error", err=True)
|
|
sys.exit(ExitCode.NETWORK_ERROR)
|
|
except Exception as e:
|
|
click.echo(f"Error: {e}", err=True)
|
|
sys.exit(ExitCode.GENERAL_ERROR)
|
|
```
|
|
|
|
### Testing Custom Exit Codes
|
|
|
|
```python
|
|
class TestCustomExitCodes:
|
|
"""Test custom exit codes"""
|
|
|
|
def test_config_error_exit_code(self, runner, tmp_path):
|
|
"""Configuration error should return 3"""
|
|
# Remove config file
|
|
result = runner.invoke(cli, ['deploy', 'production'])
|
|
assert result.exit_code == 3
|
|
assert 'configuration' in result.output.lower()
|
|
|
|
def test_network_error_exit_code(self, runner, monkeypatch):
|
|
"""Network error should return 4"""
|
|
def mock_request(*args, **kwargs):
|
|
raise NetworkError("Connection failed")
|
|
|
|
monkeypatch.setattr('requests.post', mock_request)
|
|
result = runner.invoke(cli, ['deploy', 'production'])
|
|
assert result.exit_code == 4
|
|
assert 'network' in result.output.lower()
|
|
|
|
def test_auth_error_exit_code(self, runner):
|
|
"""Authentication error should return 5"""
|
|
result = runner.invoke(cli, ['deploy', 'production', '--token', 'invalid'])
|
|
assert result.exit_code == 5
|
|
assert 'authentication' in result.output.lower()
|
|
```
|
|
|
|
## Testing Exit Codes in Scripts
|
|
|
|
### Bash Script Exit Code Testing
|
|
|
|
```typescript
|
|
describe('Script Exit Codes', () => {
|
|
test('should respect shell exit codes', () => {
|
|
// Test that CLI properly exits with script error codes
|
|
const script = `
|
|
#!/bin/bash
|
|
${CLI_PATH} deploy staging
|
|
if [ $? -ne 0 ]; then
|
|
echo "Deployment failed"
|
|
exit 1
|
|
fi
|
|
echo "Deployment succeeded"
|
|
`;
|
|
|
|
const { code, stdout } = execSync(script, { encoding: 'utf8' });
|
|
expect(code).toBe(0);
|
|
expect(stdout).toContain('Deployment succeeded');
|
|
});
|
|
|
|
test('should propagate errors in pipelines', () => {
|
|
const { code } = execSync(`${CLI_PATH} invalid | tee output.log`, {
|
|
encoding: 'utf8',
|
|
});
|
|
expect(code).not.toBe(0);
|
|
});
|
|
});
|
|
```
|
|
|
|
## Exit Code Best Practices
|
|
|
|
### 1. Document Exit Codes
|
|
|
|
```typescript
|
|
/**
|
|
* CLI Exit Codes
|
|
*
|
|
* 0 - Success
|
|
* 1 - General error
|
|
* 2 - Invalid arguments
|
|
* 3 - Configuration error
|
|
* 4 - Network error
|
|
* 5 - Authentication error
|
|
* 6 - Resource not found
|
|
* 7 - Resource already exists
|
|
* 8 - Permission denied
|
|
*/
|
|
```
|
|
|
|
### 2. Consistent Error Handling
|
|
|
|
```python
|
|
def handle_error(error: Exception) -> int:
|
|
"""
|
|
Handle errors and return appropriate exit code
|
|
|
|
Returns:
|
|
Appropriate exit code for the error type
|
|
"""
|
|
if isinstance(error, ConfigurationError):
|
|
click.echo(f"Configuration error: {error}", err=True)
|
|
return ExitCode.CONFIG_ERROR
|
|
elif isinstance(error, NetworkError):
|
|
click.echo(f"Network error: {error}", err=True)
|
|
return ExitCode.NETWORK_ERROR
|
|
elif isinstance(error, AuthenticationError):
|
|
click.echo(f"Authentication failed: {error}", err=True)
|
|
return ExitCode.AUTH_ERROR
|
|
else:
|
|
click.echo(f"Error: {error}", err=True)
|
|
return ExitCode.GENERAL_ERROR
|
|
```
|
|
|
|
### 3. Test Exit Codes with Error Messages
|
|
|
|
```typescript
|
|
test('exit code matches error type', () => {
|
|
const errorCases = [
|
|
{ args: 'deploy', expectedCode: 2, expectedMsg: 'missing required argument' },
|
|
{ args: 'login --token bad', expectedCode: 5, expectedMsg: 'authentication failed' },
|
|
{ args: 'get missing-id', expectedCode: 6, expectedMsg: 'not found' },
|
|
];
|
|
|
|
errorCases.forEach(({ args, expectedCode, expectedMsg }) => {
|
|
const { code, stderr } = runCLI(args);
|
|
expect(code).toBe(expectedCode);
|
|
expect(stderr.toLowerCase()).toContain(expectedMsg);
|
|
});
|
|
});
|
|
```
|
|
|
|
### 4. Test Help and Version Return 0
|
|
|
|
```python
|
|
def test_help_returns_success(runner):
|
|
"""Help should return 0"""
|
|
result = runner.invoke(cli, ['--help'])
|
|
assert result.exit_code == 0
|
|
|
|
def test_version_returns_success(runner):
|
|
"""Version should return 0"""
|
|
result = runner.invoke(cli, ['--version'])
|
|
assert result.exit_code == 0
|
|
```
|
|
|
|
## Common Pitfalls
|
|
|
|
### 1. Don't Use Exit Code 0 for Errors
|
|
|
|
```typescript
|
|
// ❌ Wrong - using 0 for errors
|
|
if (error) {
|
|
console.error('Error occurred');
|
|
process.exit(0); // Should be non-zero!
|
|
}
|
|
|
|
// ✅ Correct - using non-zero for errors
|
|
if (error) {
|
|
console.error('Error occurred');
|
|
process.exit(1);
|
|
}
|
|
```
|
|
|
|
### 2. Don't Ignore Exit Codes in Tests
|
|
|
|
```python
|
|
# ❌ Wrong - not checking exit code
|
|
def test_deploy(runner):
|
|
result = runner.invoke(cli, ['deploy', 'production'])
|
|
assert 'deployed' in result.output # What if it failed?
|
|
|
|
# ✅ Correct - always check exit code
|
|
def test_deploy(runner):
|
|
result = runner.invoke(cli, ['deploy', 'production'])
|
|
assert result.exit_code == 0
|
|
assert 'deployed' in result.output
|
|
```
|
|
|
|
### 3. Use Specific Exit Codes
|
|
|
|
```typescript
|
|
// ❌ Wrong - using 1 for everything
|
|
if (configError) process.exit(1);
|
|
if (networkError) process.exit(1);
|
|
if (authError) process.exit(1);
|
|
|
|
// ✅ Correct - using specific codes
|
|
if (configError) process.exit(ExitCode.CONFIG_ERROR);
|
|
if (networkError) process.exit(ExitCode.NETWORK_ERROR);
|
|
if (authError) process.exit(ExitCode.AUTH_ERROR);
|
|
```
|
|
|
|
## Testing Exit Codes in CI/CD
|
|
|
|
```yaml
|
|
# GitHub Actions example
|
|
- name: Test CLI Exit Codes
|
|
run: |
|
|
# Should succeed
|
|
./cli status && echo "Status check passed" || exit 1
|
|
|
|
# Should fail
|
|
./cli invalid-command && exit 1 || echo "Error handling works"
|
|
|
|
# Check specific exit code
|
|
./cli deploy --missing-arg
|
|
if [ $? -eq 2 ]; then
|
|
echo "Correct exit code for invalid argument"
|
|
else
|
|
echo "Wrong exit code"
|
|
exit 1
|
|
fi
|
|
```
|
|
|
|
## Resources
|
|
|
|
- [Exit Codes on Linux](https://tldp.org/LDP/abs/html/exitcodes.html)
|
|
- [POSIX Exit Codes](https://pubs.opengroup.org/onlinepubs/9699919799/)
|
|
- [GNU Exit Codes](https://www.gnu.org/software/libc/manual/html_node/Exit-Status.html)
|