#!/bin/bash # Test Template for File-Manipulation Skills # Use this template when testing skills that: # - Create, read, update, or delete files # - Modify configurations or codebases # - Generate reports or artifacts # - Work with filesystem operations set -euo pipefail # ============================================================================ # Configuration # ============================================================================ SKILL_NAME="${1:-example-file-skill}" SKILL_PATH="$HOME/.claude/skills/$SKILL_NAME" TEST_ID="$(date +%s)" TEST_DIR="/tmp/skill-test-$TEST_ID" # ============================================================================ # Load Helper Library # ============================================================================ HELPER_LIB="$HOME/.claude/skills/skill-isolation-tester/lib/docker-helpers.sh" if [[ ! -f "$HELPER_LIB" ]]; then echo "ERROR: Helper library not found: $HELPER_LIB" exit 1 fi # shellcheck source=/dev/null source "$HELPER_LIB" # ============================================================================ # Setup Cleanup Trap # ============================================================================ export SKILL_TEST_TEMP_DIR="$TEST_DIR" export SKILL_TEST_KEEP_CONTAINER="false" export SKILL_TEST_REMOVE_IMAGES="true" trap cleanup_on_exit EXIT # ============================================================================ # Pre-flight Checks # ============================================================================ echo "=== File Manipulation Skill Test: $SKILL_NAME ===" echo "Test ID: $TEST_ID" echo "" # Validate skill exists if [[ ! -d "$SKILL_PATH" ]]; then echo "ERROR: Skill not found: $SKILL_PATH" exit 1 fi # Validate Docker environment preflight_check_docker || exit 1 # ============================================================================ # Build Test Environment with Sample Files # ============================================================================ echo "" echo "=== Building Test Environment ===" mkdir -p "$TEST_DIR/test-workspace" # Create sample files for the skill to manipulate cat > "$TEST_DIR/test-workspace/sample.txt" <<'EOF' This is a sample text file for testing. Line 2 Line 3 EOF cat > "$TEST_DIR/test-workspace/config.json" <<'EOF' { "setting1": "value1", "setting2": 42, "enabled": true } EOF mkdir -p "$TEST_DIR/test-workspace/subdir" echo "Nested file" > "$TEST_DIR/test-workspace/subdir/nested.txt" # Create Dockerfile cat > "$TEST_DIR/Dockerfile" < "$TEST_DIR/before-files.txt" # Get file sizes and checksums docker exec "$SKILL_TEST_CONTAINER_ID" bash -c " cd /workspace find . -type f -exec md5sum {} \; | sort " > "$TEST_DIR/before-checksums.txt" # Count files BEFORE_FILE_COUNT=$(docker exec "$SKILL_TEST_CONTAINER_ID" find /workspace -type f | wc -l) BEFORE_DIR_COUNT=$(docker exec "$SKILL_TEST_CONTAINER_ID" find /workspace -type d | wc -l) echo "Before execution:" echo " Files: $BEFORE_FILE_COUNT" echo " Directories: $BEFORE_DIR_COUNT" # ============================================================================ # Run Skill in Container # ============================================================================ echo "" echo "=== Running Skill in Isolated Container ===" # Execute skill echo "Executing skill..." docker exec "$SKILL_TEST_CONTAINER_ID" bash -c " cd /root/.claude/skills/$SKILL_NAME # Add your skill execution command here # Example: ./file-processor.sh /workspace echo 'Skill execution placeholder - customize this for your skill' " || { EXEC_EXIT_CODE=$? echo "ERROR: Skill execution failed with exit code: $EXEC_EXIT_CODE" exit "$EXEC_EXIT_CODE" } # ============================================================================ # Take "After" Filesystem Snapshot # ============================================================================ echo "" echo "=== Taking Filesystem Snapshot (After) ===" # Get updated file list docker exec "$SKILL_TEST_CONTAINER_ID" find /workspace -type f -o -type d | sort > "$TEST_DIR/after-files.txt" # Get updated checksums docker exec "$SKILL_TEST_CONTAINER_ID" bash -c " cd /workspace find . -type f -exec md5sum {} \; | sort " > "$TEST_DIR/after-checksums.txt" # Count files AFTER_FILE_COUNT=$(docker exec "$SKILL_TEST_CONTAINER_ID" find /workspace -type f | wc -l) AFTER_DIR_COUNT=$(docker exec "$SKILL_TEST_CONTAINER_ID" find /workspace -type d | wc -l) echo "After execution:" echo " Files: $AFTER_FILE_COUNT" echo " Directories: $AFTER_DIR_COUNT" # ============================================================================ # Analyze Filesystem Changes # ============================================================================ echo "" echo "=== Analyzing Filesystem Changes ===" # Files added echo "" echo "Files Added:" comm -13 "$TEST_DIR/before-files.txt" "$TEST_DIR/after-files.txt" > "$TEST_DIR/files-added.txt" ADDED_COUNT=$(wc -l < "$TEST_DIR/files-added.txt") echo " Count: $ADDED_COUNT" if [[ $ADDED_COUNT -gt 0 ]]; then head -10 "$TEST_DIR/files-added.txt" if [[ $ADDED_COUNT -gt 10 ]]; then echo " ... and $((ADDED_COUNT - 10)) more" fi fi # Files removed echo "" echo "Files Removed:" comm -23 "$TEST_DIR/before-files.txt" "$TEST_DIR/after-files.txt" > "$TEST_DIR/files-removed.txt" REMOVED_COUNT=$(wc -l < "$TEST_DIR/files-removed.txt") echo " Count: $REMOVED_COUNT" if [[ $REMOVED_COUNT -gt 0 ]]; then head -10 "$TEST_DIR/files-removed.txt" if [[ $REMOVED_COUNT -gt 10 ]]; then echo " ... and $((REMOVED_COUNT - 10)) more" fi fi # Files modified echo "" echo "Files Modified:" comm -12 "$TEST_DIR/before-files.txt" "$TEST_DIR/after-files.txt" | while read -r file; do BEFORE_HASH=$(grep "$file" "$TEST_DIR/before-checksums.txt" 2>/dev/null | awk '{print $1}' || echo "") AFTER_HASH=$(grep "$file" "$TEST_DIR/after-checksums.txt" 2>/dev/null | awk '{print $1}' || echo "") if [[ -n "$BEFORE_HASH" && -n "$AFTER_HASH" && "$BEFORE_HASH" != "$AFTER_HASH" ]]; then echo " $file" fi done | tee "$TEST_DIR/files-modified.txt" MODIFIED_COUNT=$(wc -l < "$TEST_DIR/files-modified.txt") echo " Count: $MODIFIED_COUNT" # ============================================================================ # Validate File Permissions # ============================================================================ echo "" echo "=== Checking File Permissions ===" # Find files with unusual permissions docker exec "$SKILL_TEST_CONTAINER_ID" bash -c " find /workspace -type f -perm /111 -ls " > "$TEST_DIR/executable-files.txt" || true EXECUTABLE_COUNT=$(wc -l < "$TEST_DIR/executable-files.txt") if [[ $EXECUTABLE_COUNT -gt 0 ]]; then echo "⚠ WARNING: Found $EXECUTABLE_COUNT executable files" cat "$TEST_DIR/executable-files.txt" else echo "✓ No unexpected executable files" fi # Check for world-writable files docker exec "$SKILL_TEST_CONTAINER_ID" bash -c " find /workspace -type f -perm -002 -ls " > "$TEST_DIR/world-writable-files.txt" || true WRITABLE_COUNT=$(wc -l < "$TEST_DIR/world-writable-files.txt") if [[ $WRITABLE_COUNT -gt 0 ]]; then echo "⚠ WARNING: Found $WRITABLE_COUNT world-writable files (security risk)" cat "$TEST_DIR/world-writable-files.txt" else echo "✓ No world-writable files" fi # ============================================================================ # Check for Sensitive Data # ============================================================================ echo "" echo "=== Scanning for Sensitive Data ===" # Check for potential secrets in new files docker exec "$SKILL_TEST_CONTAINER_ID" bash -c " grep -rni 'password\|api[-_]key\|secret\|token' /workspace " 2>/dev/null | tee "$TEST_DIR/potential-secrets.txt" || true SECRET_COUNT=$(wc -l < "$TEST_DIR/potential-secrets.txt") if [[ $SECRET_COUNT -gt 0 ]]; then echo "⚠ WARNING: Found $SECRET_COUNT lines with potential secrets" echo " Review: $TEST_DIR/potential-secrets.txt" else echo "✓ No obvious secrets detected" fi # ============================================================================ # Validate Cleanup Behavior # ============================================================================ echo "" echo "=== Validating Cleanup Behavior ===" # Check for leftover temp files docker exec "$SKILL_TEST_CONTAINER_ID" bash -c " find /tmp -name '*skill*' -o -name '*.tmp' -o -name '*.temp' " > "$TEST_DIR/temp-files.txt" || true TEMP_COUNT=$(wc -l < "$TEST_DIR/temp-files.txt") if [[ $TEMP_COUNT -gt 0 ]]; then echo "⚠ WARNING: Found $TEMP_COUNT leftover temp files" cat "$TEST_DIR/temp-files.txt" else echo "✓ No leftover temp files" fi # ============================================================================ # Generate Test Report # ============================================================================ echo "" echo "=== Test Report ===" echo "" CONTAINER_EXIT_CODE=$(get_container_exit_code "$SKILL_TEST_CONTAINER_ID") if [[ $CONTAINER_EXIT_CODE -eq 0 ]]; then echo "✅ TEST PASSED" else echo "❌ TEST FAILED" fi echo "" echo "Filesystem Changes Summary:" echo " - Files added: $ADDED_COUNT" echo " - Files removed: $REMOVED_COUNT" echo " - Files modified: $MODIFIED_COUNT" echo " - Total file count change: $((AFTER_FILE_COUNT - BEFORE_FILE_COUNT))" echo "" echo "Security & Quality Checklist:" [[ $EXECUTABLE_COUNT -eq 0 ]] && echo " ✓ No unexpected executable files" || echo " ✗ Found executable files" [[ $WRITABLE_COUNT -eq 0 ]] && echo " ✓ No world-writable files" || echo " ✗ Found world-writable files" [[ $SECRET_COUNT -eq 0 ]] && echo " ✓ No secrets in files" || echo " ✗ Potential secrets found" [[ $TEMP_COUNT -eq 0 ]] && echo " ✓ Clean temp directory" || echo " ✗ Leftover temp files" echo "" echo "Detailed Reports:" echo " - Files added: $TEST_DIR/files-added.txt" echo " - Files removed: $TEST_DIR/files-removed.txt" echo " - Files modified: $TEST_DIR/files-modified.txt" echo " - Before snapshot: $TEST_DIR/before-files.txt" echo " - After snapshot: $TEST_DIR/after-files.txt" # Exit with appropriate code if [[ $CONTAINER_EXIT_CODE -eq 0 ]]; then exit 0 else exit 1 fi