Initial commit

skills/testing/examples/grpc-bufconn.md

@@ -0,0 +1,318 @@
+# gRPC Testing with bufconn and Rich Client Mocks
+
+## When to Use This Example
+
+Use this when:
+- Testing gRPC servers
+- Need bidirectional streaming tests
+- Want in-memory gRPC (no network I/O)
+- Testing server-client interactions
+- Need rich DSL for readable tests
+
+**Dependency Level**: Level 1 (In-Memory) - Uses `bufconn` for in-memory gRPC connections
+
+**Key Insight**: When testing a **gRPC server**, mock the **clients** that connect to it. When testing a **gRPC client**, mock the **server**.
+
+## Implementation
+
+### Rich gRPC Client Mock with DSL
+
+When your **System Under Test (SUT) is a gRPC server**, create rich client mocks:
+
+```go
+// internal/testutils/grpc_client_mock.go
+package testutils
+
+import (
+    "context"
+    "io"
+    "sync"
+    "testing"
+
+    "github.com/stretchr/testify/require"
+    "google.golang.org/grpc"
+    "google.golang.org/grpc/metadata"
+
+    pb "myproject/grpc_api/gen/go/traces/v1"
+)
+
+// TaskmonOnCluster is a rich gRPC client mock with DSL for testing gRPC servers.
+// It connects to your gRPC server and provides helper methods for assertions.
+type TaskmonOnCluster struct {
+    clusterRoutingKey       string
+    taskmonID               string
+    stream                  pb.RemoteTracesService_StreamTracesClient
+    mu                      sync.RWMutex
+    receivedQuery           *pb.TracesQuery
+    receivedQueriesPayloads []string
+}
+
+// OpenTaskmonToWekaHomeStream creates a gRPC client mock that connects to your server.
+// This is the constructor for the mock - returns a rich DSL object.
+func OpenTaskmonToWekaHomeStream(
+    ctx context.Context,
+    client pb.RemoteTracesServiceClient,
+    clusterRoutingKey, taskmonID string,
+) (*TaskmonOnCluster, error) {
+    // Inject metadata (like session tokens) into context
+    md := metadata.Pairs("X-Taskmon-session-token", clusterRoutingKey)
+    ctx = metadata.NewOutgoingContext(ctx, md)
+
+    // Open streaming connection to the server (your SUT)
+    stream, err := client.StreamTraces(ctx, grpc.Header(&md))
+    if err != nil {
+        return nil, err
+    }
+
+    return &TaskmonOnCluster{
+        stream:                  stream,
+        clusterRoutingKey:       clusterRoutingKey,
+        taskmonID:               taskmonID,
+        receivedQueriesPayloads: []string{},
+    }, nil
+}
+
+// SessionToken returns the session token (useful for assertions)
+func (m *TaskmonOnCluster) SessionToken() string {
+    return m.clusterRoutingKey
+}
+
+// Close closes the stream (idempotent)
+func (m *TaskmonOnCluster) Close() {
+    if m.stream == nil {
+        return
+    }
+    m.stream.CloseSend()
+}
+
+// ListenToStreamAndAssert is a helper that listens to server messages and asserts.
+// This makes tests read like documentation!
+func (m *TaskmonOnCluster) ListenToStreamAndAssert(
+    t *testing.T,
+    expectedQueryPayload,
+    resultPayload string,
+) {
+    for {
+        query, err := m.stream.Recv()
+        if err == io.EOF {
+            break
+        }
+        require.NoError(t, err, "Failed to receive query from server")
+
+        // Store received data (thread-safe)
+        m.mu.Lock()
+        m.receivedQuery = query
+        m.receivedQueriesPayloads = append(m.receivedQueriesPayloads, string(query.TracesQueryPayload))
+        m.mu.Unlock()
+
+        // Assert expected payload
+        require.Equal(t, expectedQueryPayload, string(query.TracesQueryPayload))
+
+        // Send response back to server
+        response := &pb.TracesFromServer{
+            TraceServerRoute: query.TraceServerRoute,
+            TracesPayload:    []byte(resultPayload),
+            MessageId:        query.MessageId,
+        }
+        err = m.stream.Send(response)
+        require.NoError(t, err, "Failed to send response")
+    }
+}
+
+// LastReceivedQuery returns the last received query (thread-safe)
+func (m *TaskmonOnCluster) LastReceivedQuery() *pb.TracesQuery {
+    m.mu.RLock()
+    defer m.mu.RUnlock()
+    return m.receivedQuery
+}
+
+// ReceivedQueriesPayloads returns all received payloads (thread-safe)
+func (m *TaskmonOnCluster) ReceivedQueriesPayloads() []string {
+    m.mu.RLock()
+    defer m.mu.RUnlock()
+    return m.receivedQueriesPayloads
+}
+```
+
+## Usage in Integration Tests
+
+### Complete Test Suite Example
+
+```go
+//go:build integration
+
+package integration_test
+
+import (
+    "context"
+    "net"
+    "testing"
+    "time"
+
+    "github.com/stretchr/testify/suite"
+    "google.golang.org/grpc"
+    "google.golang.org/grpc/credentials/insecure"
+    "google.golang.org/grpc/test/bufconn"
+
+    pb "myproject/grpc_api/gen/go/traces/v1"
+    "myproject/internal/remotetraces"
+    "myproject/internal/testutils"
+)
+
+type RemoteTracesTestSuite struct {
+    suite.Suite
+    lis        *bufconn.Listener  // In-memory gRPC connection
+    ctx        context.Context
+    natsServer *nserver.Server     // In-memory NATS
+}
+
+func (suite *RemoteTracesTestSuite) SetupSuite() {
+    suite.ctx = context.Background()
+
+    // Start in-memory NATS server (Level 1)
+    natsServer, err := testutils.RunNATsServer()
+    suite.Require().NoError(err)
+    suite.natsServer = natsServer
+
+    // Connect to NATS
+    natsAddress := "nats://" + natsServer.Addr().String()
+    nc, err := natsremotetraces.ConnectToRemoteTracesSession(suite.ctx, natsAddress, 2, 2, 10)
+    suite.Require().NoError(err)
+
+    // ** System Under Test: gRPC Server **
+    // Use bufconn for in-memory gRPC (no network I/O!)
+    suite.lis = bufconn.Listen(1024 * 1024)
+    s := grpc.NewServer()
+
+    // Your gRPC server implementation
+    remoteTracesServer := remotetraces.NewGRPCServer(nc, 10, 10, time.Second)
+    pb.RegisterRemoteTracesServiceServer(s, remoteTracesServer)
+
+    go func() {
+        if err := s.Serve(suite.lis); err != nil {
+            suite.NoError(err)
+        }
+    }()
+}
+
+func (suite *RemoteTracesTestSuite) bufDialer(ctx context.Context, _ string) (net.Conn, error) {
+    return suite.lis.DialContext(ctx)
+}
+
+func (suite *RemoteTracesTestSuite) TestStreamTraces() {
+    // Create gRPC client (connects to your server)
+    conn, err := grpc.NewClient("passthrough:///bufnet",
+        grpc.WithContextDialer(suite.bufDialer),
+        grpc.WithTransportCredentials(insecure.NewCredentials()),
+    )
+    suite.Require().NoError(err)
+    defer conn.Close()
+
+    client := pb.NewRemoteTracesServiceClient(conn)
+
+    // Create rich gRPC client mock (testutils DSL!)
+    clusterRoutingKey := "test-cluster-123"
+    taskmonMock, err := testutils.OpenTaskmonToWekaHomeStream(
+        suite.ctx, client, clusterRoutingKey, "taskmon-1")
+    suite.Require().NoError(err)
+    defer taskmonMock.Close()
+
+    expectedQuery := "fetch_traces_query"
+    expectedResult := "traces_result_data"
+
+    // Start listening (this makes the test readable!)
+    go taskmonMock.ListenToStreamAndAssert(suite.T(), expectedQuery, expectedResult)
+
+    // Send query to server (via NATS or HTTP API)
+    // ... your test logic here ...
+
+    // Assert using helper methods
+    suite.Eventually(func() bool {
+        return taskmonMock.LastReceivedQuery() != nil &&
+            string(taskmonMock.LastReceivedQuery().TracesQueryPayload) == expectedQuery
+    }, 5*time.Second, 500*time.Millisecond)
+}
+
+func TestRemoteTracesTestSuite(t *testing.T) {
+    suite.Run(t, new(RemoteTracesTestSuite))
+}
+```
+
+## Why This Pattern is Excellent
+
+1. **Rich DSL** - `OpenTaskmonToWekaHomeStream()` returns friendly object with helper methods
+2. **Helper Methods** - `ListenToStreamAndAssert()`, `LastReceivedQuery()`, `ReceivedQueriesPayloads()`
+3. **Thread-Safe** - Mutex protects shared state for concurrent access
+4. **Readable Tests** - Tests read like documentation, clear intent
+5. **In-Memory** - Uses `bufconn` (no network I/O, pure Go)
+6. **Reusable** - Same mock for unit, integration, and system tests
+7. **Event-Driven** - Can add channels for connection events if needed
+
+## Key Design Principles
+
+### Testing Direction
+
+- **Testing a server?** → Mock the **clients** that connect to it
+- **Testing a client?** → Mock the **server** it connects to
+
+### DSL Benefits
+
+- Use rich DSL objects with helper methods
+- Make tests read like documentation
+- Hide complexity behind clean interfaces
+- Provide thread-safe state tracking
+- Enable fluent assertions
+
+### In-Memory with bufconn
+
+`bufconn` provides an in-memory, full-duplex network connection:
+- No network I/O overhead
+- No port allocation needed
+- Faster than TCP loopback
+- Perfect for CI/CD
+- Deterministic behavior
+
+## Benefits
+
+- **No Docker required** - Pure Go, works anywhere
+- **No binary downloads** - Everything in-memory
+- **No network I/O** - Unless testing actual network code
+- **Perfect for CI/CD** - Fast, reliable, no external dependencies
+- **Lightning fast** - Microsecond startup time
+- **Thread-safe** - Concurrent test execution safe
+
+## Alternative: Testing gRPC Clients
+
+If you're testing a **gRPC client**, mock the **server** instead:
+
+```go
+// internal/testutils/grpc_server_mock.go
+type MockGRPCServer struct {
+    pb.UnimplementedRemoteTracesServiceServer
+    mu              sync.Mutex
+    receivedQueries []*pb.TracesQuery
+}
+
+func (m *MockGRPCServer) StreamTraces(stream pb.RemoteTracesService_StreamTracesServer) error {
+    // Mock server implementation
+    // Store received queries, send responses
+    // ...
+    return nil
+}
+
+// Usage
+server := testutils.NewMockGRPCServer()
+lis := bufconn.Listen(1024 * 1024)
+s := grpc.NewServer()
+pb.RegisterRemoteTracesServiceServer(s, server)
+// ... test your client against this mock server
+```
+
+## Key Takeaways
+
+1. **bufconn is Level 1** - In-memory, no external dependencies
+2. **Mock the opposite end** - Server → mock clients, Client → mock server
+3. **Rich DSL makes tests readable** - Helper methods, clear intent
+4. **Thread-safe state tracking** - Use mutexes for concurrent access
+5. **Reusable across test levels** - Same infrastructure everywhere
+6. **Check for official test harnesses first** - Many libraries provide them (like NATS)

skills/testing/examples/httptest-dsl.md

@@ -0,0 +1,281 @@
+# HTTP Test Server with DSL Pattern
+
+## When to Use This Example
+
+Use this when:
+- Testing HTTP clients or APIs
+- Need simple, readable HTTP mocking
+- Want to avoid complex mock frameworks
+- Testing REST APIs, webhooks, or HTTP integrations
+
+**Dependency Level**: Level 1 (In-Memory) - Uses stdlib `httptest.Server`
+
+## Basic httptest.Server Pattern
+
+### Simple HTTP Mock
+
+```go
+func TestAPIClient(t *testing.T) {
+    // Create test server
+    server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+        // Mock API response
+        w.WriteHeader(http.StatusOK)
+        json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
+    }))
+    defer server.Close()
+
+    // Use real HTTP client with test server URL
+    client := NewAPIClient(server.URL)
+    result, err := client.GetStatus()
+
+    assert.NoError(t, err)
+    assert.Equal(t, "ok", result.Status)
+}
+```
+
+## DSL Pattern for Readable Tests
+
+### Without DSL (Verbose)
+
+```go
+func TestUserAPI(t *testing.T) {
+    handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+        if r.Method == "GET" && r.URL.Path == "/users/1" {
+            w.WriteHeader(200)
+            json.NewEncoder(w).Encode(map[string]string{"id": "1", "name": "Alice"})
+        } else if r.Method == "POST" && r.URL.Path == "/users" {
+            // ... more complex logic
+        } else {
+            w.WriteHeader(404)
+        }
+    })
+    server := httptest.NewServer(handler)
+    defer server.Close()
+    // ... test
+}
+```
+
+### With DSL (Readable)
+
+```go
+func TestUserAPI(t *testing.T) {
+    mockAPI := httpserver.New().
+        OnGET("/users/1").
+            RespondJSON(200, User{ID: "1", Name: "Alice"}).
+        OnPOST("/users").
+            WithBodyMatcher(hasRequiredFields).
+            RespondJSON(201, User{ID: "2", Name: "Bob"}).
+        Build()
+    defer mockAPI.Close()
+
+    // Test reads like documentation!
+    client := NewAPIClient(mockAPI.URL())
+    user, err := client.GetUser("1")
+    // ... assertions
+}
+```
+
+## Implementing the DSL
+
+### Basic DSL Structure
+
+```go
+// internal/testutils/httpserver/server.go
+package httpserver
+
+import (
+    "encoding/json"
+    "net/http"
+    "net/http/httptest"
+)
+
+type MockServer struct {
+    routes  map[string]map[string]mockRoute // method -> path -> handler
+    server  *httptest.Server
+}
+
+type mockRoute struct {
+    statusCode int
+    response   any
+    matcher    func(*http.Request) bool
+}
+
+func New() *MockServerBuilder {
+    return &MockServerBuilder{
+        routes: make(map[string]map[string]mockRoute),
+    }
+}
+
+type MockServerBuilder struct {
+    routes map[string]map[string]mockRoute
+}
+
+func (b *MockServerBuilder) OnGET(path string) *RouteBuilder {
+    return &RouteBuilder{
+        builder: b,
+        method:  "GET",
+        path:    path,
+    }
+}
+
+func (b *MockServerBuilder) OnPOST(path string) *RouteBuilder {
+    return &RouteBuilder{
+        builder: b,
+        method:  "POST",
+        path:    path,
+    }
+}
+
+type RouteBuilder struct {
+    builder    *MockServerBuilder
+    method     string
+    path       string
+    statusCode int
+    response   any
+    matcher    func(*http.Request) bool
+}
+
+func (r *RouteBuilder) RespondJSON(statusCode int, response any) *MockServerBuilder {
+    if r.builder.routes[r.method] == nil {
+        r.builder.routes[r.method] = make(map[string]mockRoute)
+    }
+    r.builder.routes[r.method][r.path] = mockRoute{
+        statusCode: statusCode,
+        response:   response,
+        matcher:    r.matcher,
+    }
+    return r.builder
+}
+
+func (r *RouteBuilder) WithBodyMatcher(matcher func(*http.Request) bool) *RouteBuilder {
+    r.matcher = matcher
+    return r
+}
+
+func (b *MockServerBuilder) Build() *MockServer {
+    mock := &MockServer{routes: b.routes}
+
+    handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+        methodRoutes, ok := mock.routes[r.Method]
+        if !ok {
+            w.WriteHeader(http.StatusMethodNotAllowed)
+            return
+        }
+
+        route, ok := methodRoutes[r.URL.Path]
+        if !ok {
+            w.WriteHeader(http.StatusNotFound)
+            return
+        }
+
+        if route.matcher != nil && !route.matcher(r) {
+            w.WriteHeader(http.StatusBadRequest)
+            return
+        }
+
+        w.WriteHeader(route.statusCode)
+        json.NewEncoder(w).Encode(route.response)
+    })
+
+    mock.server = httptest.NewServer(handler)
+    return mock
+}
+
+func (m *MockServer) URL() string {
+    return m.server.URL
+}
+
+func (m *MockServer) Close() {
+    m.server.Close()
+}
+```
+
+## Simple In-Memory Patterns
+
+### In-Memory Repository
+
+```go
+// user/inmem.go
+package user
+
+type InMemoryRepository struct {
+    mu    sync.RWMutex
+    users map[UserID]User
+}
+
+func NewInMemoryRepository() *InMemoryRepository {
+    return &InMemoryRepository{
+        users: make(map[UserID]User),
+    }
+}
+
+func (r *InMemoryRepository) Save(ctx context.Context, u User) error {
+    r.mu.Lock()
+    defer r.mu.Unlock()
+    r.users[u.ID] = u
+    return nil
+}
+
+func (r *InMemoryRepository) Get(ctx context.Context, id UserID) (*User, error) {
+    r.mu.RLock()
+    defer r.mu.RUnlock()
+
+    u, ok := r.users[id]
+    if !ok {
+        return nil, ErrNotFound
+    }
+    return &u, nil
+}
+```
+
+### Test Email Sender
+
+```go
+// user/test_emailer.go
+package user
+
+import (
+    "bytes"
+    "fmt"
+    "sync"
+)
+
+type TestEmailer struct {
+    mu     sync.Mutex
+    buffer bytes.Buffer
+}
+
+func NewTestEmailer() *TestEmailer {
+    return &TestEmailer{}
+}
+
+func (e *TestEmailer) Send(to Email, subject, body string) error {
+    e.mu.Lock()
+    defer e.mu.Unlock()
+
+    fmt.Fprintf(&e.buffer, "To: %s\nSubject: %s\n%s\n\n", to, subject, body)
+    return nil
+}
+
+func (e *TestEmailer) SentEmails() string {
+    e.mu.Lock()
+    defer e.mu.Unlock()
+    return e.buffer.String()
+}
+```
+
+## Benefits
+
+- **Simple** - Built on stdlib, no external dependencies
+- **Readable** - DSL makes tests self-documenting
+- **Fast** - In-memory, microsecond startup
+- **Flexible** - Easy to extend with new methods
+- **Reusable** - Same pattern for all HTTP testing
+
+## Key Takeaways
+
+1. **Start with httptest.Server** - Simple and powerful
+2. **Add DSL for readability** - When tests get complex
+3. **Keep implementations simple** - In-memory maps, buffers
+4. **Thread-safe** - Use mutexes for concurrent access
+5. **Test your test infrastructure** - It's production code

skills/testing/examples/integration-patterns.md

@@ -0,0 +1,248 @@
+# Integration Test Patterns
+
+## Purpose
+
+Integration tests verify that components work together correctly. They test the seams between packages, ensure proper data flow, and validate that integrated components behave as expected.
+
+**When to Write**: After unit testing individual components, test how they interact.
+
+## File Organization
+
+### Option 1: In Package with Build Tags (Preferred)
+
+```go
+//go:build integration
+
+package user_test
+
+import (
+    "testing"
+    "myproject/internal/testutils"
+)
+
+func TestUserService_Integration(t *testing.T) {
+    // Integration test
+}
+```
+
+### Option 2: Separate Package
+
+```
+user/
+├── user.go
+├── user_test.go              # Unit tests
+└── integration/
+    └── user_integration_test.go  # Integration tests
+```
+
+## Pattern 1: Service + Repository (In-Memory)
+
+**Use when**: Testing service logic with data persistence
+
+```go
+//go:build integration
+
+package user_test
+
+import (
+    "context"
+    "testing"
+
+    "github.com/stretchr/testify/require"
+    "myproject/user"
+)
+
+func TestUserService_CreateAndRetrieve(t *testing.T) {
+    // Setup: In-memory repository (Level 1)
+    repo := user.NewInMemoryRepository()
+    svc := user.NewUserService(repo, nil)
+
+    ctx := context.Background()
+
+    // Create user
+    userID, _ := user.NewUserID("usr_123")
+    email, _ := user.NewEmail("alice@example.com")
+    newUser := user.User{
+        ID:    userID,
+        Name:  "Alice",
+        Email: email,
+    }
+
+    err := svc.CreateUser(ctx, newUser)
+    require.NoError(t, err)
+
+    // Retrieve user
+    retrieved, err := svc.GetUser(ctx, userID)
+    require.NoError(t, err)
+    require.Equal(t, "Alice", retrieved.Name)
+    require.Equal(t, email, retrieved.Email)
+}
+```
+
+## Pattern 2: Testing with Real External Service
+
+**Use when**: Need to test against real service behavior (Victoria Metrics, NATS, etc.)
+
+```go
+//go:build integration
+
+package metrics_test
+
+import (
+    "context"
+    "testing"
+
+    "github.com/stretchr/testify/require"
+    "myproject/internal/testutils"
+    "myproject/metrics"
+)
+
+func TestMetricsIngest_WithVictoriaMetrics(t *testing.T) {
+    // Start real Victoria Metrics (Level 2 - binary)
+    vmServer, err := testutils.RunVictoriaMetricsServer()
+    require.NoError(t, err)
+    defer vmServer.Shutdown()
+
+    // Create service with real dependency
+    svc := metrics.NewIngester(vmServer.WriteURL())
+
+    // Test ingestion
+    err = svc.IngestMetric(context.Background(), "test_metric", 42.0)
+    require.NoError(t, err)
+
+    // Force flush and verify
+    vmServer.ForceFlush(context.Background())
+    results, err := testutils.QueryVictoriaMetrics(vmServer.QueryURL(), "test_metric")
+    require.NoError(t, err)
+    require.Len(t, results, 1)
+}
+```
+
+## Pattern 3: Multi-Component Workflow
+
+**Use when**: Testing complete workflows across multiple components
+
+```go
+//go:build integration
+
+package workflow_test
+
+import (
+    "context"
+    "testing"
+    "time"
+
+    "github.com/stretchr/testify/suite"
+    "myproject/internal/testutils"
+    "myproject/user"
+    "myproject/notification"
+)
+
+type UserWorkflowSuite struct {
+    suite.Suite
+    userRepo    *user.InMemoryRepository
+    emailer     *user.TestEmailer
+    natsServer  *nserver.Server
+    userService *user.UserService
+    notifSvc    *notification.NotificationService
+}
+
+func (s *UserWorkflowSuite) SetupSuite() {
+    // Setup in-memory NATS (Level 1)
+    natsServer, err := testutils.RunNATsServer()
+    s.Require().NoError(err)
+    s.natsServer = natsServer
+
+    // Setup components
+    s.userRepo = user.NewInMemoryRepository()
+    s.emailer = user.NewTestEmailer()
+    s.userService = user.NewUserService(s.userRepo, s.emailer)
+
+    natsAddr := "nats://" + natsServer.Addr().String()
+    s.notifSvc = notification.NewService(natsAddr)
+}
+
+func (s *UserWorkflowSuite) TearDownSuite() {
+    s.natsServer.Shutdown()
+}
+
+func (s *UserWorkflowSuite) TestCreateUser_TriggersNotification() {
+    ctx := context.Background()
+
+    // Subscribe to notifications
+    received := make(chan string, 1)
+    s.notifSvc.Subscribe("user.created", func(msg string) {
+        received <- msg
+    })
+
+    // Create user
+    userID, _ := user.NewUserID("usr_123")
+    email, _ := user.NewEmail("alice@example.com")
+    newUser := user.User{ID: userID, Name: "Alice", Email: email}
+
+    err := s.userService.CreateUser(ctx, newUser)
+    s.Require().NoError(err)
+
+    // Verify notification sent
+    select {
+    case msg := <-received:
+        s.Contains(msg, "Alice")
+    case <-time.After(2 * time.Second):
+        s.Fail("timeout waiting for notification")
+    }
+
+    // Verify email sent
+    emails := s.emailer.SentEmails()
+    s.Contains(emails, "alice@example.com")
+}
+
+func TestUserWorkflowSuite(t *testing.T) {
+    suite.Run(t, new(UserWorkflowSuite))
+}
+```
+
+## Dependency Priority
+
+1. **Level 1: In-Memory** (Preferred) - httptest, in-memory maps, NATS harness
+2. **Level 2: Binary** (When needed) - Victoria Metrics, standalone services
+3. **Level 3: Test-containers** (Last resort) - Docker containers, slow startup
+
+## Best Practices
+
+### DO:
+- Test seams between components
+- Use in-memory implementations when possible
+- Test happy path and error scenarios
+- Use testify suites for complex setup
+- Focus on data flow and integration points
+
+### DON'T:
+- Don't test business logic (that's unit tests)
+- Don't use heavy mocking (use real implementations)
+- Don't require Docker unless absolutely necessary
+- Don't duplicate unit test coverage
+- Don't skip cleanup (always defer)
+
+## Running Integration Tests
+
+```bash
+# Skip integration tests (default)
+go test ./...
+
+# Run with integration tests
+go test -tags=integration ./...
+
+# Run only integration tests
+go test -tags=integration ./... -run Integration
+
+# With coverage
+go test -tags=integration -coverprofile=coverage.out ./...
+```
+
+## Key Takeaways
+
+1. **Test component interactions** - Not individual units
+2. **Prefer real implementations** - Over mocks when possible
+3. **Use build tags** - Keep unit tests fast
+4. **Reuse testutils** - Same infrastructure across tests
+5. **Test workflows** - Not just individual operations

skills/testing/examples/jsonrpc-mock.md

@@ -0,0 +1,263 @@
+# JSON-RPC Server Mock with DSL
+
+## When to Use This Example
+
+Use this when:
+- Testing JSON-RPC clients
+- Need to mock JSON-RPC server responses
+- Want configurable mock behavior per method
+- Need to track and assert on received requests
+- Testing with OpenTelemetry trace propagation
+
+**Dependency Level**: Level 1 (In-Memory) - Uses `httptest.Server` for in-memory HTTP
+
+**Key Insight**: When testing a **JSON-RPC client**, mock the **server** it calls. Use rich DSL for readable test setup.
+
+## Implementation
+
+### Rich JSON-RPC Server Mock
+
+```go
+// internal/testutils/jrpc_server_mock.go
+package testutils
+
+import (
+    "errors"
+    "fmt"
+    "net/http"
+    "net/http/httptest"
+
+    "github.com/gorilla/rpc/v2/json2"
+    "go.opentelemetry.io/otel"
+    "go.opentelemetry.io/otel/propagation"
+    "go.opentelemetry.io/otel/trace"
+)
+
+var ErrMethodNotFound = errors.New("method not found")
+
+// TraceQuery holds received JSON-RPC queries for assertions
+type TraceQuery struct {
+    Method string
+    Params string
+}
+
+// JrpcTraceServerMock is a rich JSON-RPC server mock with DSL.
+// Uses httptest.Server for in-memory HTTP (Level 1).
+type JrpcTraceServerMock struct {
+    tracer          trace.Tracer
+    server          *httptest.Server
+    mockResponses   map[string]any    // method -> response
+    queriesReceived []TraceQuery      // for assertions
+}
+
+// StartJrpcTraceServerMock starts an in-memory JSON-RPC server.
+// Returns a rich DSL object for configuring mock responses.
+func StartJrpcTraceServerMock() *JrpcTraceServerMock {
+    mock := &JrpcTraceServerMock{
+        mockResponses: make(map[string]any),
+        tracer:        otel.Tracer("trace-server-mock"),
+    }
+
+    mux := mock.createHTTPHandlers()
+    mock.server = httptest.NewServer(mux)
+
+    return mock
+}
+
+// AddMockResponse configures the mock to return a response for a method.
+// This is the DSL - chain multiple calls for different methods!
+func (m *JrpcTraceServerMock) AddMockResponse(method string, response any) {
+    m.mockResponses[method] = response
+}
+
+// GetQueriesReceived returns all queries received (for assertions)
+func (m *JrpcTraceServerMock) GetQueriesReceived() []TraceQuery {
+    return m.queriesReceived
+}
+
+// Close shuts down the server (idempotent)
+func (m *JrpcTraceServerMock) Close() {
+    m.server.Close()
+}
+
+// Address returns the server address (for client configuration)
+func (m *JrpcTraceServerMock) Address() string {
+    return m.server.Listener.Addr().String()
+}
+
+func (m *JrpcTraceServerMock) createHTTPHandlers() *http.ServeMux {
+    mux := http.NewServeMux()
+    codec := json2.NewCodec()
+
+    mux.HandleFunc("/reader", func(w http.ResponseWriter, r *http.Request) {
+        // Extract OpenTelemetry context for realistic testing
+        reqCtx := r.Context()
+        reqCtx = otel.GetTextMapPropagator().Extract(reqCtx, propagation.HeaderCarrier(r.Header))
+        reqCtx, span := m.tracer.Start(reqCtx, "jrpc-trace-server",
+            trace.WithSpanKind(trace.SpanKindServer))
+        defer span.End()
+
+        if r.Method != http.MethodPost {
+            w.WriteHeader(http.StatusMethodNotAllowed)
+            return
+        }
+
+        receivedReq := codec.NewRequest(r)
+        method, err := receivedReq.Method()
+        if err != nil {
+            w.WriteHeader(http.StatusBadRequest)
+            return
+        }
+
+        // Check if we have a mock response configured
+        if response, exists := m.mockResponses[method]; exists {
+            args := struct{}{}
+            if err := receivedReq.ReadRequest(&args); err != nil {
+                receivedReq.WriteError(w, http.StatusBadRequest, err)
+                return
+            }
+
+            // Store query for assertions
+            m.queriesReceived = append(m.queriesReceived, TraceQuery{
+                Method: method,
+                Params: fmt.Sprintf("%+v", args),
+            })
+
+            // Write mock response
+            receivedReq.WriteResponse(w, response)
+            return
+        }
+
+        // Method not configured
+        params := []string{}
+        receivedReq.ReadRequest(&params)
+        receivedReq.WriteError(w, http.StatusBadRequest, ErrMethodNotFound)
+    })
+
+    return mux
+}
+```
+
+## Usage Examples
+
+### Setup in Test Suite
+
+```go
+func (suite *TaskmonTestSuite) SetupSuite() {
+    // Start in-memory JSON-RPC server mock (Level 1)
+    suite.jrpcServerMock = testutils.StartJrpcTraceServerMock()
+
+    // Configure mock responses using DSL
+    suite.jrpcServerMock.AddMockResponse("protocol", struct {
+        Version string `json:"version"`
+        Date    string `json:"date"`
+    }{
+        Version: "3.18.0",
+        Date:    "Sep-04-2018",
+    })
+
+    suite.jrpcServerMock.AddMockResponse("get_traces", struct {
+        Traces []string `json:"traces"`
+    }{
+        Traces: []string{"trace1", "trace2"},
+    })
+
+    // Configure your client to use the mock server
+    client := jrpc.NewClient(suite.jrpcServerMock.Address() + "/reader")
+}
+
+func (suite *TaskmonTestSuite) TearDownSuite() {
+    suite.jrpcServerMock.Close()
+}
+```
+
+### Test with Assertions
+
+```go
+func (suite *TaskmonTestSuite) TestProtocolVersion() {
+    // Call your code that makes JSON-RPC requests
+    version, err := suite.taskmon.GetProtocolVersion()
+    suite.Require().NoError(err)
+    suite.Equal("3.18.0", version.Version)
+
+    // Assert on received queries
+    queries := suite.jrpcServerMock.GetQueriesReceived()
+    suite.Require().Len(queries, 1)
+    suite.Equal("protocol", queries[0].Method)
+}
+
+func (suite *TaskmonTestSuite) TestGetTraces() {
+    // Call your code
+    traces, err := suite.taskmon.GetTraces()
+    suite.Require().NoError(err)
+    suite.Equal([]string{"trace1", "trace2"}, traces)
+
+    // Verify the right method was called
+    queries := suite.jrpcServerMock.GetQueriesReceived()
+    suite.Require().Len(queries, 2) // protocol + get_traces
+    suite.Equal("get_traces", queries[1].Method)
+}
+```
+
+## Why This Pattern is Excellent
+
+1. **Rich DSL** - `AddMockResponse()` for easy, readable configuration
+2. **Readable Setup** - Tests are self-documenting, clear intent
+3. **In-Memory** - Uses `httptest.Server` (Level 1, no network I/O)
+4. **Query Tracking** - `GetQueriesReceived()` for assertions on what was called
+5. **OpenTelemetry Integration** - Realistic trace propagation for observability testing
+6. **Idempotent Cleanup** - Safe to call `Close()` multiple times
+7. **Flexible** - Configure any method/response combination dynamically
+
+## Key Design Principles
+
+### DSL for Configuration
+
+Mock setup should read like configuration:
+```go
+mock.AddMockResponse("method_name", expectedResponse)
+mock.AddMockResponse("another_method", anotherResponse)
+```
+
+### Query Tracking for Assertions
+
+Always track what was received:
+- Method names called
+- Parameters passed
+- Order of calls
+- Number of calls
+
+### Built on httptest.Server
+
+httptest.Server provides:
+- In-memory HTTP (no network I/O)
+- Automatic address allocation
+- Clean lifecycle management
+- Standard library, no dependencies
+
+## Pattern Comparison
+
+| Pattern | Use When |
+|---------|----------|
+| **httptest.Server** | Simple HTTP mocking |
+| **NATS test harness** | Need real NATS (pub/sub) |
+| **gRPC client mock** | Testing gRPC **server** |
+| **JSON-RPC server mock** | Testing JSON-RPC **client** |
+
+## Benefits
+
+- **In-Memory** - No network I/O, pure Go
+- **Fast** - Microsecond startup time
+- **Configurable** - Dynamic response configuration per test
+- **Trackable** - Full visibility into received requests
+- **OpenTelemetry-aware** - Realistic trace propagation
+- **Reusable** - Same infrastructure across test levels
+
+## Key Takeaways
+
+1. **Mock servers should have rich DSL** - Makes setup readable
+2. **Track received requests** - Essential for assertions
+3. **Use httptest.Server** - Perfect for HTTP-based protocols
+4. **Make setup read like configuration** - Self-documenting tests
+5. **Support trace propagation** - Realistic observability testing
+6. **Idempotent cleanup** - Safe resource management

skills/testing/examples/nats-in-memory.md

@@ -0,0 +1,175 @@
+# NATS In-Memory Test Server
+
+## When to Use This Example
+
+Use this when:
+- Testing message queue integrations with NATS
+- Need pub/sub functionality in tests
+- Want fast, in-memory NATS server (no Docker, no binary)
+- Testing event-driven architectures
+
+**Dependency Level**: Level 1 (In-Memory) - Pure Go, official test harness
+
+## Implementation
+
+### Setup Test Infrastructure
+
+Many official SDKs provide test harnesses. Here's NATS:
+
+```go
+// internal/testutils/nats.go
+package testutils
+
+import (
+    nserver "github.com/nats-io/nats-server/v2/server"
+    natsserver "github.com/nats-io/nats-server/v2/test"
+    "github.com/projectdiscovery/freeport"
+)
+
+// RunNATsServer runs a NATS server in-memory for testing.
+// Uses the official NATS SDK test harness - no binary download needed!
+func RunNATsServer() (*nserver.Server, error) {
+    opts := natsserver.DefaultTestOptions
+
+    // Allocate free port to prevent conflicts in parallel tests
+    tcpPort, err := freeport.GetFreePort("127.0.0.1", freeport.TCP)
+    if err != nil {
+        return nil, err
+    }
+
+    opts.Port = tcpPort.Port
+
+    // Start NATS server in-memory (pure Go!)
+    return natsserver.RunServer(&opts), nil
+}
+
+// RunNATsServerWithJetStream runs NATS with JetStream enabled
+func RunNATsServerWithJetStream() (*nserver.Server, error) {
+    opts := natsserver.DefaultTestOptions
+
+    tcpPort, err := freeport.GetFreePort("127.0.0.1", freeport.TCP)
+    if err != nil {
+        return nil, err
+    }
+
+    opts.Port = tcpPort.Port
+    opts.JetStream = true
+
+    return natsserver.RunServer(&opts), nil
+}
+```
+
+## Usage in Integration Tests
+
+### Basic Pub/Sub Test
+
+```go
+//go:build integration
+
+package integration_test
+
+import (
+    "context"
+    "testing"
+    "time"
+    "github.com/nats-io/nats.go"
+    "github.com/stretchr/testify/require"
+    "myproject/internal/testutils"
+)
+
+func TestNATSPubSub_Integration(t *testing.T) {
+    // Start NATS server in-memory (Level 1 - pure Go!)
+    natsServer, err := testutils.RunNATsServer()
+    require.NoError(t, err)
+    defer natsServer.Shutdown()
+
+    // Connect to in-memory NATS
+    natsAddress := "nats://" + natsServer.Addr().String()
+    nc, err := nats.Connect(natsAddress)
+    require.NoError(t, err)
+    defer nc.Close()
+
+    // Test pub/sub
+    received := make(chan string, 1)
+    _, err = nc.Subscribe("test.subject", func(msg *nats.Msg) {
+        received <- string(msg.Data)
+    })
+    require.NoError(t, err)
+
+    // Publish message
+    err = nc.Publish("test.subject", []byte("hello"))
+    require.NoError(t, err)
+
+    // Wait for message
+    select {
+    case msg := <-received:
+        require.Equal(t, "hello", msg)
+    case <-time.After(1 * time.Second):
+        t.Fatal("timeout waiting for message")
+    }
+}
+```
+
+### Real-World Usage Example (gRPC + NATS)
+
+```go
+// tests/gointegration/remote_traces_test.go
+type RemoteTracesTestSuite struct {
+    suite.Suite
+    natsServer  *nserver.Server
+    natsAddress string
+    nc          *nats.Conn
+    // ... other fields
+}
+
+func (suite *RemoteTracesTestSuite) SetupSuite() {
+    // Start NATS server in-memory
+    natsServer, err := testutils.RunNATsServer()
+    suite.Require().NoError(err)
+
+    suite.natsServer = natsServer
+    suite.natsAddress = "nats://" + natsServer.Addr().String()
+
+    // Connect application to in-memory NATS
+    suite.nc, err = natsremotetraces.ConnectToRemoteTracesSession(
+        suite.ctx, suite.natsAddress, numWorkers, numWorkers, channelSize)
+    suite.Require().NoError(err)
+
+    // Start gRPC server with NATS backend
+    // ... rest of setup
+}
+
+func (suite *RemoteTracesTestSuite) TearDownSuite() {
+    suite.nc.Close()
+    suite.natsServer.Shutdown() // Clean shutdown
+}
+
+func (suite *RemoteTracesTestSuite) TestMessageFlow() {
+    // Test your application logic that uses NATS
+    // ...
+}
+```
+
+## Why This is Excellent
+
+- **Pure Go** - NATS server imported as library (no binary download)
+- **Official** - Uses NATS SDK's official test harness
+- **Fast** - Starts in microseconds
+- **Reliable** - Same behavior as production NATS
+- **Portable** - Works anywhere Go runs
+- **No Docker** - No external dependencies
+- **Parallel-Safe** - Free port allocation prevents conflicts
+
+## Other Libraries with Test Harnesses
+
+- **Redis**: `github.com/alicebob/miniredis` - Pure Go in-memory Redis
+- **NATS**: `github.com/nats-io/nats-server/v2/test` (shown above)
+- **PostgreSQL**: `github.com/jackc/pgx/v5/pgxpool` with pgx mock
+- **MongoDB**: `github.com/tryvium-travels/memongo` - In-memory MongoDB
+
+## Key Takeaways
+
+1. **Check for official test harnesses first** - Many popular libraries provide them
+2. **Use free port allocation** - Prevents conflicts in parallel tests
+3. **Clean shutdown** - Always call `Shutdown()` in teardown
+4. **Reusable infrastructure** - Same setup for unit, integration, and system tests

skills/testing/examples/system-patterns.md

@@ -0,0 +1,288 @@
+# System Test Patterns
+
+## Purpose
+
+System tests are black-box tests that verify the entire application works correctly from an external perspective. They test via CLI or API, simulating real user interactions.
+
+**Location**: `tests/` directory at project root (separate from package code)
+
+## Principles
+
+### Black Box Testing
+- Test only via public interfaces (CLI, API)
+- No access to internal packages
+- Simulate real user behavior
+- Test critical workflows end-to-end
+
+### Independence in Go
+- Strive for pure Go tests (no Docker required)
+- Use in-memory mocks from `testutils`
+- Binary dependencies when needed
+- Avoid docker-compose in CI
+
+## CLI Testing Patterns
+
+### Pattern 1: Simple Command Execution
+
+```go
+// tests/cli_test.go
+package tests
+
+import (
+    "os/exec"
+    "strings"
+    "testing"
+
+    "github.com/stretchr/testify/require"
+)
+
+func TestCLI_Version(t *testing.T) {
+    // Execute CLI command
+    cmd := exec.Command("./myapp", "version")
+    output, err := cmd.CombinedOutput()
+
+    require.NoError(t, err)
+    require.Contains(t, string(output), "myapp version")
+}
+
+func TestCLI_Help(t *testing.T) {
+    cmd := exec.Command("./myapp", "--help")
+    output, err := cmd.CombinedOutput()
+
+    require.NoError(t, err)
+    require.Contains(t, string(output), "Usage:")
+}
+```
+
+### Pattern 2: CLI with In-Memory Mocks
+
+```go
+// tests/cli_metrics_test.go
+package tests
+
+import (
+    "context"
+    "os/exec"
+    "testing"
+
+    "github.com/stretchr/testify/require"
+    "myproject/internal/testutils"
+)
+
+func TestCLI_MetricsIngest(t *testing.T) {
+    // Start Victoria Metrics (Level 2 - binary)
+    vmServer, err := testutils.RunVictoriaMetricsServer()
+    require.NoError(t, err)
+    defer vmServer.Shutdown()
+
+    // Test CLI against real Victoria Metrics
+    cmd := exec.Command("./myapp", "ingest",
+        "--metrics-url", vmServer.WriteURL(),
+        "--metric-name", "cli_test_metric",
+        "--value", "100")
+
+    output, err := cmd.CombinedOutput()
+    require.NoError(t, err)
+    require.Contains(t, string(output), "Metric ingested successfully")
+
+    // Verify with helpers
+    vmServer.ForceFlush(context.Background())
+    results, err := testutils.QueryVictoriaMetrics(vmServer.QueryURL(), "cli_test_metric")
+    require.NoError(t, err)
+    require.Len(t, results, 1)
+}
+```
+
+### Pattern 3: CLI with File System
+
+```go
+// tests/cli_config_test.go
+package tests
+
+import (
+    "os"
+    "os/exec"
+    "path/filepath"
+    "testing"
+
+    "github.com/stretchr/testify/require"
+)
+
+func TestCLI_ConfigFile(t *testing.T) {
+    // Create temp directory
+    tempDir := t.TempDir()
+    configPath := filepath.Join(tempDir, "config.yaml")
+
+    // Write config file
+    configContent := `
+server:
+  port: 8080
+  host: localhost
+`
+    err := os.WriteFile(configPath, []byte(configContent), 0644)
+    require.NoError(t, err)
+
+    // Test CLI with config file
+    cmd := exec.Command("./myapp", "start", "--config", configPath, "--dry-run")
+    output, err := cmd.CombinedOutput()
+
+    require.NoError(t, err)
+    require.Contains(t, string(output), "Server would start on localhost:8080")
+}
+```
+
+## API Testing Patterns
+
+### Pattern 1: HTTP API with In-Memory Mocks
+
+```go
+// tests/api_test.go
+package tests
+
+import (
+    "bytes"
+    "encoding/json"
+    "io"
+    "net/http"
+    "os/exec"
+    "testing"
+    "time"
+
+    "github.com/stretchr/testify/require"
+    "myproject/internal/testutils"
+)
+
+func TestAPI_UserWorkflow(t *testing.T) {
+    // Start in-memory NATS (Level 1)
+    natsServer, err := testutils.RunNATsServer()
+    require.NoError(t, err)
+    defer natsServer.Shutdown()
+
+    natsAddr := "nats://" + natsServer.Addr().String()
+
+    // Start API server
+    cmd := exec.Command("./myapp", "serve",
+        "--port", "0", // Random free port
+        "--nats-url", natsAddr)
+
+    // Start in background
+    err = cmd.Start()
+    require.NoError(t, err)
+    defer cmd.Process.Kill()
+
+    // Wait for API to be ready
+    time.Sleep(500 * time.Millisecond)
+
+    // Get actual port (from logs or endpoint)
+    apiURL := "http://localhost:8080" // Or parse from logs
+
+    // Test API workflow
+    // 1. Create user
+    createReq := map[string]string{
+        "name":  "Alice",
+        "email": "alice@example.com",
+    }
+    body, _ := json.Marshal(createReq)
+
+    resp, err := http.Post(apiURL+"/users", "application/json", bytes.NewBuffer(body))
+    require.NoError(t, err)
+    defer resp.Body.Close()
+    require.Equal(t, http.StatusCreated, resp.StatusCode)
+
+    // Parse response
+    var createResp map[string]string
+    json.NewDecoder(resp.Body).Decode(&createResp)
+    userID := createResp["id"]
+
+    // 2. Retrieve user
+    resp, err = http.Get(apiURL + "/users/" + userID)
+    require.NoError(t, err)
+    defer resp.Body.Close()
+    require.Equal(t, http.StatusOK, resp.StatusCode)
+
+    var user map[string]string
+    json.NewDecoder(resp.Body).Decode(&user)
+    require.Equal(t, "Alice", user["name"])
+}
+```
+
+## Architecture for Independence
+
+### Dependency Injection Pattern
+
+Design your application to accept dependency URLs:
+
+```go
+// cmd/myapp/main.go
+func main() {
+    // Allow overriding dependencies via flags
+    natsURL := flag.String("nats-url", "nats://localhost:4222", "NATS server URL")
+    metricsURL := flag.String("metrics-url", "http://localhost:8428", "Metrics server URL")
+    flag.Parse()
+
+    // Use provided URLs (allows in-memory mocks in tests)
+    app := app.New(*natsURL, *metricsURL)
+    app.Run()
+}
+```
+
+### Test with In-Memory Dependencies
+
+```go
+// tests/app_test.go
+func TestApp_WithMocks(t *testing.T) {
+    // Start all mocks
+    natsServer, _ := testutils.RunNATsServer()
+    defer natsServer.Shutdown()
+
+    vmServer, _ := testutils.RunVictoriaMetricsServer()
+    defer vmServer.Shutdown()
+
+    // Test app with mocked dependencies (pure Go, no Docker!)
+    cmd := exec.Command("./myapp", "serve",
+        "--nats-url", "nats://"+natsServer.Addr().String(),
+        "--metrics-url", vmServer.WriteURL())
+
+    // ... test application
+}
+```
+
+## Running System Tests
+
+```bash
+# Build application first
+go build -o myapp ./cmd/myapp
+
+# Run system tests
+go test -v ./tests/...
+
+# With coverage
+go test -v -coverprofile=coverage.out ./tests/...
+
+# Specific test
+go test -v ./tests/... -run TestCLI_MetricsIngest
+```
+
+## Best Practices
+
+### DO:
+- Test via CLI/API only (black box)
+- Use in-memory mocks from testutils
+- Test critical end-to-end workflows
+- Build binary before running tests
+- Use temp directories for file operations
+
+### DON'T:
+- Don't import internal packages
+- Don't test every edge case (that's unit/integration tests)
+- Don't require Docker in CI
+- Don't use sleep for timing (use polling/channels)
+- Don't skip cleanup
+
+## Key Takeaways
+
+1. **Black box only** - Test via public interfaces
+2. **Independent in Go** - No Docker required
+3. **Use testutils mocks** - Reuse infrastructure
+4. **Test critical paths** - Not every scenario
+5. **Fast execution** - Should run quickly in CI

skills/testing/examples/test-organization.md

@@ -0,0 +1,260 @@
+# Test Organization and File Structure
+
+## File Organization
+
+### Basic Structure
+
+```
+user/
+├── user.go
+├── user_test.go          # Unit tests for user.go
+├── service.go
+├── service_test.go       # Unit tests for service.go
+├── repository.go
+└── repository_test.go    # Unit tests for repository.go
+```
+
+### With Integration and System Tests
+
+```
+project/
+├── user/
+│   ├── user.go
+│   ├── user_test.go              # Unit tests (pkg_test)
+│   ├── service.go
+│   ├── service_test.go           # Unit tests (pkg_test)
+│   └── integration_test.go       # Integration tests with //go:build integration
+├── internal/
+│   └── testutils/                # Reusable test infrastructure
+│       ├── nats.go              # In-memory NATS server
+│       ├── victoria.go          # Victoria Metrics binary management
+│       └── httpserver/          # HTTP mock DSL
+│           ├── server.go
+│           └── server_test.go   # Test the infrastructure!
+└── tests/                        # System tests (black box)
+    ├── cli_test.go              # CLI testing via exec.Command
+    └── api_test.go              # API testing via HTTP client
+```
+
+## Package Naming
+
+### Use `pkg_test` for Unit Tests
+
+```go
+// ✅ External package - tests public API only
+package user_test
+
+import (
+    "testing"
+    "github.com/yourorg/project/user"
+)
+
+func TestService_CreateUser(t *testing.T) {
+    // Test through public API
+    svc, _ := user.NewUserService(repo, notifier)
+    err := svc.CreateUser(ctx, testUser)
+    // ...
+}
+```
+
+### Avoid Same Package Testing
+
+```go
+// ❌ Same package - can test private methods (don't do this)
+package user
+
+import "testing"
+
+func TestInternalValidation(t *testing.T) {
+    // Testing private function - bad practice
+    result := validateEmailInternal("test@example.com")
+    // ...
+}
+```
+
+## Build Tags for Integration Tests
+
+### Using Build Tags
+
+```go
+//go:build integration
+
+package user_test
+
+import (
+    "context"
+    "testing"
+    "myproject/internal/testutils"
+)
+
+func TestUserService_Integration(t *testing.T) {
+    // Integration test with real dependencies
+    natsServer, _ := testutils.RunNATsServer()
+    defer natsServer.Shutdown()
+
+    // Test with real NATS
+    // ...
+}
+```
+
+### Running Tests
+
+```bash
+# Run only unit tests (default - no build tags)
+go test ./...
+
+# Run unit + integration tests
+go test -tags=integration ./...
+
+# Run specific package integration tests
+go test -tags=integration ./user
+
+# Run system tests only
+go test ./tests/...
+
+# Run all tests
+go test -tags=integration ./...
+```
+
+## Makefile/Taskfile Integration
+
+### Taskfile.yml Example
+
+```yaml
+version: '3'
+
+tasks:
+  test:
+    desc: Run unit tests
+    cmds:
+      - go test -v -race ./...
+
+  test:integration:
+    desc: Run integration tests
+    cmds:
+      - go test -v -race -tags=integration ./...
+
+  test:system:
+    desc: Run system tests
+    cmds:
+      - go test -v -race ./tests/...
+
+  test:all:
+    desc: Run all tests
+    cmds:
+      - task: test:integration
+      - task: test:system
+
+  test:coverage:
+    desc: Run tests with coverage
+    cmds:
+      - go test -v -race -coverprofile=coverage.out ./...
+      - go tool cover -html=coverage.out -o coverage.html
+```
+
+### Makefile Example
+
+```makefile
+.PHONY: test test-integration test-system test-all coverage
+
+test:
+	go test -v -race ./...
+
+test-integration:
+	go test -v -race -tags=integration ./...
+
+test-system:
+	go test -v -race ./tests/...
+
+test-all: test-integration test-system
+
+coverage:
+	go test -v -race -coverprofile=coverage.out ./...
+	go tool cover -html=coverage.out -o coverage.html
+```
+
+## Test File Naming
+
+### Unit Tests
+- `*_test.go` - Standard test files
+- Located next to the code being tested
+- Use `pkg_test` package name
+
+### Integration Tests
+- `integration_test.go` or `*_integration_test.go`
+- Use `//go:build integration` tag
+- Can be in same directory or separate `integration/` folder
+- Use `pkg_test` package name
+
+### System Tests
+- `*_test.go` in `tests/` directory at project root
+- No build tags needed (separate directory)
+- Use `tests` or `main_test` package name
+
+## CI/CD Integration
+
+### GitHub Actions Example
+
+```yaml
+name: Tests
+
+on: [push, pull_request]
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+      - name: Run unit tests
+        run: go test -v -race ./...
+
+  integration-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+      - name: Run integration tests
+        run: go test -v -race -tags=integration ./...
+
+  system-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+      - name: Build application
+        run: go build -o myapp ./cmd/myapp
+      - name: Run system tests
+        run: go test -v -race ./tests/...
+```
+
+## testutils Package Structure
+
+```
+internal/testutils/
+├── nats.go              # NATS in-memory server helpers
+├── victoria.go          # Victoria Metrics binary management
+├── prometheus.go        # Prometheus payload helpers
+├── grpc_client_mock.go  # gRPC client mock with DSL
+├── jrpc_server_mock.go  # JSON-RPC server mock with DSL
+└── httpserver/          # HTTP mock server with DSL
+    ├── server.go
+    ├── server_test.go   # Test the infrastructure!
+    ├── dsl.go
+    └── README.md
+```
+
+## Key Principles
+
+1. **Co-locate unit tests** - Next to the code being tested
+2. **Use pkg_test package** - Forces public API testing
+3. **Build tags for integration** - Keep unit tests fast by default
+4. **Separate system tests** - In `tests/` directory
+5. **Test your test infrastructure** - Treat testutils as production code
+6. **Reusable infrastructure** - Share across all test levels

skills/testing/examples/victoria-metrics.md

@@ -0,0 +1,566 @@
+# Victoria Metrics Binary Test Server
+
+## When to Use This Example
+
+Use this when:
+- Testing Prometheus Remote Write integrations
+- Need real Victoria Metrics for testing metrics ingestion
+- Testing PromQL queries
+- Want production-like behavior without Docker
+- Testing metrics pipelines end-to-end
+
+**Dependency Level**: Level 2 (Binary) - Standalone executable via `exec.Command`
+
+**Why Binary Instead of In-Memory:**
+- Victoria Metrics is complex; reimplementing as in-memory mock isn't practical
+- Need real PromQL engine behavior
+- Need actual data persistence and querying
+- Binary startup is fast (< 1 second) and requires no Docker
+
+## Implementation
+
+### Victoria Server Infrastructure
+
+This example shows how to download, manage, and run Victoria Metrics binary for testing:
+
+```go
+// internal/testutils/victoria.go
+package testutils
+
+import (
+    "context"
+    "errors"
+    "fmt"
+    "net/http"
+    "os"
+    "os/exec"
+    "path/filepath"
+    "runtime"
+    "sync"
+    "time"
+
+    "github.com/projectdiscovery/freeport"
+)
+
+const (
+    DefaultVictoriaMetricsVersion = "v1.128.0"
+    VictoriaMetricsVersionEnvVar  = "TEST_VICTORIA_METRICS_VERSION"
+)
+
+var (
+    ErrVictoriaMetricsNotHealthy = errors.New("victoria metrics did not become healthy")
+    ErrDownloadFailed            = errors.New("download failed")
+
+    // binaryDownloadMu protects concurrent downloads (prevent race conditions)
+    binaryDownloadMu sync.Mutex
+)
+
+// VictoriaServer represents a running Victoria Metrics test instance
+type VictoriaServer struct {
+    cmd          *exec.Cmd
+    port         int
+    dataPath     string
+    writeURL     string
+    queryURL     string
+    version      string
+    binaryPath   string
+    shutdownOnce sync.Once
+    shutdownErr  error
+}
+
+// WriteURL returns the URL for writing metrics (Prometheus Remote Write endpoint)
+func (vs *VictoriaServer) WriteURL() string {
+    return vs.writeURL
+}
+
+// QueryURL returns the URL for querying metrics (Prometheus-compatible query endpoint)
+func (vs *VictoriaServer) QueryURL() string {
+    return vs.queryURL
+}
+
+// Port returns the port Victoria Metrics is listening on
+func (vs *VictoriaServer) Port() int {
+    return vs.port
+}
+
+// ForceFlush forces Victoria Metrics to flush buffered samples from memory to disk,
+// making them immediately queryable. This is useful for testing to avoid waiting
+// for the automatic flush cycle.
+func (vs *VictoriaServer) ForceFlush(ctx context.Context) error {
+    url := fmt.Sprintf("http://localhost:%d/internal/force_flush", vs.port)
+    req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+    if err != nil {
+        return fmt.Errorf("failed to create force flush request: %w", err)
+    }
+
+    resp, err := http.DefaultClient.Do(req)
+    if err != nil {
+        return fmt.Errorf("failed to force flush: %w", err)
+    }
+    defer resp.Body.Close()
+
+    if resp.StatusCode != http.StatusOK {
+        return fmt.Errorf("force flush failed: status %d", resp.StatusCode)
+    }
+
+    return nil
+}
+
+// Shutdown stops Victoria Metrics and cleans up resources.
+// Safe to call multiple times (idempotent).
+func (vs *VictoriaServer) Shutdown() error {
+    vs.shutdownOnce.Do(func() {
+        if vs.cmd == nil || vs.cmd.Process == nil {
+            return
+        }
+
+        // Send interrupt signal for graceful shutdown
+        if err := vs.cmd.Process.Signal(os.Interrupt); err != nil {
+            vs.shutdownErr = err
+            return
+        }
+
+        // Wait for process to exit (with timeout)
+        done := make(chan error, 1)
+        go func() {
+            done <- vs.cmd.Wait()
+        }()
+
+        select {
+        case <-time.After(5 * time.Second):
+            vs.cmd.Process.Kill()
+            vs.shutdownErr = errors.New("shutdown timeout")
+        case err := <-done:
+            if err != nil && err.Error() != "signal: interrupt" {
+                vs.shutdownErr = err
+            }
+        }
+
+        // Cleanup data directory
+        if vs.dataPath != "" {
+            os.RemoveAll(vs.dataPath)
+        }
+    })
+    return vs.shutdownErr
+}
+
+// RunVictoriaMetricsServer starts a Victoria Metrics instance for testing.
+// It downloads the binary if needed, starts the server, and waits for it to be healthy.
+func RunVictoriaMetricsServer() (*VictoriaServer, error) {
+    version := getVictoriaMetricsVersion()
+
+    // Ensure binary exists (downloads if missing)
+    binaryPath, err := ensureVictoriaBinary(version)
+    if err != nil {
+        return nil, err
+    }
+
+    // Get free port (prevents conflicts in parallel tests)
+    freePort, err := freeport.GetFreePort("127.0.0.1", freeport.TCP)
+    if err != nil {
+        return nil, fmt.Errorf("failed to get free port: %w", err)
+    }
+    port := freePort.Port
+
+    // Create temporary data directory
+    dataPath, err := os.MkdirTemp("", "victoria-metrics-test-*")
+    if err != nil {
+        return nil, fmt.Errorf("failed to create temp directory: %w", err)
+    }
+
+    // Start Victoria Metrics
+    cmd := exec.Command(
+        binaryPath,
+        fmt.Sprintf("-httpListenAddr=:%d", port),
+        "-storageDataPath="+dataPath,
+        "-retentionPeriod=1d",
+        "-inmemoryDataFlushInterval=1ms", // Force immediate data flush for testing
+    )
+
+    cmd.Stdout = os.Stdout
+    cmd.Stderr = os.Stderr
+
+    if err := cmd.Start(); err != nil {
+        os.RemoveAll(dataPath)
+        return nil, fmt.Errorf("failed to start victoria metrics: %w", err)
+    }
+
+    baseURL := fmt.Sprintf("http://localhost:%d", port)
+    server := &VictoriaServer{
+        cmd:        cmd,
+        port:       port,
+        dataPath:   dataPath,
+        writeURL:   baseURL + "/api/v1/write",
+        queryURL:   baseURL + "/api/v1/query",
+        version:    version,
+        binaryPath: binaryPath,
+    }
+
+    // Wait for server to become healthy
+    if err := waitForHealth(baseURL); err != nil {
+        server.Shutdown()
+        return nil, err
+    }
+
+    return server, nil
+}
+
+func getVictoriaMetricsVersion() string {
+    if version := os.Getenv(VictoriaMetricsVersionEnvVar); version != "" {
+        return version
+    }
+    return DefaultVictoriaMetricsVersion
+}
+
+// ensureVictoriaBinary ensures the Victoria Metrics binary exists, downloading if necessary.
+// Thread-safe with double-check locking to prevent race conditions.
+func ensureVictoriaBinary(version string) (string, error) {
+    binaryName := fmt.Sprintf("victoria-metrics-%s-%s-%s", version, runtime.GOOS, getVMArch())
+    binaryPath := filepath.Join(".bin", binaryName)
+
+    // Quick check without lock (optimization)
+    if _, err := os.Stat(binaryPath); err == nil {
+        return binaryPath, nil
+    }
+
+    // Acquire lock to prevent concurrent downloads
+    binaryDownloadMu.Lock()
+    defer binaryDownloadMu.Unlock()
+
+    // Double-check after acquiring lock (another goroutine might have downloaded it)
+    if _, err := os.Stat(binaryPath); err == nil {
+        return binaryPath, nil
+    }
+
+    // Create .bin directory
+    if err := os.MkdirAll(".bin", 0755); err != nil {
+        return "", fmt.Errorf("failed to create .bin directory: %w", err)
+    }
+
+    // Download to temporary location with unique name
+    tempPath := fmt.Sprintf("%s.tmp.%d", binaryPath, os.Getpid())
+    defer os.Remove(tempPath)
+
+    downloadURL := fmt.Sprintf(
+        "https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/%s/victoria-metrics-%s-%s-%s.tar.gz",
+        version, runtime.GOOS, getVMArch(), version,
+    )
+
+    if err := downloadAndExtract(downloadURL, tempPath); err != nil {
+        return "", fmt.Errorf("failed to download: %w", err)
+    }
+
+    if err := os.Chmod(tempPath, 0755); err != nil {
+        return "", fmt.Errorf("failed to make binary executable: %w", err)
+    }
+
+    // Atomic rename - only one goroutine succeeds if multiple try
+    if err := os.Rename(tempPath, binaryPath); err != nil {
+        // If rename fails, check if another goroutine succeeded
+        if _, statErr := os.Stat(binaryPath); statErr == nil {
+            return binaryPath, nil // Another goroutine won the race
+        }
+        return "", fmt.Errorf("failed to rename binary: %w", err)
+    }
+
+    return binaryPath, nil
+}
+
+func getVMArch() string {
+    switch runtime.GOARCH {
+    case "amd64":
+        return "amd64"
+    case "arm64":
+        return "arm64"
+    default:
+        return runtime.GOARCH
+    }
+}
+
+func waitForHealth(baseURL string) error {
+    healthURL := baseURL + "/health"
+    maxRetries := 30
+    retryInterval := time.Second
+
+    ctx := context.Background()
+    for range maxRetries {
+        req, err := http.NewRequestWithContext(ctx, http.MethodGet, healthURL, nil)
+        if err != nil {
+            time.Sleep(retryInterval)
+            continue
+        }
+
+        resp, err := http.DefaultClient.Do(req)
+        if err == nil {
+            statusOK := resp.StatusCode == http.StatusOK
+            resp.Body.Close()
+            if statusOK {
+                return nil
+            }
+        }
+
+        time.Sleep(retryInterval)
+    }
+
+    return ErrVictoriaMetricsNotHealthy
+}
+```
+
+### Helper Functions for Prometheus/Victoria Metrics Testing
+
+Add practical helpers that make tests clear and maintainable:
+
+```go
+// internal/testutils/prometheus.go
+package testutils
+
+import (
+    "context"
+    "encoding/json"
+    "errors"
+    "fmt"
+    "io"
+    "net/http"
+    "net/url"
+    "testing"
+    "time"
+
+    "github.com/gogo/protobuf/proto"
+    "github.com/golang/snappy"
+    "github.com/prometheus/prometheus/prompb"
+    "github.com/stretchr/testify/require"
+)
+
+var (
+    ErrQueryFailed     = errors.New("victoria metrics query failed")
+    ErrQueryNonSuccess = errors.New("query returned non-success status")
+)
+
+// CreatePrometheusPayload creates a valid Prometheus Remote Write payload
+// with a sample metric. The payload is protobuf-encoded and snappy-compressed,
+// ready to be sent to Victoria Metrics' /api/v1/write endpoint.
+func CreatePrometheusPayload(metricName string, value float64, labels map[string]string) ([]byte, error) {
+    // Create timestamp (current time in milliseconds)
+    timestampMs := time.Now().UnixMilli()
+
+    // Build label pairs
+    labelPairs := make([]prompb.Label, 0, len(labels)+1)
+    labelPairs = append(labelPairs, prompb.Label{
+        Name:  "__name__",
+        Value: metricName,
+    })
+    for name, val := range labels {
+        labelPairs = append(labelPairs, prompb.Label{
+            Name:  name,
+            Value: val,
+        })
+    }
+
+    // Create a single time series with one sample
+    timeseries := []prompb.TimeSeries{
+        {
+            Labels: labelPairs,
+            Samples: []prompb.Sample{
+                {
+                    Value:     value,
+                    Timestamp: timestampMs,
+                },
+            },
+        },
+    }
+
+    // Create WriteRequest
+    writeRequest := &prompb.WriteRequest{
+        Timeseries: timeseries,
+    }
+
+    // Marshal to protobuf
+    data, err := proto.Marshal(writeRequest)
+    if err != nil {
+        return nil, fmt.Errorf("failed to marshal protobuf: %w", err)
+    }
+
+    // Compress with snappy
+    compressed := snappy.Encode(nil, data)
+
+    return compressed, nil
+}
+
+// VMQueryResult represents a single result from a Victoria Metrics query.
+type VMQueryResult struct {
+    Metric map[string]string // label name -> label value
+    Value  []any             // [timestamp, value_string]
+}
+
+// VMQueryResponse represents the full Victoria Metrics API response.
+type VMQueryResponse struct {
+    Status string `json:"status"`
+    Data   struct {
+        ResultType string          `json:"result_type"`
+        Result     []VMQueryResult `json:"result"`
+    } `json:"data"`
+}
+
+// QueryVictoriaMetrics executes a PromQL query against Victoria Metrics.
+// The query is performed via the /api/v1/query endpoint with time buffer
+// for clock skew and delayed indexing.
+func QueryVictoriaMetrics(queryURL, query string) ([]VMQueryResult, error) {
+    // Query with current time + 1 minute to catch any clock skew or delayed indexing
+    currentTime := time.Now().Add(1 * time.Minute)
+    fullURL := fmt.Sprintf("%s?query=%s&time=%d", queryURL, url.QueryEscape(query), currentTime.Unix())
+
+    // Execute HTTP request
+    ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+    defer cancel()
+
+    req, err := http.NewRequestWithContext(ctx, http.MethodGet, fullURL, nil)
+    if err != nil {
+        return nil, fmt.Errorf("failed to create request: %w", err)
+    }
+
+    resp, err := http.DefaultClient.Do(req)
+    if err != nil {
+        return nil, fmt.Errorf("failed to execute query: %w", err)
+    }
+    defer resp.Body.Close()
+
+    if resp.StatusCode != http.StatusOK {
+        return nil, fmt.Errorf("%w: %s", ErrQueryFailed, resp.Status)
+    }
+
+    // Read response body
+    bodyBytes, err := io.ReadAll(resp.Body)
+    if err != nil {
+        return nil, fmt.Errorf("failed to read response body: %w", err)
+    }
+
+    // Parse JSON response
+    var queryResp VMQueryResponse
+    if err := json.Unmarshal(bodyBytes, &queryResp); err != nil {
+        return nil, fmt.Errorf("failed to decode response: %w", err)
+    }
+
+    if queryResp.Status != "success" {
+        return nil, fmt.Errorf("%w: %s", ErrQueryNonSuccess, queryResp.Status)
+    }
+
+    return queryResp.Data.Result, nil
+}
+
+// AssertLabelExists checks if at least one result contains a label with the given name and value.
+// Fails the test if the label is not found.
+func AssertLabelExists(t *testing.T, results []VMQueryResult, labelName, labelValue string) {
+    t.Helper()
+
+    for _, result := range results {
+        if val, exists := result.Metric[labelName]; exists && val == labelValue {
+            return // Found it!
+        }
+    }
+
+    // Label not found - fail with helpful message
+    require.Fail(t, "Label not found",
+        "Expected to find label %s=%s in query results, but it was not present",
+        labelName, labelValue)
+}
+```
+
+## Usage Examples
+
+### Integration Test
+
+```go
+// internal/api/stats/prometheus_ingest_test.go
+func TestPrometheusIngest_WithVictoriaMetrics(t *testing.T) {
+    // Start real Victoria Metrics server (Level 2)
+    vmServer, err := testutils.RunVictoriaMetricsServer()
+    require.NoError(t, err)
+    defer vmServer.Shutdown()
+
+    // Create valid Prometheus payload using helper
+    payload, err := testutils.CreatePrometheusPayload("test_metric", 42.0, map[string]string{
+        "service": "api",
+        "env":     "test",
+    })
+    require.NoError(t, err)
+
+    // Send to Victoria Metrics
+    req := httptest.NewRequest(http.MethodPost, vmServer.WriteURL(), bytes.NewBuffer(payload))
+    req.Header.Set("Content-Type", "application/x-protobuf")
+    req.Header.Set("Content-Encoding", "snappy")
+
+    resp, err := http.DefaultClient.Do(req)
+    require.NoError(t, err)
+    defer resp.Body.Close()
+
+    assert.Equal(t, http.StatusNoContent, resp.StatusCode)
+
+    // Force flush to make data queryable immediately
+    err = vmServer.ForceFlush(context.Background())
+    require.NoError(t, err)
+
+    // Query using helper
+    results, err := testutils.QueryVictoriaMetrics(vmServer.QueryURL(), `test_metric{service="api"}`)
+    require.NoError(t, err)
+    require.Len(t, results, 1)
+
+    // Assert using helper
+    testutils.AssertLabelExists(t, results, "env", "test")
+}
+```
+
+### System Test
+
+```go
+// tests/prometheus_ingestion_test.go
+func TestE2E_PrometheusIngestion(t *testing.T) {
+    // Same Victoria Metrics infrastructure!
+    vmServer, err := testutils.RunVictoriaMetricsServer()
+    require.NoError(t, err)
+    defer vmServer.Shutdown()
+
+    // Test CLI against real Victoria Metrics
+    cmd := exec.Command("./myapp", "ingest",
+        "--metrics-url", vmServer.WriteURL(),
+        "--metric-name", "cli_test_metric",
+        "--value", "100")
+
+    output, err := cmd.CombinedOutput()
+    require.NoError(t, err)
+    assert.Contains(t, string(output), "Metric ingested successfully")
+
+    // Verify with helpers
+    vmServer.ForceFlush(context.Background())
+    results, err := testutils.QueryVictoriaMetrics(vmServer.QueryURL(), "cli_test_metric")
+    require.NoError(t, err)
+    require.Len(t, results, 1)
+}
+```
+
+## Key Features
+
+- **Binary download with OS/arch detection** - Works on macOS/Linux, amd64/arm64
+- **Thread-safe download** - Mutex + double-check locking prevents race conditions
+- **Free port allocation** - Prevents conflicts in parallel tests
+- **Idempotent shutdown** - Safe to call multiple times with `sync.Once`
+- **Resource cleanup** - Proper temp directory and process cleanup
+- **Helper functions** - `ForceFlush()` for immediate data availability
+- **Prometheus helpers** - Create payloads, query, assert on results
+
+## Benefits
+
+- **Production-like testing** - Testing against REAL Victoria Metrics, not mocks
+- **Reusable** - Same `testutils` infrastructure for unit, integration, and system tests
+- **Readable** - Helper functions make tests read like documentation
+- **No Docker** - No Docker required, works in any environment
+- **Fast** - Binary starts in < 1 second
+- **Portable** - Works anywhere Go runs
+- **Maintainable** - Changes to test infrastructure are centralized
+
+## Key Takeaways
+
+1. **Binary level is good for complex services** - When in-memory is too complex
+2. **Download management is critical** - Thread-safe, cached, version-controlled
+3. **Helper functions make tests readable** - DSL for common operations
+4. **Reuse across test levels** - Same infrastructure for unit, integration, system
+5. **Force flush is essential** - Make data immediately queryable in tests