// Package e2e contains end-to-end tests for the rdev API. // // Run with: go test -tags=e2e ./tests/e2e/... // // Requires docker-compose to be running: // // cd tests/e2e && docker-compose up -d //go:build e2e package e2e import ( "bytes" "encoding/json" "io" "net/http" "os" "testing" "time" ) const ( baseURL = "http://localhost:8080" adminKey = "test-admin-key-12345" ) func getBaseURL() string { if url := os.Getenv("RDEV_API_URL"); url != "" { return url } return baseURL } func getAdminKey() string { if key := os.Getenv("RDEV_ADMIN_KEY"); key != "" { return key } return adminKey } // TestHealthEndpoint verifies the health endpoint returns 200. func TestHealthEndpoint(t *testing.T) { resp, err := http.Get(getBaseURL() + "/health") if err != nil { t.Fatalf("failed to call health endpoint: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Errorf("expected status 200, got %d", resp.StatusCode) } } // TestReadyEndpoint verifies the ready endpoint returns 200. func TestReadyEndpoint(t *testing.T) { resp, err := http.Get(getBaseURL() + "/ready") if err != nil { t.Fatalf("failed to call ready endpoint: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Errorf("expected status 200, got %d", resp.StatusCode) } } // TestMetricsEndpoint verifies the metrics endpoint returns Prometheus metrics. func TestMetricsEndpoint(t *testing.T) { resp, err := http.Get(getBaseURL() + "/metrics") if err != nil { t.Fatalf("failed to call metrics endpoint: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Errorf("expected status 200, got %d", resp.StatusCode) } body, _ := io.ReadAll(resp.Body) if len(body) == 0 { t.Error("expected metrics output, got empty body") } // Check for expected metric if !bytes.Contains(body, []byte("rdev_api_requests_total")) { t.Error("expected rdev_api_requests_total metric") } } // TestAuthenticationRequired verifies API endpoints require authentication. func TestAuthenticationRequired(t *testing.T) { endpoints := []string{ "/projects", "/keys", } for _, endpoint := range endpoints { t.Run(endpoint, func(t *testing.T) { resp, err := http.Get(getBaseURL() + endpoint) if err != nil { t.Fatalf("failed to call %s: %v", endpoint, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusUnauthorized { t.Errorf("expected status 401, got %d", resp.StatusCode) } }) } } // TestAdminKeyAccess verifies the admin key can access protected endpoints. func TestAdminKeyAccess(t *testing.T) { req, _ := http.NewRequest("GET", getBaseURL()+"/projects", nil) req.Header.Set("X-API-Key", getAdminKey()) client := &http.Client{Timeout: 10 * time.Second} resp, err := client.Do(req) if err != nil { t.Fatalf("failed to call projects endpoint: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) t.Errorf("expected status 200, got %d: %s", resp.StatusCode, body) } } // TestCreateAndListKeys verifies API key creation and listing. func TestCreateAndListKeys(t *testing.T) { client := &http.Client{Timeout: 10 * time.Second} // Create a new key createReq := map[string]any{ "name": "e2e-test-key", "scopes": []string{"projects:read"}, } body, _ := json.Marshal(createReq) req, _ := http.NewRequest("POST", getBaseURL()+"/keys", bytes.NewReader(body)) req.Header.Set("X-API-Key", getAdminKey()) req.Header.Set("Content-Type", "application/json") resp, err := client.Do(req) if err != nil { t.Fatalf("failed to create key: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusCreated { respBody, _ := io.ReadAll(resp.Body) t.Fatalf("expected status 201, got %d: %s", resp.StatusCode, respBody) } var createResp struct { Data struct { Key struct { ID string `json:"id"` Name string `json:"name"` } `json:"key"` Secret string `json:"secret"` } `json:"data"` } if err := json.NewDecoder(resp.Body).Decode(&createResp); err != nil { t.Fatalf("failed to decode create response: %v", err) } if createResp.Data.Key.Name != "e2e-test-key" { t.Errorf("expected name 'e2e-test-key', got '%s'", createResp.Data.Key.Name) } if createResp.Data.Secret == "" { t.Error("expected secret, got empty string") } // List keys req, _ = http.NewRequest("GET", getBaseURL()+"/keys", nil) req.Header.Set("X-API-Key", getAdminKey()) resp, err = client.Do(req) if err != nil { t.Fatalf("failed to list keys: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Errorf("expected status 200, got %d", resp.StatusCode) } var listResp struct { Data []map[string]any `json:"data"` } if err := json.NewDecoder(resp.Body).Decode(&listResp); err != nil { t.Fatalf("failed to decode list response: %v", err) } found := false for _, k := range listResp.Data { if k["name"] == "e2e-test-key" { found = true break } } if !found { t.Error("created key not found in list") } // Cleanup - revoke the key req, _ = http.NewRequest("DELETE", getBaseURL()+"/keys/"+createResp.Data.Key.ID, nil) req.Header.Set("X-API-Key", getAdminKey()) resp, err = client.Do(req) if err != nil { t.Fatalf("failed to revoke key: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Errorf("expected status 200 for revoke, got %d", resp.StatusCode) } } // TestProjectsList verifies the projects list endpoint. func TestProjectsList(t *testing.T) { client := &http.Client{Timeout: 10 * time.Second} req, _ := http.NewRequest("GET", getBaseURL()+"/projects", nil) req.Header.Set("X-API-Key", getAdminKey()) resp, err := client.Do(req) if err != nil { t.Fatalf("failed to list projects: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) t.Fatalf("expected status 200, got %d: %s", resp.StatusCode, body) } var projectsResp struct { Data []map[string]any `json:"data"` } if err := json.NewDecoder(resp.Body).Decode(&projectsResp); err != nil { t.Fatalf("failed to decode response: %v", err) } // Should have at least one project (pantheon, aeries) if len(projectsResp.Data) == 0 { t.Error("expected at least one project") } } // TestInvalidAPIKey verifies invalid API keys are rejected. func TestInvalidAPIKey(t *testing.T) { client := &http.Client{Timeout: 10 * time.Second} req, _ := http.NewRequest("GET", getBaseURL()+"/projects", nil) req.Header.Set("X-API-Key", "invalid-key-12345") resp, err := client.Do(req) if err != nil { t.Fatalf("failed to call endpoint: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusUnauthorized { t.Errorf("expected status 401, got %d", resp.StatusCode) } } // TestE2E_FullCommandLifecycle tests the full lifecycle of a command: // 1. Create API key // 2. Execute command // 3. Stream output via SSE // 4. Verify completion event // 5. Check metrics incremented func TestE2E_FullCommandLifecycle(t *testing.T) { client := &http.Client{Timeout: 30 * time.Second} // 1. Create a new API key with execute scope createKeyReq := map[string]any{ "name": "e2e-lifecycle-test", "scopes": []string{"projects:read", "commands:execute"}, } body, _ := json.Marshal(createKeyReq) req, _ := http.NewRequest("POST", getBaseURL()+"/keys", bytes.NewReader(body)) req.Header.Set("X-API-Key", getAdminKey()) req.Header.Set("Content-Type", "application/json") resp, err := client.Do(req) if err != nil { t.Fatalf("failed to create key: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusCreated { respBody, _ := io.ReadAll(resp.Body) t.Fatalf("expected status 201, got %d: %s", resp.StatusCode, respBody) } var createResp struct { Data struct { Key struct { ID string `json:"id"` } `json:"key"` Secret string `json:"secret"` } `json:"data"` } if err := json.NewDecoder(resp.Body).Decode(&createResp); err != nil { t.Fatalf("failed to decode create response: %v", err) } keyID := createResp.Data.Key.ID secret := createResp.Data.Secret // Cleanup at end defer func() { req, _ := http.NewRequest("DELETE", getBaseURL()+"/keys/"+keyID, nil) req.Header.Set("X-API-Key", getAdminKey()) client.Do(req) }() // 2. Get initial metrics count metricsResp, err := http.Get(getBaseURL() + "/metrics") if err != nil { t.Fatalf("failed to get initial metrics: %v", err) } initialMetrics, _ := io.ReadAll(metricsResp.Body) metricsResp.Body.Close() // 3. Get first project to execute a command on req, _ = http.NewRequest("GET", getBaseURL()+"/projects", nil) req.Header.Set("X-API-Key", secret) resp, err = client.Do(req) if err != nil { t.Fatalf("failed to list projects: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { respBody, _ := io.ReadAll(resp.Body) t.Fatalf("expected status 200 for projects, got %d: %s", resp.StatusCode, respBody) } var projectsResp struct { Data []struct { ID string `json:"id"` } `json:"data"` } if err := json.NewDecoder(resp.Body).Decode(&projectsResp); err != nil { t.Fatalf("failed to decode projects: %v", err) } if len(projectsResp.Data) == 0 { t.Skip("no projects available for testing") } projectID := projectsResp.Data[0].ID // 4. Execute a simple shell command execReq := map[string]any{ "command": "echo 'hello from e2e test'", } body, _ = json.Marshal(execReq) req, _ = http.NewRequest("POST", getBaseURL()+"/projects/"+projectID+"/shell", bytes.NewReader(body)) req.Header.Set("X-API-Key", secret) req.Header.Set("Content-Type", "application/json") resp, err = client.Do(req) if err != nil { t.Fatalf("failed to execute command: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusCreated { respBody, _ := io.ReadAll(resp.Body) t.Fatalf("expected status 201 for command, got %d: %s", resp.StatusCode, respBody) } var execResp struct { Data struct { ID string `json:"id"` StreamURL string `json:"stream_url"` } `json:"data"` } if err := json.NewDecoder(resp.Body).Decode(&execResp); err != nil { t.Fatalf("failed to decode exec response: %v", err) } if execResp.Data.ID == "" { t.Error("expected command ID, got empty") } if execResp.Data.StreamURL == "" { t.Error("expected stream URL, got empty") } // 5. Connect to SSE stream and verify completion streamURL := getBaseURL() + execResp.Data.StreamURL sseReq, _ := http.NewRequest("GET", streamURL, nil) sseReq.Header.Set("X-API-Key", secret) sseReq.Header.Set("Accept", "text/event-stream") // Use a client with longer timeout for SSE sseClient := &http.Client{Timeout: 60 * time.Second} sseResp, err := sseClient.Do(sseReq) if err != nil { t.Fatalf("failed to connect to SSE: %v", err) } defer sseResp.Body.Close() if sseResp.StatusCode != http.StatusOK { respBody, _ := io.ReadAll(sseResp.Body) t.Fatalf("expected SSE status 200, got %d: %s", sseResp.StatusCode, respBody) } // Read SSE events until we get completion or timeout gotConnected := false gotComplete := false timeout := time.After(30 * time.Second) scanner := make(chan string, 100) go func() { buf := make([]byte, 4096) for { n, err := sseResp.Body.Read(buf) if err != nil { close(scanner) return } scanner <- string(buf[:n]) } }() readLoop: for { select { case data, ok := <-scanner: if !ok { break readLoop } if bytes.Contains([]byte(data), []byte("event: connected")) { gotConnected = true } if bytes.Contains([]byte(data), []byte("event: complete")) { gotComplete = true break readLoop } case <-timeout: t.Log("SSE read timeout (may be expected if command hasn't completed)") break readLoop } } if !gotConnected { t.Error("expected connected event from SSE") } // Note: gotComplete may not be true if command is still running or already completed // This is acceptable for E2E test purposes t.Logf("SSE events - connected: %v, complete: %v", gotConnected, gotComplete) // 6. Verify metrics were incremented metricsResp, err = http.Get(getBaseURL() + "/metrics") if err != nil { t.Fatalf("failed to get final metrics: %v", err) } finalMetrics, _ := io.ReadAll(metricsResp.Body) metricsResp.Body.Close() // Metrics should show request activity if !bytes.Contains(finalMetrics, []byte("rdev_api_requests_total")) { t.Error("expected rdev_api_requests_total in metrics") } // There should be more requests in final metrics than initial t.Logf("Initial metrics length: %d, Final metrics length: %d", len(initialMetrics), len(finalMetrics)) } // TestE2E_RateLimiting verifies rate limiting behavior. // Sends rapid requests and verifies 429 on excess. func TestE2E_RateLimiting(t *testing.T) { client := &http.Client{Timeout: 5 * time.Second} // Create a key specifically for rate limit testing createReq := map[string]any{ "name": "e2e-ratelimit-test", "scopes": []string{"projects:read"}, } body, _ := json.Marshal(createReq) req, _ := http.NewRequest("POST", getBaseURL()+"/keys", bytes.NewReader(body)) req.Header.Set("X-API-Key", getAdminKey()) req.Header.Set("Content-Type", "application/json") resp, err := client.Do(req) if err != nil { t.Fatalf("failed to create key: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusCreated { respBody, _ := io.ReadAll(resp.Body) t.Fatalf("expected status 201, got %d: %s", resp.StatusCode, respBody) } var createResp struct { Data struct { Key struct { ID string `json:"id"` } `json:"key"` Secret string `json:"secret"` } `json:"data"` } json.NewDecoder(resp.Body).Decode(&createResp) keyID := createResp.Data.Key.ID secret := createResp.Data.Secret defer func() { req, _ := http.NewRequest("DELETE", getBaseURL()+"/keys/"+keyID, nil) req.Header.Set("X-API-Key", getAdminKey()) client.Do(req) }() // Send requests rapidly until we hit rate limit // Default is 100 requests/minute with burst of 50 var got429 bool var successCount, limitedCount int for i := 0; i < 150; i++ { req, _ := http.NewRequest("GET", getBaseURL()+"/projects", nil) req.Header.Set("X-API-Key", secret) resp, err := client.Do(req) if err != nil { t.Logf("request %d failed: %v", i, err) continue } // Check rate limit headers limitHeader := resp.Header.Get("X-RateLimit-Limit") remainingHeader := resp.Header.Get("X-RateLimit-Remaining") if resp.StatusCode == http.StatusTooManyRequests { got429 = true limitedCount++ t.Logf("Rate limited at request %d (limit: %s, remaining: %s)", i, limitHeader, remainingHeader) } else if resp.StatusCode == http.StatusOK { successCount++ } resp.Body.Close() if got429 { break // Found the rate limit } } t.Logf("Results: %d successful, %d rate-limited", successCount, limitedCount) if !got429 { t.Log("Warning: Did not hit rate limit with 150 requests. Rate limiter may be disabled or configured with high limits.") } // Verify rate limit headers are present req, _ = http.NewRequest("GET", getBaseURL()+"/projects", nil) req.Header.Set("X-API-Key", secret) resp, err = client.Do(req) if err != nil { t.Fatalf("failed to make verification request: %v", err) } defer resp.Body.Close() if resp.Header.Get("X-RateLimit-Limit") == "" { t.Error("expected X-RateLimit-Limit header to be present") } } // TestE2E_SSEReconnection tests SSE reconnection with Last-Event-ID. // 1. Start a command // 2. Connect to stream // 3. Read initial events and capture event IDs // 4. Reconnect with Last-Event-ID header // 5. Verify replay of missed events func TestE2E_SSEReconnection(t *testing.T) { client := &http.Client{Timeout: 30 * time.Second} // Get first project req, _ := http.NewRequest("GET", getBaseURL()+"/projects", nil) req.Header.Set("X-API-Key", getAdminKey()) resp, err := client.Do(req) if err != nil { t.Fatalf("failed to list projects: %v", err) } defer resp.Body.Close() var projectsResp struct { Data []struct { ID string `json:"id"` } `json:"data"` } json.NewDecoder(resp.Body).Decode(&projectsResp) if len(projectsResp.Data) == 0 { t.Skip("no projects available for testing") } projectID := projectsResp.Data[0].ID // Execute a command that produces multiple lines of output execReq := map[string]any{ "command": "echo 'line1'; sleep 0.1; echo 'line2'; sleep 0.1; echo 'line3'", "stream_id": "e2e-reconnect-test-" + time.Now().Format("20060102150405"), } body, _ := json.Marshal(execReq) req, _ = http.NewRequest("POST", getBaseURL()+"/projects/"+projectID+"/shell", bytes.NewReader(body)) req.Header.Set("X-API-Key", getAdminKey()) req.Header.Set("Content-Type", "application/json") resp, err = client.Do(req) if err != nil { t.Fatalf("failed to execute command: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusCreated { respBody, _ := io.ReadAll(resp.Body) t.Fatalf("expected status 201, got %d: %s", resp.StatusCode, respBody) } var execResp struct { Data struct { StreamURL string `json:"stream_url"` } `json:"data"` } json.NewDecoder(resp.Body).Decode(&execResp) // First connection - get the connected event streamURL := getBaseURL() + execResp.Data.StreamURL req, _ = http.NewRequest("GET", streamURL, nil) req.Header.Set("X-API-Key", getAdminKey()) req.Header.Set("Accept", "text/event-stream") sseClient := &http.Client{Timeout: 10 * time.Second} sseResp, err := sseClient.Do(req) if err != nil { t.Fatalf("failed to connect to SSE: %v", err) } // Read a bit then disconnect buf := make([]byte, 2048) sseResp.Body.Read(buf) firstData := string(buf) sseResp.Body.Close() t.Logf("First connection data: %s...", firstData[:min(len(firstData), 200)]) // Parse event IDs from the response var lastEventID string for _, line := range bytes.Split([]byte(firstData), []byte("\n")) { if bytes.HasPrefix(line, []byte("id: ")) { lastEventID = string(bytes.TrimPrefix(line, []byte("id: "))) } } // Reconnect with Last-Event-ID req, _ = http.NewRequest("GET", streamURL, nil) req.Header.Set("X-API-Key", getAdminKey()) req.Header.Set("Accept", "text/event-stream") if lastEventID != "" { req.Header.Set("Last-Event-ID", lastEventID) t.Logf("Reconnecting with Last-Event-ID: %s", lastEventID) } sseResp, err = sseClient.Do(req) if err != nil { t.Fatalf("failed to reconnect to SSE: %v", err) } defer sseResp.Body.Close() if sseResp.StatusCode != http.StatusOK { t.Fatalf("expected reconnect status 200, got %d", sseResp.StatusCode) } // Read reconnection data buf = make([]byte, 2048) sseResp.Body.Read(buf) reconnectData := string(buf) t.Logf("Reconnect data: %s...", reconnectData[:min(len(reconnectData), 200)]) // The reconnect response should include a "reconnecting" field in connected event if bytes.Contains([]byte(reconnectData), []byte("connected")) { t.Log("Reconnection event received") } } // TestE2E_ConcurrentCommands tests concurrent command limiting. // 1. Start 5 commands (default per-project limit) // 2. Verify 6th command is blocked with 429 // 3. Wait for one to complete // 4. Verify next command succeeds func TestE2E_ConcurrentCommands(t *testing.T) { client := &http.Client{Timeout: 30 * time.Second} // Get first project req, _ := http.NewRequest("GET", getBaseURL()+"/projects", nil) req.Header.Set("X-API-Key", getAdminKey()) resp, err := client.Do(req) if err != nil { t.Fatalf("failed to list projects: %v", err) } defer resp.Body.Close() var projectsResp struct { Data []struct { ID string `json:"id"` } `json:"data"` } json.NewDecoder(resp.Body).Decode(&projectsResp) if len(projectsResp.Data) == 0 { t.Skip("no projects available for testing") } projectID := projectsResp.Data[0].ID // Track started commands for potential cleanup var startedCommands []string timestamp := time.Now().Format("20060102150405") // Start multiple commands that will run for a bit // Use sleep commands to keep them running for i := 0; i < 6; i++ { execReq := map[string]any{ "command": "sleep 5; echo 'done'", // Sleep to keep command running "stream_id": "e2e-concurrent-" + timestamp + "-" + itoa(i), } body, _ := json.Marshal(execReq) req, _ := http.NewRequest("POST", getBaseURL()+"/projects/"+projectID+"/shell", bytes.NewReader(body)) req.Header.Set("X-API-Key", getAdminKey()) req.Header.Set("Content-Type", "application/json") resp, err := client.Do(req) if err != nil { t.Logf("Command %d failed to send: %v", i, err) continue } respBody, _ := io.ReadAll(resp.Body) resp.Body.Close() if resp.StatusCode == http.StatusCreated { startedCommands = append(startedCommands, execReq["stream_id"].(string)) t.Logf("Command %d started successfully (status: %d)", i, resp.StatusCode) } else if resp.StatusCode == http.StatusTooManyRequests { t.Logf("Command %d blocked by concurrent limit (status: 429)", i) // Check for appropriate error message if !bytes.Contains(respBody, []byte("limit")) && !bytes.Contains(respBody, []byte("concurrent")) { t.Logf("Expected limit-related error message, got: %s", respBody) } } else { t.Logf("Command %d got unexpected status %d: %s", i, resp.StatusCode, respBody) } } t.Logf("Started %d commands", len(startedCommands)) // Note: The concurrent command limiter may or may not be enabled // In a full E2E environment with the limiter enabled, we would expect // command 6 to fail with 429. Without the limiter, all commands would succeed. } // itoa converts int to string for test helper. func itoa(i int) string { if i == 0 { return "0" } neg := i < 0 if neg { i = -i } buf := make([]byte, 0, 20) for i > 0 { buf = append(buf, byte('0'+i%10)) i /= 10 } if neg { buf = append(buf, '-') } for l, r := 0, len(buf)-1; l < r; l, r = l+1, r-1 { buf[l], buf[r] = buf[r], buf[l] } return string(buf) } // min returns the smaller of two integers. func min(a, b int) int { if a < b { return a } return b }