## Phase 8: Enterprise Extractor Improvements ✅ - 14 security extractors (TLS, JWT, SQL injection, XSS, etc.) - 10 framework-specific extractors (Spring, Django, Rails, etc.) - Config file security detection (YAML, TOML) ## Phase 9: Autonomous Extractor Generation ✅ - Shadow mode executor with TP/FP tracking - Graduation pipeline with confidence thresholds - Auto-rollback on regression detection - Cross-project pattern syncing ## UAT Suite Complete (14 scripts, 90 tests) - test-core-detection.sh (6 tests) - test-declarative-extractors.sh (5 tests) - test-domain-frameworks.sh (5 tests) - test-domain-unreal.sh (3 tests) - test-llm-extraction.sh (6 tests) - test-eval-harness.sh (5 tests) - test-cross-language.sh (3 tests) - test-precommit-performance.sh (4 tests) - test-output-formats.sh (8 tests) - test-drift-detection.sh (6 tests) - test-exit-codes.sh (12 tests) + 3 more scripts ## Other Changes - Updated roadmap to mark Phase 8-9 complete - Added .gitignore entries for build artifacts - Updated pre-commit: 800 line limit, exclude tests/data/cmd Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
209 lines
5.0 KiB
Go
209 lines
5.0 KiB
Go
package httpkit
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"math/rand"
|
|
"net/http"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
defaultTimeout = 30 * time.Second
|
|
maxBackoff = 30 * time.Second
|
|
)
|
|
|
|
// Config configures an httpkit Client.
|
|
type Config struct {
|
|
// Timeout for each HTTP request attempt. Default: 30s.
|
|
// Ignored if HTTPClient is provided.
|
|
Timeout time.Duration
|
|
|
|
// MaxRetries is the number of retry attempts after the initial request.
|
|
// 0 means no retries (single attempt). Default: 0.
|
|
MaxRetries int
|
|
|
|
// Headers are applied to every request. Per-request headers in Request
|
|
// override these on key collision.
|
|
Headers map[string]string
|
|
|
|
// Logger for retry and error events. Default: slog.Default().
|
|
Logger *slog.Logger
|
|
|
|
// HTTPClient allows injecting a pre-configured http.Client.
|
|
// When set, Timeout is ignored. Use this to share connection pools
|
|
// across multiple httpkit Clients.
|
|
HTTPClient *http.Client
|
|
}
|
|
|
|
// Client is a configured HTTP client with retry and error classification.
|
|
type Client struct {
|
|
http *http.Client
|
|
maxRetries int
|
|
headers map[string]string
|
|
logger *slog.Logger
|
|
}
|
|
|
|
// NewClient creates a new httpkit Client.
|
|
func NewClient(cfg Config) *Client {
|
|
httpClient := cfg.HTTPClient
|
|
if httpClient == nil {
|
|
timeout := cfg.Timeout
|
|
if timeout == 0 {
|
|
timeout = defaultTimeout
|
|
}
|
|
httpClient = &http.Client{Timeout: timeout}
|
|
}
|
|
|
|
logger := cfg.Logger
|
|
if logger == nil {
|
|
logger = slog.Default()
|
|
}
|
|
|
|
return &Client{
|
|
http: httpClient,
|
|
maxRetries: cfg.MaxRetries,
|
|
headers: cfg.Headers,
|
|
logger: logger,
|
|
}
|
|
}
|
|
|
|
// Request describes an HTTP request to execute.
|
|
type Request struct {
|
|
Method string
|
|
URL string
|
|
Body any // JSON-marshaled if non-nil
|
|
Headers map[string]string // Per-request headers (override defaults)
|
|
}
|
|
|
|
// Do executes an HTTP request with automatic retry on retryable errors.
|
|
//
|
|
// On 2xx: returns (responseBody, nil).
|
|
// On non-2xx after retries: returns (nil, *APIError).
|
|
// On transport error after retries: returns (nil, error).
|
|
//
|
|
// The retry loop uses exponential backoff with full jitter and respects
|
|
// context cancellation between attempts.
|
|
func (c *Client) Do(ctx context.Context, req Request) ([]byte, error) {
|
|
var lastErr error
|
|
|
|
for attempt := 0; attempt <= c.maxRetries; attempt++ {
|
|
if attempt > 0 {
|
|
if !IsRetryable(lastErr) {
|
|
return nil, lastErr
|
|
}
|
|
|
|
backoff := c.backoff(attempt)
|
|
c.logger.Info("retrying request",
|
|
"attempt", attempt,
|
|
"max_retries", c.maxRetries,
|
|
"url", req.URL,
|
|
"backoff_ms", backoff.Milliseconds(),
|
|
)
|
|
|
|
select {
|
|
case <-time.After(backoff):
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
}
|
|
|
|
body, err := c.doOnce(ctx, req)
|
|
if err == nil {
|
|
return body, nil
|
|
}
|
|
lastErr = err
|
|
|
|
// Don't retry if parent context is done.
|
|
if ctx.Err() != nil {
|
|
return nil, lastErr
|
|
}
|
|
}
|
|
|
|
if c.maxRetries > 0 {
|
|
return nil, fmt.Errorf("failed after %d attempts: %w", c.maxRetries+1, lastErr)
|
|
}
|
|
return nil, lastErr
|
|
}
|
|
|
|
// doOnce executes a single HTTP request attempt.
|
|
func (c *Client) doOnce(ctx context.Context, req Request) ([]byte, error) {
|
|
var bodyReader io.Reader
|
|
if req.Body != nil {
|
|
data, err := json.Marshal(req.Body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("marshal request body: %w", err)
|
|
}
|
|
bodyReader = bytes.NewReader(data)
|
|
}
|
|
|
|
httpReq, err := http.NewRequestWithContext(ctx, req.Method, req.URL, bodyReader)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("create request: %w", err)
|
|
}
|
|
|
|
// Apply default headers first.
|
|
for k, v := range c.headers {
|
|
httpReq.Header.Set(k, v)
|
|
}
|
|
// Apply per-request headers (override defaults).
|
|
for k, v := range req.Headers {
|
|
httpReq.Header.Set(k, v)
|
|
}
|
|
// Auto-set Content-Type for JSON bodies.
|
|
if req.Body != nil && httpReq.Header.Get("Content-Type") == "" {
|
|
httpReq.Header.Set("Content-Type", "application/json")
|
|
}
|
|
|
|
resp, err := c.http.Do(httpReq)
|
|
if err != nil {
|
|
if errors.Is(err, context.DeadlineExceeded) || isTimeoutError(err) {
|
|
return nil, fmt.Errorf("%w: %v", ErrTimeout, err)
|
|
}
|
|
return nil, err
|
|
}
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
resp.Body.Close()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read response body: %w", err)
|
|
}
|
|
|
|
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
|
return body, nil
|
|
}
|
|
|
|
return nil, &APIError{
|
|
StatusCode: resp.StatusCode,
|
|
Body: string(body),
|
|
sentinel: ClassifyStatus(resp.StatusCode),
|
|
}
|
|
}
|
|
|
|
// backoff returns the sleep duration for a retry attempt using
|
|
// exponential backoff with full jitter.
|
|
func (c *Client) backoff(attempt int) time.Duration {
|
|
base := time.Second
|
|
ceiling := min(base*time.Duration(1<<uint(attempt)), maxBackoff) // 2s, 4s, 8s, ...
|
|
// Full jitter: uniform random in [0, ceiling).
|
|
return time.Duration(rand.Int63n(int64(ceiling)))
|
|
}
|
|
|
|
// isTimeoutError checks for net.Error timeout, which http.Client uses
|
|
// for its Timeout field.
|
|
func isTimeoutError(err error) bool {
|
|
type timeouter interface {
|
|
Timeout() bool
|
|
}
|
|
var te timeouter
|
|
if errors.As(err, &te) {
|
|
return te.Timeout()
|
|
}
|
|
return false
|
|
}
|