Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
Fixes issues from code review of resilience implementation:
- Wire saga system in main.go (SagaRepository, SagaExecutor, SagaHandler)
- Fix CompletedSteps() to include skipped steps for dependency resolution
- Fix reverse loop bug in saga compensation (use standard swap pattern)
- Add circuit breaker state change callbacks for Prometheus metrics
Phase 1 (Build Resilience):
- Add failure:retry to all component Kaniko build steps
- Add preflight registry health check before builds
- Add services-deployed sync point to decouple docs from critical path
Phase 2 (API Resilience):
- Add pipeline retry endpoint (POST /projects/{id}/pipelines/{number}/retry)
- Wire circuit breakers with metrics callbacks
- Add /health/circuits endpoint for circuit breaker status
Phase 3 (Saga Engine):
- Full domain model (Saga, SagaStep, RetryPolicy, BackoffType)
- PostgreSQL saga repository with CRUD and step management
- Saga executor with retry, compensation, skip step support
- Saga API handlers with CRUD and control operations
Phase 4 (Observability):
- Add saga metrics (total, step_duration, retry, circuit_breaker_state)
- Add logging fields (saga_id, saga_name, step_name)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
307 lines
8.0 KiB
Go
307 lines
8.0 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/orchard9/rdev/internal/domain"
|
|
)
|
|
|
|
// mockOperationRepo implements port.OperationRepository for testing.
|
|
type mockOperationRepo struct {
|
|
operations []*domain.Operation
|
|
err error
|
|
}
|
|
|
|
func (m *mockOperationRepo) Create(_ context.Context, _ *domain.Operation) error {
|
|
return nil
|
|
}
|
|
|
|
func (m *mockOperationRepo) Update(_ context.Context, _ *domain.Operation) error {
|
|
return nil
|
|
}
|
|
|
|
func (m *mockOperationRepo) Get(_ context.Context, _ string) (*domain.Operation, error) {
|
|
return nil, domain.ErrOperationNotFound
|
|
}
|
|
|
|
func (m *mockOperationRepo) GetByCommitSHA(_ context.Context, _, _ string) (*domain.Operation, error) {
|
|
return nil, domain.ErrOperationNotFound
|
|
}
|
|
|
|
func (m *mockOperationRepo) List(_ context.Context, filter domain.OperationFilters) ([]*domain.Operation, error) {
|
|
if m.err != nil {
|
|
return nil, m.err
|
|
}
|
|
var result []*domain.Operation
|
|
for _, op := range m.operations {
|
|
if filter.ProjectID != "" && op.ProjectID != filter.ProjectID {
|
|
continue
|
|
}
|
|
result = append(result, op)
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (m *mockOperationRepo) AddStep(_ context.Context, _ string, _ domain.OperationStep) error {
|
|
return nil
|
|
}
|
|
|
|
func (m *mockOperationRepo) UpdateStep(_ context.Context, _ string, _ domain.OperationStep) error {
|
|
return nil
|
|
}
|
|
|
|
func (m *mockOperationRepo) Complete(_ context.Context, _ string, _ domain.OperationStatus, _ map[string]any, _, _ string) error {
|
|
return nil
|
|
}
|
|
|
|
func (m *mockOperationRepo) SetCommitSHA(_ context.Context, _, _ string) error {
|
|
return nil
|
|
}
|
|
|
|
func (m *mockOperationRepo) SetTriggeredBy(_ context.Context, _, _ string) error {
|
|
return nil
|
|
}
|
|
|
|
func (m *mockOperationRepo) DeleteOlderThan(_ context.Context, _ time.Time) (int64, error) {
|
|
return 0, nil
|
|
}
|
|
|
|
// mockRegistryChecker implements port.RegistryChecker for testing.
|
|
type mockRegistryChecker struct {
|
|
status domain.RegistryStatus
|
|
}
|
|
|
|
func (m *mockRegistryChecker) Check(_ context.Context) domain.RegistryStatus {
|
|
return m.status
|
|
}
|
|
|
|
// mockCIProvider implements port.CIProvider for testing.
|
|
type mockCIProvider struct {
|
|
pipelines []*domain.CIPipeline
|
|
steps *domain.CIPipelineSteps
|
|
err error
|
|
}
|
|
|
|
func (m *mockCIProvider) ActivateRepo(_ context.Context, _, _, _ string) (*domain.CIRepo, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (m *mockCIProvider) DeactivateRepo(_ context.Context, _, _ string) error {
|
|
return nil
|
|
}
|
|
|
|
func (m *mockCIProvider) GetRepo(_ context.Context, _, _ string) (*domain.CIRepo, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (m *mockCIProvider) ListRepos(_ context.Context) ([]*domain.CIRepo, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (m *mockCIProvider) AddSecret(_ context.Context, _, _ string, _ domain.CISecret) error {
|
|
return nil
|
|
}
|
|
|
|
func (m *mockCIProvider) DeleteSecret(_ context.Context, _, _, _ string) error {
|
|
return nil
|
|
}
|
|
|
|
func (m *mockCIProvider) ListPipelines(_ context.Context, _, _ string) ([]*domain.CIPipeline, error) {
|
|
if m.err != nil {
|
|
return nil, m.err
|
|
}
|
|
return m.pipelines, nil
|
|
}
|
|
|
|
func (m *mockCIProvider) GetPipeline(_ context.Context, _, _ string, _ int64) (*domain.CIPipeline, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (m *mockCIProvider) GetPipelineSteps(_ context.Context, _, _ string, _ int64) (*domain.CIPipelineSteps, error) {
|
|
if m.steps != nil {
|
|
return m.steps, nil
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func (m *mockCIProvider) TriggerBuild(_ context.Context, _, _, _ string) (int64, error) {
|
|
return 0, nil
|
|
}
|
|
|
|
func (m *mockCIProvider) RetryPipeline(_ context.Context, _, _ string, _ int64) (*domain.CIPipeline, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func TestDiagnosticsService_GetDiagnostics_Healthy(t *testing.T) {
|
|
opRepo := &mockOperationRepo{
|
|
operations: []*domain.Operation{
|
|
{
|
|
ID: "op-1",
|
|
ProjectID: "test-project",
|
|
Type: domain.OperationTypeBuild,
|
|
Status: domain.OperationStatusCompleted,
|
|
StartedAt: time.Now().Add(-1 * time.Hour),
|
|
},
|
|
},
|
|
}
|
|
registry := &mockRegistryChecker{
|
|
status: domain.RegistryStatus{
|
|
Healthy: true,
|
|
URL: "https://registry.example.com",
|
|
Latency: "10ms",
|
|
LastChecked: time.Now(),
|
|
},
|
|
}
|
|
ci := &mockCIProvider{
|
|
pipelines: []*domain.CIPipeline{
|
|
{
|
|
Number: 42,
|
|
Status: "success",
|
|
Branch: "main",
|
|
Commit: "abc123",
|
|
},
|
|
},
|
|
}
|
|
|
|
svc := NewDiagnosticsService(opRepo, registry, ci, DiagnosticsServiceConfig{
|
|
DefaultGitOwner: "test-org",
|
|
})
|
|
|
|
diag, err := svc.GetDiagnostics(context.Background(), "test-project")
|
|
if err != nil {
|
|
t.Fatalf("GetDiagnostics() error = %v", err)
|
|
}
|
|
|
|
if diag.ProjectID != "test-project" {
|
|
t.Errorf("ProjectID = %q, want %q", diag.ProjectID, "test-project")
|
|
}
|
|
if diag.Summary != domain.DiagnosticsSummaryHealthy {
|
|
t.Errorf("Summary = %q, want %q", diag.Summary, domain.DiagnosticsSummaryHealthy)
|
|
}
|
|
if len(diag.Issues) != 0 {
|
|
t.Errorf("Issues count = %d, want 0", len(diag.Issues))
|
|
}
|
|
if len(diag.RecentOperations) != 1 {
|
|
t.Errorf("RecentOperations count = %d, want 1", len(diag.RecentOperations))
|
|
}
|
|
if diag.Registry == nil {
|
|
t.Error("Registry is nil, want non-nil")
|
|
}
|
|
if diag.CI == nil || !diag.CI.Available {
|
|
t.Error("CI not available")
|
|
}
|
|
}
|
|
|
|
func TestDiagnosticsService_GetDiagnostics_Unhealthy(t *testing.T) {
|
|
opRepo := &mockOperationRepo{
|
|
operations: []*domain.Operation{
|
|
{
|
|
ID: "op-1",
|
|
ProjectID: "test-project",
|
|
Type: domain.OperationTypeBuild,
|
|
Status: domain.OperationStatusFailed,
|
|
StartedAt: time.Now().Add(-1 * time.Hour),
|
|
Error: "deployment failed",
|
|
},
|
|
},
|
|
}
|
|
registry := &mockRegistryChecker{
|
|
status: domain.RegistryStatus{
|
|
Healthy: false,
|
|
URL: "https://registry.example.com",
|
|
Error: "connection refused",
|
|
LastChecked: time.Now(),
|
|
},
|
|
}
|
|
ci := &mockCIProvider{
|
|
pipelines: []*domain.CIPipeline{
|
|
{
|
|
Number: 43,
|
|
Status: "failure",
|
|
Branch: "main",
|
|
Commit: "def456",
|
|
Finished: time.Now().Add(-30 * time.Minute),
|
|
},
|
|
},
|
|
steps: &domain.CIPipelineSteps{
|
|
PipelineNumber: 43,
|
|
URL: "https://ci.example.com/build/43",
|
|
Steps: []domain.CIPipelineStep{
|
|
{
|
|
Name: "test",
|
|
Status: "failure",
|
|
Error: "tests failed",
|
|
Log: "FAIL: TestSomething",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
svc := NewDiagnosticsService(opRepo, registry, ci, DiagnosticsServiceConfig{
|
|
DefaultGitOwner: "test-org",
|
|
})
|
|
|
|
diag, err := svc.GetDiagnostics(context.Background(), "test-project")
|
|
if err != nil {
|
|
t.Fatalf("GetDiagnostics() error = %v", err)
|
|
}
|
|
|
|
if diag.Summary != domain.DiagnosticsSummaryUnhealthy {
|
|
t.Errorf("Summary = %q, want %q", diag.Summary, domain.DiagnosticsSummaryUnhealthy)
|
|
}
|
|
|
|
// Should have 3 issues: failed operation, unhealthy registry, failed CI
|
|
if len(diag.Issues) < 3 {
|
|
t.Errorf("Issues count = %d, want at least 3", len(diag.Issues))
|
|
}
|
|
|
|
// Check CI failure details
|
|
if diag.CI == nil || diag.CI.LastFailure == nil {
|
|
t.Fatal("CI.LastFailure is nil")
|
|
}
|
|
if diag.CI.LastFailure.FailedStep != "test" {
|
|
t.Errorf("FailedStep = %q, want %q", diag.CI.LastFailure.FailedStep, "test")
|
|
}
|
|
}
|
|
|
|
func TestDiagnosticsService_GetDiagnostics_Degraded(t *testing.T) {
|
|
opRepo := &mockOperationRepo{
|
|
operations: []*domain.Operation{
|
|
{
|
|
ID: "op-1",
|
|
ProjectID: "test-project",
|
|
Type: domain.OperationTypeBuild,
|
|
Status: domain.OperationStatusCompleted,
|
|
StartedAt: time.Now().Add(-1 * time.Hour),
|
|
},
|
|
},
|
|
}
|
|
registry := &mockRegistryChecker{
|
|
status: domain.RegistryStatus{
|
|
Healthy: true,
|
|
URL: "https://registry.example.com",
|
|
LastChecked: time.Now(),
|
|
},
|
|
}
|
|
|
|
// No CI provider - should produce a warning but not error
|
|
svc := NewDiagnosticsService(opRepo, registry, nil, DiagnosticsServiceConfig{
|
|
DefaultGitOwner: "test-org",
|
|
})
|
|
|
|
diag, err := svc.GetDiagnostics(context.Background(), "test-project")
|
|
if err != nil {
|
|
t.Fatalf("GetDiagnostics() error = %v", err)
|
|
}
|
|
|
|
// Without CI, status should still be healthy (CI unavailable is not an issue)
|
|
if diag.Summary != domain.DiagnosticsSummaryHealthy {
|
|
t.Errorf("Summary = %q, want %q", diag.Summary, domain.DiagnosticsSummaryHealthy)
|
|
}
|
|
if diag.CI == nil || diag.CI.Available {
|
|
t.Error("CI should be not available when no provider")
|
|
}
|
|
}
|