rdev/cmd/rdev-api/main.go

// Package main provides the entry point for the rdev API server.
package main

import (
	"context"
	"os"
	"strings"
	"time"

	citadeladapter "github.com/orchard9/rdev/internal/adapter/citadel"
	"github.com/orchard9/rdev/internal/adapter/cloudflare"
	"github.com/orchard9/rdev/internal/adapter/cockroach"
	"github.com/orchard9/rdev/internal/adapter/codeagent"
	"github.com/orchard9/rdev/internal/adapter/codeagent/claudecode"
	"github.com/orchard9/rdev/internal/adapter/codeagent/opencode"
	"github.com/orchard9/rdev/internal/adapter/deployer"
	gcsadapter "github.com/orchard9/rdev/internal/adapter/gcs"
	"github.com/orchard9/rdev/internal/adapter/gitea"
	"github.com/orchard9/rdev/internal/adapter/kubernetes"
	"github.com/orchard9/rdev/internal/adapter/memory"
	notifyadapter "github.com/orchard9/rdev/internal/adapter/notify"
	"github.com/orchard9/rdev/internal/adapter/postgres"
	redisadapter "github.com/orchard9/rdev/internal/adapter/redis"
	sdlcadapter "github.com/orchard9/rdev/internal/adapter/sdlc"
	"github.com/orchard9/rdev/internal/adapter/templates"
	"github.com/orchard9/rdev/internal/adapter/woodpecker"
	"github.com/orchard9/rdev/internal/adapter/zot"
	"github.com/orchard9/rdev/internal/auth"
	"github.com/orchard9/rdev/internal/db"
	"github.com/orchard9/rdev/internal/envutil"
	"github.com/orchard9/rdev/internal/handlers"
	"github.com/orchard9/rdev/internal/logging"
	"github.com/orchard9/rdev/internal/metrics"
	"github.com/orchard9/rdev/internal/middleware"
	"github.com/orchard9/rdev/internal/port"
	"github.com/orchard9/rdev/internal/service"
	"github.com/orchard9/rdev/internal/telemetry"
	"github.com/orchard9/rdev/internal/webhook"
	"github.com/orchard9/rdev/internal/worker"
	"github.com/orchard9/rdev/pkg/api"
)

// version is set via ldflags at build time: -ldflags="-X main.version=v0.8.0"
var version = "dev"

func main() {
	// Initialize structured logging from environment configuration
	logCfg := logging.ConfigFromEnv()
	appLogger := logging.New(logCfg)
	logging.SetDefault(appLogger)

	// Create slog.Logger for compatibility with components that haven't migrated yet
	logger := appLogger.Slog()

	// Initialize telemetry (OpenTelemetry)
	telCfg := telemetry.DefaultConfig()
	telCfg.Logger = logger
	tel, err := telemetry.New(context.Background(), telCfg)
	if err != nil {
		logger.Error("failed to initialize telemetry", "error", err)
		os.Exit(1)
	}

	// Load configuration from environment
	cfg := loadConfig()

	// Validate required security configuration
	if cfg.CredentialEncryptionKey == "" {
		logger.Warn("CREDENTIAL_ENCRYPTION_KEY not set - credential store will use insecure default",
			"hint", "Generate with: openssl rand -base64 32")
		// Use a deterministic fallback for development only
		cfg.CredentialEncryptionKey = "rdev-dev-key-not-for-production"
	}

	// Initialize database with auto-migrations
	database, err := db.New(db.Config{
		Host:     cfg.DBHost,
		Port:     cfg.DBPort,
		User:     cfg.DBUser,
		Password: cfg.DBPassword,
		Database: cfg.DBName,
		SSLMode:  cfg.DBSSLMode,
	}, logger)
	if err != nil {
		logger.Error("failed to connect to database", "error", err)
		os.Exit(1)
	}
	defer func() { _ = database.Close() }()

	// Initialize auth service (hexagonal: repo → service → auth wrapper)
	apiKeyRepo := postgres.NewAPIKeyRepository(database.DB)
	apiKeySvc := service.NewAPIKeyService(apiKeyRepo, cfg.AdminKey)
	authService := auth.NewService(apiKeySvc, cfg.AdminKey)

	// Initialize credential store (for infrastructure secrets)
	credentialStore := postgres.NewCredentialStore(database.DB, cfg.CredentialEncryptionKey)

	// Load infrastructure config from credential store (falls back to env vars)
	infraCfg := loadInfraConfig(context.Background(), credentialStore, cfg, logger)

	// Initialize Citadel client (optional - for log environment provisioning and audit shipping)
	var citadelClient *citadeladapter.Client
	if cfg.CitadelURL != "" && cfg.CitadelAPIKey != "" {
		citadelClient = citadeladapter.NewClient(citadeladapter.Config{
			URL:    cfg.CitadelURL,
			APIKey: cfg.CitadelAPIKey,
		}, logger)
		logger.Info("citadel client initialized", "url", cfg.CitadelURL)
	}

	// Create adapters (dependency injection)
	namespace := envutil.GetEnv("K8S_NAMESPACE", "rdev")

	// Initialize K8s client (falls back gracefully if unavailable)
	k8sClient := kubernetes.NewClientOrNil(kubernetes.ClientConfig{
		Namespace:  namespace,
		Kubeconfig: os.Getenv("KUBECONFIG"),
	})
	projectRepo := kubernetes.NewProjectRepositoryWithClient(namespace, k8sClient, logger)
	k8sExecutor := kubernetes.NewExecutor(namespace)
	streamPub := memory.NewStreamPublisher()
	if k8sClient != nil {
		if err := projectRepo.StartWatching(context.Background()); err != nil {
			logger.Warn("failed to start project watcher", "error", err)
		}
	}

	var auditLogger port.AuditLogger
	var auditShipper *citadeladapter.AuditShipper
	pgAuditLogger := postgres.NewAuditLogger(database.DB)
	if citadelClient != nil && cfg.CitadelPlatformTenantID != "" {
		auditShipper = citadeladapter.NewAuditShipper(pgAuditLogger, citadelClient, cfg.CitadelPlatformTenantID, logger)
		auditLogger = auditShipper
		logger.Info("audit logger wrapped with citadel shipper", "tenant_id", cfg.CitadelPlatformTenantID)
	} else {
		auditLogger = pgAuditLogger
	}
	rateLimiter := postgres.NewRateLimiter(database.DB)
	stopRateLimitCleanup := rateLimiter.StartCleanupWorker(context.Background(), 5*time.Minute)
	commandQueue := postgres.NewCommandQueueRepository(database.DB)
	workQueueRepo := postgres.NewWorkQueueRepository(database.DB)

	webhookRepo := postgres.NewWebhookRepository(database.DB)
	webhookDispatcher := webhook.NewDispatcher(webhookRepo, &webhook.DispatcherConfig{
		WorkerCount:  10,
		MaxRetries:   3,
		Timeout:      30 * time.Second,
		RetryBackoff: 5 * time.Second,
		Logger:       logger,
	})
	if err := webhookDispatcher.Start(); err != nil {
		logger.Error("failed to start webhook dispatcher", "error", err)
		os.Exit(1)
	}

	// Infrastructure adapters (optional - only if configured)
	var giteaClient *gitea.Client
	if infraCfg.GiteaToken != "" && infraCfg.GiteaURL != "" {
		var err error
		giteaClient, err = gitea.NewClient(infraCfg.GiteaURL, infraCfg.GiteaToken, infraCfg.GiteaDefaultOrg)
		if err != nil {
			logger.Warn("failed to initialize gitea client", "error", err)
		}
	}
	var dnsClient *cloudflare.Client
	if infraCfg.CloudflareToken != "" && infraCfg.CloudflareZoneID != "" {
		dnsClient = cloudflare.NewClient(infraCfg.CloudflareToken, infraCfg.CloudflareZoneID, infraCfg.DefaultDomain)
	}

	var deployerAdapter *deployer.Deployer
	if k8sClient != nil {
		deployerAdapter = deployer.NewDeployer(k8sClient, deployer.Config{
			Namespace:       infraCfg.DeployNamespace,
			IngressClass:    "traefik",
			TLSIssuer:       infraCfg.DeployTLSIssuer,
			DefaultDomain:   infraCfg.DefaultDomain,
			DefaultReplicas: 1,
		})
	}
	var woodpeckerClient *woodpecker.Client
	if infraCfg.WoodpeckerURL != "" && infraCfg.WoodpeckerAPIToken != "" {
		var err error
		woodpeckerClient, err = woodpecker.NewClient(infraCfg.WoodpeckerURL, infraCfg.WoodpeckerAPIToken, woodpecker.WithLogger(logger))
		if err != nil {
			logger.Warn("failed to initialize woodpecker client", "error", err)
		}
	}
	var templateProvider *templates.Provider
	if infraCfg.GiteaToken != "" && infraCfg.GiteaURL != "" {
		templateProvider = templates.NewProvider(infraCfg.GiteaURL, infraCfg.GiteaToken, logger)
	}

	// Initialize database provisioner (optional - for project database isolation)
	var dbProvisioner port.DatabaseProvisioner
	if infraCfg.CRDBHost != "" {
		var err error
		dbProvisioner, err = cockroach.NewProvisioner(cockroach.Config{
			Host:    infraCfg.CRDBHost,
			Port:    infraCfg.CRDBPort,
			User:    infraCfg.CRDBUser,
			SSLMode: infraCfg.CRDBSSLMode,
		}, logger)
		if err != nil {
			logger.Warn("failed to initialize cockroachdb provisioner", "error", err)
		} else {
			logger.Info("cockroachdb provisioner initialized", "host", infraCfg.CRDBHost)
		}
	}

	// Initialize cache provisioner (optional - for project cache isolation via Redis ACLs)
	var cacheProvisioner port.CacheProvisioner
	if infraCfg.RedisHost != "" && infraCfg.RedisPassword != "" {
		var err error
		cacheProvisioner, err = redisadapter.NewProvisioner(redisadapter.Config{
			Host:     infraCfg.RedisHost,
			Port:     infraCfg.RedisPort,
			Password: infraCfg.RedisPassword,
		}, logger)
		if err != nil {
			logger.Warn("failed to initialize redis provisioner", "error", err)
		} else {
			logger.Info("redis provisioner initialized", "host", infraCfg.RedisHost)
		}
	}
	defer closeProvisioner(cacheProvisioner, "redis", logger)

	// Initialize storage provisioner (optional - for project storage via GCS)
	var storageProvisioner port.StorageProvisioner
	if infraCfg.GCSProjectID != "" {
		var err error
		storageProvisioner, err = gcsadapter.NewProvisioner(gcsadapter.Config{
			GoogleProjectID: infraCfg.GCSProjectID,
			CredentialsPath: infraCfg.GCSCredentialsPath,
			Location:        infraCfg.GCSLocation,
		}, logger)
		if err != nil {
			logger.Warn("failed to initialize gcs provisioner", "error", err)
		} else {
			logger.Info("gcs provisioner initialized", "project_id", infraCfg.GCSProjectID, "location", infraCfg.GCSLocation)
		}
	}
	defer closeProvisioner(storageProvisioner, "gcs", logger)

	// Initialize notify provisioner (optional - for project email delivery)
	var notifyProvisioner port.NotifyProvisioner
	if infraCfg.NotifyURL != "" && infraCfg.NotifyAdminKey != "" {
		np := notifyadapter.NewProvisioner(notifyadapter.Config{
			BaseURL:  infraCfg.NotifyURL,
			AdminKey: infraCfg.NotifyAdminKey,
			Host:     infraCfg.NotifyHost,
			From:     infraCfg.NotifyFrom,
		}, logger)
		if err := np.TestConnection(context.Background()); err != nil {
			logger.Warn("notify provisioner connection test failed, disabling", "error", err)
		} else {
			notifyProvisioner = np
			logger.Info("notify provisioner initialized", "url", infraCfg.NotifyURL, "host", infraCfg.NotifyHost)
		}
	}

	// Initialize registry client (for monitoring and image cleanup on project teardown)
	var registryClient *zot.Client
	if infraCfg.RegistryURL != "" {
		registryURL := infraCfg.RegistryURL
		// Ensure URL has protocol
		if !strings.HasPrefix(registryURL, "http") {
			registryURL = "https://" + registryURL
		}
		registryClient = zot.NewClient(registryURL).WithLogger(logger)
		logger.Info("registry client initialized", "url", registryURL)
	}

	// Initialize CodeAgent registry (multi-provider support)
	agentRegistry := codeagent.NewRegistry()

	// Register Claude Code adapter (default - always available)
	claudeCodeAdapter := claudecode.NewAdapter(namespace)
	agentRegistry.Register(claudeCodeAdapter)
	logger.Info("registered Claude Code agent", "provider", claudeCodeAdapter.Provider())

	// Register OpenCode adapter (optional - only if configured)
	if cfg.OpenCodeURL != "" {
		openCodeAdapter := opencode.NewAdapter(opencode.ClientConfig{
			BaseURL:  cfg.OpenCodeURL,
			Username: cfg.OpenCodeUsername,
			Password: cfg.OpenCodePassword,
			Timeout:  30 * time.Second,
		})
		agentRegistry.Register(openCodeAdapter)
		logger.Info("registered OpenCode agent", "provider", openCodeAdapter.Provider(), "url", cfg.OpenCodeURL)
	}

	// Create services
	projectService := service.NewProjectService(projectRepo, k8sExecutor, streamPub).
		WithAuditLogger(auditLogger).
		WithCommandQueue(commandQueue).
		WithWebhookDispatcher(webhookDispatcher).
		WithCodeAgentRegistry(agentRegistry)

	// Create work service (for worker pool task management)
	workService := service.NewWorkService(workQueueRepo).WithWebhookDispatcher(webhookDispatcher)

	// Create conversation service (for Foundary chat persistence)
	conversationRepo := postgres.NewConversationRepository(database.DB)
	conversationService := service.NewConversationService(conversationRepo)

	// Create blueprint service (for Foundary structured specs)
	blueprintRepo := postgres.NewBlueprintRepository(database.DB)
	blueprintService := service.NewBlueprintService(blueprintRepo)

	// Create architect service (for Foundary conversational orchestration)
	architectService := service.NewArchitectService(
		conversationService,
		blueprintService,
		agentRegistry,
		projectRepo,
		nil, // uses defaults: claudebox-0, rdev namespace
	)

	// Create question service (for Foundary structured questions)
	questionRepo := postgres.NewQuestionRepository(database.DB)
	questionService := service.NewQuestionService(questionRepo)

	// Initialize operation tracking (for debugging project failures)
	operationRepo := postgres.NewOperationRepository(database.DB)
	operationService := service.NewOperationService(operationRepo)

	// Initialize worker pool infrastructure
	workerRegistryRepo := postgres.NewWorkerRegistryRepository(database.DB)
	buildAuditRepo := postgres.NewBuildAuditRepository(database.DB)

	// Create worker service (manages worker lifecycle and task assignment)
	workerService := service.NewWorkerService(workerRegistryRepo, workQueueRepo).
		WithBuildAudit(buildAuditRepo)

	// Start worker health checker (marks stale workers offline)
	go workerService.StartHealthChecker(context.Background())

	// Create build service (orchestrates build submission and tracking)
	buildService := service.NewBuildService(workQueueRepo, buildAuditRepo)

	// Create verify service (orchestrates verify task submission and tracking)
	verifyService := service.NewVerifyService(workQueueRepo)

	// Create checkout repository and service (for sidecar development flow)
	checkoutRepo := postgres.NewCheckoutRepository(database.DB)
	var checkoutService *service.CheckoutService
	if giteaClient != nil {
		checkoutService = service.NewCheckoutService(
			checkoutRepo,
			giteaClient,
			projectRepo,
			service.CheckoutServiceConfig{
				GiteaURL:      infraCfg.GiteaURL,
				DefaultOwner:  infraCfg.GiteaDefaultOrg,
				DefaultExpiry: 24 * time.Hour,
			},
		).WithWorkQueue(workQueueRepo)
	}

	// Create session service (for interactive remote development)
	sessionRepo := postgres.NewSessionRepository(database.DB)
	var previewManager *kubernetes.PreviewManager
	if k8sClient != nil {
		previewManager = kubernetes.NewPreviewManager(k8sClient, kubernetes.PreviewConfig{
			Namespace:    namespace,
			IngressClass: "traefik",
			TLSIssuer:    infraCfg.DeployTLSIssuer,
		})
	}
	var sessionService *service.SessionService
	if checkoutService != nil && previewManager != nil {
		sessionService = service.NewSessionService(
			sessionRepo,
			checkoutService,
			projectRepo,
			previewManager,
			service.SessionServiceConfig{
				PreviewDomain: "preview.threesix.ai",
				DefaultExpiry: 24 * time.Hour,
			},
		)
	}

	// SDLC lifecycle management (kubectl exec into project pods)
	sdlcPodExec := kubernetes.NewSDLCExecutor(kubernetes.SDLCExecutorConfig{Namespace: namespace, Logger: logger})

	// Worker-based SDLC executor (for skeleton/monorepo projects without dedicated pods)
	workerSDLCExec := sdlcadapter.NewWorkerSDLCExecutor(sdlcadapter.WorkerSDLCExecutorConfig{
		WorkQueue: workQueueRepo,
		DB:        database.DB,
		Logger:    logger,
	})

	// Create SDLC service with dual executor support
	sdlcService := service.NewSDLCServiceWithWorker(
		sdlcPodExec,
		workerSDLCExec,
		projectRepo,
		database.DB,
	)

	// Pod git operations (shared between build executor and SDLC orchestrator)
	var podGitOps *worker.PodGitOperations
	if infraCfg.GiteaToken != "" {
		podGitOps = worker.NewPodGitOperations(worker.PodGitOperationsConfig{
			Namespace:  "rdev",
			GiteaToken: infraCfg.GiteaToken,
		})
	}

	// SDLC orchestrator (execute/resolve/commit via agents and git)
	var gitCommitter service.PodGitCommitter
	if podGitOps != nil {
		gitCommitter = &podGitCommitterAdapter{podGitOps: podGitOps}
	}
	sdlcOrchestrator := service.NewSDLCOrchestratorService(
		sdlcService,
		agentRegistry,
		gitCommitter,
		projectRepo,
		buildService,
		database.DB,
	)

	// Create app
	app := api.New("rdev-api",
		api.WithPort(cfg.Port),
		api.WithLogger(logger),
	)

	// Add telemetry middleware (first to capture all requests)
	app.Use(telemetry.Middleware(telCfg.ServiceName))

	// Add request logging middleware (enriches context with request ID and logs requests)
	logMiddlewareCfg := logging.DefaultMiddlewareConfig()
	logMiddlewareCfg.Logger = appLogger
	app.Use(logging.Middleware(logMiddlewareCfg))

	// Add metrics middleware (before auth to track all requests)
	app.Use(metrics.Middleware)

	// Add auth middleware (skips /health, /ready, /docs, /openapi.json, /metrics)
	app.Use(auth.Middleware(authService))

	// Add rate limiting middleware (after auth, so we have API key context)
	rateLimitCfg := middleware.DefaultRateLimitConfig()
	rateLimitCfg.Limiter = rateLimiter
	app.Use(middleware.RateLimitMiddleware(rateLimitCfg))

	// Register metrics endpoint (no auth required)
	app.Router().Handle("/metrics", metrics.Handler())

	// Initialize handlers
	projectsHandler := handlers.NewProjectsHandlerWithService(projectService)
	keysHandler := handlers.NewKeysHandler(authService)
	claudeConfigHandler := handlers.NewClaudeConfigHandlerWithService(projectService, projectRepo, k8sExecutor)
	auditHandler := handlers.NewAuditHandler(auditLogger)
	queueHandler := handlers.NewQueueHandler(commandQueue, projectRepo)
	webhookHandler := handlers.NewWebhookHandler(webhookRepo, projectRepo)
	workHandler := handlers.NewWorkHandler(workService)
	conversationsHandler := handlers.NewConversationsHandler(conversationService)
	blueprintsHandler := handlers.NewBlueprintsHandler(blueprintService)
	architectHandler := handlers.NewArchitectHandler(architectService)
	questionsHandler := handlers.NewQuestionsHandler(questionService)

	// Initialize domain and slug repositories
	projectDomainRepo := postgres.NewProjectDomainRepository(database.DB)
	slugGenerator := postgres.NewSlugRepository(database.DB)

	// Initialize project infrastructure service (orchestrates full project lifecycle)
	projectInfraService := service.NewProjectInfraService(
		database.DB,
		giteaClient,
		dnsClient,
		deployerAdapter,
		woodpeckerClient, // CI provider for auto-activating repos
		templateProvider, // Template provider for seeding repos
		projectDomainRepo,
		slugGenerator,
		service.ProjectInfraConfig{
			DefaultGitOwner: infraCfg.GiteaDefaultOrg,
			DefaultDomain:   infraCfg.DefaultDomain,
			ClusterIP:       infraCfg.ClusterIP,
		},
	)
	// Wire optional database, cache, storage, and registry provisioners
	if dbProvisioner != nil {
		projectInfraService = projectInfraService.WithDatabaseProvisioner(dbProvisioner)
	}
	if cacheProvisioner != nil {
		projectInfraService = projectInfraService.WithCacheProvisioner(cacheProvisioner)
	}
	if storageProvisioner != nil {
		projectInfraService = projectInfraService.WithStorageProvisioner(storageProvisioner)
	}
	if registryClient != nil {
		projectInfraService = projectInfraService.WithRegistryProvider(registryClient)
	}
	if citadelClient != nil {
		projectInfraService = projectInfraService.WithCitadelClient(citadelClient)
	}
	if notifyProvisioner != nil {
		projectInfraService = projectInfraService.WithNotifyProvisioner(notifyProvisioner)
	}

	// Create domain service adapter for infrastructure handler
	domainServiceAdapter := handlers.NewDomainServiceAdapter(projectInfraService)

	// Initialize infrastructure handler (for threesix.ai git/deploy/dns/ci)
	infraHandler := handlers.NewInfrastructureHandler(
		giteaClient,
		dnsClient,
		deployerAdapter,
		projectRepo,
		woodpeckerClient,
		domainServiceAdapter,
		handlers.InfrastructureConfig{
			DefaultGitOwner: infraCfg.GiteaDefaultOrg,
			DefaultDomain:   infraCfg.DefaultDomain,
			ClusterIP:       infraCfg.ClusterIP,
		},
	)

	// Initialize project management handler
	projectMgmtHandler := handlers.NewProjectManagementHandler(projectInfraService).
		SetOperationService(operationService)

	// Initialize component service and handler (for monorepo component management)
	var componentsHandler *handlers.ComponentsHandler
	if infraCfg.GiteaToken != "" && infraCfg.GiteaURL != "" && templateProvider != nil {
		bulkFileClient := gitea.NewBulkFileClient(infraCfg.GiteaURL, infraCfg.GiteaToken)
		componentService := service.NewComponentService(
			database.DB,
			templateProvider,
			bulkFileClient,
			deployerAdapter, // Creates initial K8s deployment for new components
			service.ComponentServiceConfig{
				DefaultGitOwner: infraCfg.GiteaDefaultOrg,
				RegistryURL:     infraCfg.RegistryURL,
			},
		).
			WithDatabaseProvisioner(dbProvisioner).
			WithCacheProvisioner(cacheProvisioner).
			WithStorageProvisioner(storageProvisioner).
			WithCredentialStore(credentialStore)
		componentsHandler = handlers.NewComponentsHandler(componentService).
			SetOperationService(operationService)
		logger.Info("component service initialized",
			"db_provisioner", dbProvisioner != nil,
			"cache_provisioner", cacheProvisioner != nil,
		)
	}

	// Initialize Woodpecker webhook handler (for CI/CD auto-deploy)
	woodpeckerHandler := handlers.NewWoodpeckerWebhookHandler(
		deployerAdapter,
		dnsClient,
		handlers.WoodpeckerWebhookConfig{
			WebhookSecret: infraCfg.WoodpeckerWebhookSecret,
			DefaultDomain: infraCfg.DefaultDomain,
			RegistryURL:   infraCfg.RegistryURL,
			ClusterIP:     infraCfg.ClusterIP,
			Logger:        logger,
		},
	).SetOperationService(operationService)

	// Initialize credentials handler (superadmin only)
	credentialsHandler := handlers.NewCredentialsHandler(credentialStore)

	// Initialize agents handler (for code agent management)
	agentsHandler := handlers.NewAgentsHandler(agentRegistry)

	// Initialize worker pool handlers
	workersHandler := handlers.NewWorkersHandler(workerService).WithWorkService(workService).WithWorkQueue(workQueueRepo)
	buildsHandler := handlers.NewBuildsHandler(buildService)
	createAndBuildHandler := handlers.NewCreateAndBuildHandler(projectInfraService, buildService)

	sdlcHandler := handlers.NewSDLCHandler(sdlcService)
	sdlcOrchestratorHandler := handlers.NewSDLCOrchestratorHandler(sdlcOrchestrator)

	// Initialize checkout handler (for sidecar development flow)
	var checkoutHandler *handlers.CheckoutHandler
	if checkoutService != nil {
		checkoutHandler = handlers.NewCheckoutHandler(checkoutService)
	}

	// Initialize sessions handler (for interactive remote development)
	var sessionsHandler *handlers.SessionsHandler
	if sessionService != nil {
		sessionsHandler = handlers.NewSessionsHandler(sessionService, k8sExecutor, streamPub)
	}

	// Initialize saga system (resilient workflow orchestration)
	sagaRepo := postgres.NewSagaRepository(database.DB)
	sagaExecutor := service.NewSagaExecutor(sagaRepo, logger)
	sagaHandler := handlers.NewSagaHandler(sagaRepo, sagaExecutor)

	// SDLC generate service (async artifact generation via work queue)
	apiBaseURL := envutil.GetEnv("RDEV_API_URL", "https://rdev.masq-ops.orchard9.ai")
	sdlcGenerateService := service.NewSDLCGenerateService(
		sdlcService,
		buildService,
		database.DB,
		service.SDLCGenerateServiceConfig{
			BaseURL: apiBaseURL,
		},
	)
	sdlcGenerateHandler := handlers.NewSDLCGenerateHandler(sdlcGenerateService)

	// SDLC callback service (handles build completion to update artifact status)
	sdlcCallbackService := service.NewSDLCCallbackService(sdlcService)
	sdlcCallbackHandler := handlers.NewSDLCCallbackHandler(sdlcCallbackService, cfg.InternalToken)

	// Initialize verify handler (for visual verification tasks)
	verifyHandler := handlers.NewVerifyHandler(verifyService, streamPub)

	// Initialize operations handler (for debugging project failures)
	operationsHandler := handlers.NewOperationsHandler(operationRepo)

	// Initialize diagnostics service (aggregates health data for debugging)
	diagnosticsService := service.NewDiagnosticsService(
		operationRepo,
		registryClient,
		woodpeckerClient,
		service.DiagnosticsServiceConfig{
			DefaultGitOwner: infraCfg.GiteaDefaultOrg,
		},
	)
	diagnosticsHandler := handlers.NewDiagnosticsHandler(diagnosticsService, projectRepo)

	// Initialize external health checker (background monitoring of registry, CI, git)
	var externalHealthChecker *worker.ExternalHealthChecker
	if registryClient != nil || woodpeckerClient != nil || giteaClient != nil {
		externalHealthChecker = worker.NewExternalHealthChecker(
			registryClient,
			woodpeckerClient,
			giteaClient,
			worker.ExternalHealthConfig{
				CheckInterval: 30 * time.Second,
			},
		)
		externalHealthChecker.Start()
		logger.Info("external health checker started")
	}

	// Override default health/ready endpoints with full dependency checks
	healthHandler := handlers.NewHealthHandler("rdev-api", database.DB, nil).
		WithAgentRegistry(agentRegistry)
	if registryClient != nil {
		healthHandler = healthHandler.WithRegistryChecker(registryClient)
	}
	if externalHealthChecker != nil {
		healthHandler = healthHandler.WithExternalHealthChecker(externalHealthChecker)
	}

	app.Router().Get("/health", healthHandler.Health)
	app.Router().Get("/ready", healthHandler.Ready)
	app.Router().Get("/health/circuits", healthHandler.Circuits)

	// Register routes
	projectsHandler.Mount(app.Router())
	keysHandler.Mount(app.Router())
	claudeConfigHandler.Mount(app.Router())
	auditHandler.Mount(app.Router())
	queueHandler.Mount(app.Router())
	webhookHandler.Mount(app.Router())
	workHandler.Mount(app.Router())
	conversationsHandler.Mount(app.Router())
	blueprintsHandler.Mount(app.Router())
	architectHandler.Mount(app.Router())
	questionsHandler.Mount(app.Router())
	infraHandler.Mount(app.Router())
	projectMgmtHandler.Mount(app.Router())
	if componentsHandler != nil {
		componentsHandler.Mount(app.Router())
	}
	woodpeckerHandler.Mount(app.Router())
	credentialsHandler.Mount(app.Router())
	agentsHandler.Mount(app.Router())
	workersHandler.Mount(app.Router())
	buildsHandler.Mount(app.Router())
	createAndBuildHandler.Mount(app.Router())
	operationsHandler.Mount(app.Router())
	diagnosticsHandler.Mount(app.Router())
	sdlcHandler.Mount(app.Router())
	sdlcOrchestratorHandler.Mount(app.Router())
	sdlcGenerateHandler.Mount(app.Router())
	sdlcCallbackHandler.Mount(app.Router())
	if checkoutHandler != nil {
		checkoutHandler.Mount(app.Router())
	}
	if sessionsHandler != nil {
		sessionsHandler.Mount(app.Router())
	}
	verifyHandler.Mount(app.Router())
	sagaHandler.Mount(app.Router())

	// Start queue processor worker (per-project command queue)
	queueProcessor := worker.NewQueueProcessor(
		commandQueue,
		k8sExecutor,
		projectRepo,
		streamPub,
		&worker.QueueProcessorConfig{
			PollPeriod: 5 * time.Second,
		},
	).WithWebhookDispatcher(webhookDispatcher)
	if err := queueProcessor.Start(); err != nil {
		logger.Error("failed to start queue processor", "error", err)
		os.Exit(1)
	}

	// Start work executor (cross-project worker pool, git via kubectl exec)
	buildExecutor := worker.NewBuildExecutor(agentRegistry, podGitOps, streamPub, nil)
	// VerifyExecutor for visual captures via Playwright pod
	verifyExecutor := worker.NewVerifyExecutor(k8sExecutor, streamPub, &worker.VerifyExecutorConfig{
		Namespace: namespace,
		PodName:   "playwright-0",
	})
	// SDLCTaskExecutor for skeleton project SDLC commands
	var sdlcTaskExecutor *worker.SDLCTaskExecutor
	if podGitOps != nil {
		sdlcTaskExecutor = worker.NewSDLCTaskExecutor(worker.SDLCTaskExecutorConfig{
			Namespace: namespace,
			PodGitOps: podGitOps,
		})
	}
	workerCfg := worker.DefaultWorkExecutorConfig()
	workExecutor := worker.NewWorkExecutor(
		workerService,
		workService,
		buildExecutor,
		verifyExecutor,
		sdlcTaskExecutor,
		workerCfg,
	)
	if err := workExecutor.Start(); err != nil {
		logger.Error("failed to start work executor", "error", err)
		os.Exit(1)
	}
	healthHandler.WithWorkExecutor(workExecutor)

	// Start queue maintenance worker (stale task recovery, worker health, cleanup, metrics)
	queueMaintenance := worker.NewQueueMaintenance(
		workQueueRepo,
		workerRegistryRepo,
		&worker.QueueMaintenanceConfig{
			StaleTaskTimeout:   60 * time.Minute,
			StaleWorkerTimeout: 2 * time.Minute,
			CleanupAge:         7 * 24 * time.Hour,
			MaintenancePeriod:  1 * time.Minute,
			MetricsPeriod:      15 * time.Second,
			BuildAudit:         buildAuditRepo, // Sync build audit when requeuing stale tasks
		},
	)
	queueMaintenance.Start()

	// Start operation cleanup worker (30-day retention)
	operationCleanup := worker.NewOperationCleanup(operationRepo, &worker.OperationCleanupConfig{
		RetentionPeriod: 30 * 24 * time.Hour,
		CleanupInterval: 1 * time.Hour,
	})
	operationCleanup.Start()

	// Start checkout cleanup worker (revokes expired checkout tokens)
	var checkoutCleanup *worker.CheckoutCleanup
	if checkoutService != nil {
		checkoutCleanup = worker.NewCheckoutCleanup(checkoutService, nil)
		checkoutCleanup.Start()
	}

	// Start session cleanup worker (tears down expired session previews)
	var sessionCleanup *worker.SessionCleanup
	if sessionService != nil {
		sessionCleanup = worker.NewSessionCleanup(sessionService, nil)
		sessionCleanup.Start()
	}

	// Start resource GC worker (cleans up orphaned K8s resources)
	var resourceGC *worker.ResourceGC
	if deployerAdapter != nil {
		resourceGC = worker.NewResourceGC(deployerAdapter, database.DB, nil)
		resourceGC.Start()
	}

	// Enable API documentation
	app.EnableDocs(buildOpenAPISpec())

	app.OnShutdown(func(ctx context.Context) error {
		if externalHealthChecker != nil {
			externalHealthChecker.Stop()
		}
		workExecutor.Stop()
		queueMaintenance.Stop()
		operationCleanup.Stop()
		if checkoutCleanup != nil {
			checkoutCleanup.Stop()
		}
		if sessionCleanup != nil {
			sessionCleanup.Stop()
		}
		if resourceGC != nil {
			resourceGC.Stop()
		}
		queueProcessor.Stop()
		webhookDispatcher.Stop()
		if auditShipper != nil {
			auditShipper.Close()
		}
		projectRepo.StopWatching()
		stopRateLimitCleanup()
		closeProvisioner(dbProvisioner, "database", logger)
		closeProvisioner(cacheProvisioner, "cache", logger)
		if err := tel.Shutdown(ctx); err != nil {
			logger.Error("telemetry shutdown error", "error", err)
		}
		return database.Close()
	})

	logger.Info("rdev-api starting",
		"version", version,
		"port", cfg.Port,
		"db_host", cfg.DBHost,
		"admin_key_set", cfg.AdminKey != "",
	)

	app.Run()
}