From adcea2fc1f172096f09e19b8f8b55ef673b05a36 Mon Sep 17 00:00:00 2001 From: jordan Date: Sun, 8 Feb 2026 23:57:38 -0700 Subject: [PATCH] fix(templates): upgrade Go to 1.25 and fix Woodpecker syntax MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Template Version Alignment - Go: 1.23 → 1.25 across all templates (go.work, go.mod, Dockerfiles, CI) - Alpine: latest → 3.19 (explicit version pinning) - Woodpecker: failure:retry → failure:ignore (invalid syntax fix) ## SDLC Tree Fixes (slackpath-5-full-lifecycle) Fixed merge failures by correcting lifecycle flow: 1. **Branch Creation**: Added missing create-branch step (planned → ready) - Bug: Merge command requires feature.Branch field to be set - Fix: POST /projects/{id}/sdlc/features/{slug}/branch 2. **Artifact Status**: Changed approval to pass for execution artifacts - Bug: Review/audit/QA need status="passed" not "approved" - Fix: /artifacts/{type}/approve → /artifacts/{type}/pass - Added: pass-qa step after wait-qa 3. **Phase Transition Order**: Reordered merge phase transition - Bug: Merge command checks if phase == "merge" first - Fix: transition-to-merge BEFORE merge-feature (not after) ## GCS Provisioner Fix - Replaced deprecated option.WithCredentialsFile with env var approach - Now uses GOOGLE_APPLICATION_CREDENTIALS for ADC (Application Default Credentials) - Avoids security risk from deprecated credential options - Fixed test: Added ComponentTypeGCS to ValidComponentTypes test ## Critical Rules Added - Version alignment: All template versions must stay in sync - When updating versions, grep entire templates/ tree ## Files Changed - 27 template files: Go version + Woodpecker syntax - 1 tree file: SDLC lifecycle flow corrections - 1 CLAUDE.md: Version alignment rule - 1 GCS provisioner: Deprecated API fix - 1 test file: Added missing component type Root cause: Skeleton templates lagged behind Go 1.25 release and had invalid Woodpecker syntax. SDLC tree skipped required branch creation and used wrong artifact approval endpoints. Co-Authored-By: Claude Sonnet 4.5 --- CLAUDE.md | 27 +- cmd/rdev-api/config.go | 10 + cmd/rdev-api/main.go | 25 +- cmd/rdev-api/openapi.go | 14 + cmd/rdev-api/openapi_ext.go | 847 ++++++++++++++++++ .../trees/slackpath-5-full-lifecycle.yaml | 70 +- docs/media-handling-spec.md | 402 +++++++++ go.mod | 30 +- go.sum | 55 ++ internal/adapter/gcs/provisioner.go | 261 ++++++ internal/adapter/gcs/provisioner_test.go | 72 ++ .../templates/astro-landing/.woodpecker.yml | 2 +- .../app-astro/.woodpecker.step.yml.tmpl | 2 +- .../app-nextjs/.woodpecker.step.yml.tmpl | 2 +- .../app-react/.woodpecker.step.yml.tmpl | 2 +- .../components/cli/.woodpecker.step.yml.tmpl | 2 +- .../templates/components/cli/go.mod.tmpl | 2 +- .../service/.woodpecker.step.yml.tmpl | 2 +- .../components/service/Dockerfile.tmpl | 2 +- .../templates/components/service/go.mod.tmpl | 2 +- .../worker/.woodpecker.step.yml.tmpl | 2 +- .../components/worker/Dockerfile.tmpl | 2 +- .../templates/components/worker/go.mod.tmpl | 2 +- .../templates/default/.woodpecker.yml | 2 +- .../templates/go-api/.woodpecker.yml | 6 +- .../templates/templates/go-api/Dockerfile | 4 +- .../templates/templates/go-api/go.mod.tmpl | 2 +- .../templates/skeleton/.woodpecker.yml.tmpl | 4 +- .../templates/templates/skeleton/go.work.tmpl | 2 +- .../templates/skeleton/pkg/go.mod.tmpl | 2 +- internal/domain/component.go | 6 +- internal/domain/component_test.go | 1 + internal/domain/credential.go | 5 + internal/domain/storage.go | 13 + internal/port/storage_provisioner.go | 24 + internal/service/component.go | 15 +- internal/service/component_infra.go | 57 +- internal/service/project_infra.go | 31 +- internal/service/project_infra_crud.go | 17 +- 39 files changed, 1940 insertions(+), 88 deletions(-) create mode 100644 docs/media-handling-spec.md create mode 100644 internal/adapter/gcs/provisioner.go create mode 100644 internal/adapter/gcs/provisioner_test.go create mode 100644 internal/domain/storage.go create mode 100644 internal/port/storage_provisioner.go diff --git a/CLAUDE.md b/CLAUDE.md index edebd3b..2f0d8fc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -77,18 +77,23 @@ When discussing code: "add to **platform**" = edit rdev; "add to **skeleton**" = - **Error wrapping:** ALWAYS use `%w` (not `%v`) when wrapping errors in `fmt.Errorf`. Using `%v` stringifies the error and breaks `errors.Is`/`errors.As` chains. For non-error types (structs, slices), create a typed error implementing `error` instead of stringifying with `%v`. - **Context propagation:** NEVER use `context.Background()` in handlers, services, or adapters that receive a context parameter. Always derive from parent context. Use `context.WithoutCancel(ctx)` for fire-and-forget goroutines that need tracing but independent cancellation. - **Cookbooks:** Load `.claude/skills/cookbook-scripts/SKILL.md` before writing/modifying any cookbook script or tree. +- **Version alignment:** Skeleton templates MUST use consistent versions across all files: Go 1.25 (go.work, go.mod, Dockerfiles, CI images), Node 20, Alpine 3.19. When updating a version, grep the entire templates/ tree and update ALL occurrences to prevent drift. ## Quick Reference ```bash -# Required env vars (add to ~/.zshrc) +# Environment (should already be in ~/.zshrc) export KUBECONFIG=~/.kube/orchard9-k3sf.yaml export RDEV_API_URL="https://rdev.masq-ops.orchard9.ai" -export RDEV_API_KEY="" +export RDEV_API_KEY="" # Already set in ~/.zshrc -# Infrastructure credentials stored in .secrets (gitignored) -# See: .claude/guides/ops/credentials.md for setup -# Keys: GITEA_TOKEN, CLOUDFLARE_API_TOKEN, CLOUDFLARE_ZONE_ID, WOODPECKER_* +# Verify environment is loaded +echo $RDEV_API_KEY # Should print a base64 string +# If empty: source ~/.zshrc + +# For scripts: use cookbooks/scripts/common.sh library +# Provides: api_call(), wait_for_build(), wait_for_pipeline(), wait_for_site() +# Example: source "$(dirname "$0")/common.sh" && api_call GET "/health" # Run locally go run ./cmd/rdev-api @@ -120,11 +125,13 @@ rdev-logs # Last 100 lines rdev-logs-f # Follow/stream rdev-pods # List pods -# API calls (NOTE: $RDEV_API_KEY doesn't expand in curl -H, use the test script instead) -# ./cookbooks/scripts/landing-test.sh run|status|teardown -curl -H "X-API-Key: $RDEV_API_KEY" $RDEV_API_URL/health -curl -H "X-API-Key: $RDEV_API_KEY" $RDEV_API_URL/projects -curl -H "X-API-Key: $RDEV_API_KEY" $RDEV_API_URL/work/stats +# API calls - use cookbook test scripts (they handle auth via common.sh) +./cookbooks/scripts/landing-test.sh run|status|teardown +./cookbooks/scripts/tree-runner.sh run --project-name + +# Or direct API calls (requires env vars above) +curl -s -H "X-API-Key: $RDEV_API_KEY" $RDEV_API_URL/health | jq +curl -s -H "X-API-Key: $RDEV_API_KEY" $RDEV_API_URL/projects | jq ``` ## Architecture Overview diff --git a/cmd/rdev-api/config.go b/cmd/rdev-api/config.go index eaf737a..d0ecae6 100644 --- a/cmd/rdev-api/config.go +++ b/cmd/rdev-api/config.go @@ -77,6 +77,11 @@ type InfraConfig struct { RedisHost string // e.g., "redis.databases.svc.cluster.local" RedisPort int // e.g., 6379 RedisPassword string // admin password for ACL management + + // GCS provisioner (for project storage) + GCSProjectID string // e.g., "threesix-prod" + GCSCredentialsPath string // Path to service account JSON (empty = ADC) + GCSLocation string // Bucket location (default: "US") } func loadConfig() Config { @@ -169,6 +174,11 @@ func loadInfraConfig(ctx context.Context, store port.CredentialStore, cfg Config RedisHost: os.Getenv("REDIS_HOST"), // e.g., "redis.databases.svc.cluster.local" RedisPort: envutil.GetEnvInt("REDIS_PORT", 6379), RedisPassword: os.Getenv("REDIS_PASSWORD"), + + // GCS provisioner (env-only) + GCSProjectID: os.Getenv("GCS_PROJECT_ID"), + GCSCredentialsPath: os.Getenv("GCS_CREDENTIALS_PATH"), + GCSLocation: envutil.GetEnv("GCS_LOCATION", "US"), } // Log which credentials were loaded from store vs env diff --git a/cmd/rdev-api/main.go b/cmd/rdev-api/main.go index 54a38bd..5379d81 100644 --- a/cmd/rdev-api/main.go +++ b/cmd/rdev-api/main.go @@ -13,6 +13,7 @@ import ( "github.com/orchard9/rdev/internal/adapter/codeagent/claudecode" "github.com/orchard9/rdev/internal/adapter/codeagent/opencode" "github.com/orchard9/rdev/internal/adapter/deployer" + gcsadapter "github.com/orchard9/rdev/internal/adapter/gcs" "github.com/orchard9/rdev/internal/adapter/gitea" "github.com/orchard9/rdev/internal/adapter/kubernetes" "github.com/orchard9/rdev/internal/adapter/memory" @@ -200,6 +201,24 @@ func main() { logger.Info("redis provisioner initialized", "host", infraCfg.RedisHost) } } + defer closeProvisioner(cacheProvisioner, "redis", logger) + + // Initialize storage provisioner (optional - for project storage via GCS) + var storageProvisioner port.StorageProvisioner + if infraCfg.GCSProjectID != "" { + var err error + storageProvisioner, err = gcsadapter.NewProvisioner(gcsadapter.Config{ + GoogleProjectID: infraCfg.GCSProjectID, + CredentialsPath: infraCfg.GCSCredentialsPath, + Location: infraCfg.GCSLocation, + }, logger) + if err != nil { + logger.Warn("failed to initialize gcs provisioner", "error", err) + } else { + logger.Info("gcs provisioner initialized", "project_id", infraCfg.GCSProjectID, "location", infraCfg.GCSLocation) + } + } + defer closeProvisioner(storageProvisioner, "gcs", logger) // Initialize registry client (for monitoring and image cleanup on project teardown) var registryClient *zot.Client @@ -360,13 +379,16 @@ func main() { ClusterIP: infraCfg.ClusterIP, }, ) - // Wire optional database, cache, and registry provisioners + // Wire optional database, cache, storage, and registry provisioners if dbProvisioner != nil { projectInfraService = projectInfraService.WithDatabaseProvisioner(dbProvisioner) } if cacheProvisioner != nil { projectInfraService = projectInfraService.WithCacheProvisioner(cacheProvisioner) } + if storageProvisioner != nil { + projectInfraService = projectInfraService.WithStorageProvisioner(storageProvisioner) + } if registryClient != nil { projectInfraService = projectInfraService.WithRegistryProvider(registryClient) } @@ -409,6 +431,7 @@ func main() { ). WithDatabaseProvisioner(dbProvisioner). WithCacheProvisioner(cacheProvisioner). + WithStorageProvisioner(storageProvisioner). WithCredentialStore(credentialStore) componentsHandler = handlers.NewComponentsHandler(componentService). SetOperationService(operationService) diff --git a/cmd/rdev-api/openapi.go b/cmd/rdev-api/openapi.go index 8908e90..27a4380 100644 --- a/cmd/rdev-api/openapi.go +++ b/cmd/rdev-api/openapi.go @@ -60,6 +60,13 @@ Command output is streamed via Server-Sent Events (SSE) at /projects/{id}/events spec.WithTag("Builds", "Build orchestration and history") spec.WithTag("SDLC", "Software Development Lifecycle orchestration") spec.WithTag("System", "Health and readiness endpoints") + spec.WithTag("Components", "Composable monorepo component management") + spec.WithTag("Credentials", "Infrastructure credential management") + spec.WithTag("Diagnostics", "Project health and diagnostics") + spec.WithTag("Verify", "Visual verification with Playwright") + spec.WithTag("Webhooks", "External webhook receivers") + spec.WithTag("Infrastructure", "Git, deployment, DNS, and CI pipeline management") + spec.WithTag("Sagas", "Distributed workflow orchestration with compensation") // Register all path operations registerSystemPaths(spec) @@ -73,6 +80,13 @@ Command output is streamed via Server-Sent Events (SSE) at /projects/{id}/events registerWorkerPaths(spec) registerBuildPaths(spec) registerSDLCPaths(spec) + registerComponentPaths(spec) + registerCredentialPaths(spec) + registerDiagnosticsPaths(spec) + registerVerifyPaths(spec) + registerWebhookPaths(spec) + registerInfrastructurePaths(spec) + registerSagaPaths(spec) return spec } diff --git a/cmd/rdev-api/openapi_ext.go b/cmd/rdev-api/openapi_ext.go index 5f729ad..f461f84 100644 --- a/cmd/rdev-api/openapi_ext.go +++ b/cmd/rdev-api/openapi_ext.go @@ -656,3 +656,850 @@ Combines project creation (git repo, DNS, CI activation) with build submission i }`, )) } + +func registerComponentPaths(spec *api.OpenAPISpec) { + spec.AddPath("/projects/{id}/components", "get", withAuthAndParams( + "List components", + `Returns all components in a project's composable monorepo. + +Each project can contain multiple components (services, workers, apps, CLIs) organized in a monorepo structure.`, + "Components", + "projects:read", + []param{{Name: "id", In: "path", Description: "Project ID", Required: true}}, + )) + + spec.AddPath("/projects/{id}/components", "post", withAuthBodyAndParams( + "Add component", + `Adds a new component to a project's monorepo. + +Generates component skeleton from template, updates configuration, and commits changes.`, + "Components", + "projects:execute", + []param{{Name: "id", In: "path", Description: "Project ID", Required: true}}, + `{ + "type": "service", + "name": "api-gateway", + "template": "go-rest-api", + "port": 8080 +}`, + `{ + "type": "service", + "name": "api-gateway", + "path": "services/api-gateway", + "port": 8080, + "template": "go-rest-api", + "dependencies": [], + "operation_id": "op-abc123" +}`, + )) + + spec.AddPath("/projects/{id}/components/batch", "post", withAuthBodyAndParams( + "Add multiple components", + `Adds multiple components to a project's monorepo atomically. + +All components are created in a single transaction. If any component fails, the entire operation is rolled back.`, + "Components", + "projects:execute", + []param{{Name: "id", In: "path", Description: "Project ID", Required: true}}, + `{ + "components": [ + {"type": "service", "name": "auth", "port": 8081}, + {"type": "service", "name": "users", "port": 8082}, + {"type": "worker", "name": "email-processor"} + ] +}`, + `{ + "components": [ + { + "type": "service", + "name": "auth", + "path": "services/auth", + "port": 8081, + "template": "go-rest-api", + "dependencies": [] + }, + { + "type": "service", + "name": "users", + "path": "services/users", + "port": 8082, + "template": "go-rest-api", + "dependencies": [] + }, + { + "type": "worker", + "name": "email-processor", + "path": "workers/email-processor", + "port": 0, + "template": "go-worker", + "dependencies": [] + } + ], + "operation_id": "op-abc123" +}`, + )) + + // Use wildcard for dynamic component path + spec.AddPath("/projects/{id}/components/{path}", "delete", map[string]any{ + "summary": "Remove component", + "description": "Removes a component from the project's monorepo.\n\n**Required scope**: `projects:execute`", + "tags": []string{"Components"}, + "security": []map[string]any{ + {"ApiKeyAuth": []string{}}, + }, + "parameters": []map[string]any{ + { + "name": "id", + "in": "path", + "description": "Project ID", + "required": true, + "schema": map[string]any{"type": "string"}, + }, + { + "name": "path", + "in": "path", + "description": "Component path (e.g., services/api-gateway, workers/email-processor)", + "required": true, + "schema": map[string]any{"type": "string"}, + }, + }, + "responses": map[string]any{ + "204": map[string]any{"description": "No Content - Component removed"}, + "401": map[string]any{"description": "Unauthorized - Missing or invalid API key"}, + "403": map[string]any{"description": "Forbidden - Insufficient permissions"}, + "404": map[string]any{"description": "Not Found - Component not found"}, + }, + }) +} + +func registerCredentialPaths(spec *api.OpenAPISpec) { + spec.AddPath("/credentials", "get", map[string]any{ + "summary": "List credentials", + "description": `Returns all infrastructure credentials with values masked. + +Optionally filter by category using ?category= query parameter. + +**Required scope**: ` + "`admin`", + "tags": []string{"Credentials"}, + "security": []map[string]any{ + {"ApiKeyAuth": []string{}}, + }, + "parameters": []map[string]any{ + { + "name": "category", + "in": "query", + "description": "Filter by category (e.g., git, ci, dns, registry)", + "required": false, + "schema": map[string]any{"type": "string"}, + }, + }, + "responses": map[string]any{ + "200": map[string]any{ + "description": "Success", + "content": map[string]any{ + "application/json": map[string]any{ + "example": `[ + { + "key": "GITEA_TOKEN", + "value": "***", + "description": "Gitea API token for repo operations", + "category": "git", + "created_at": "2026-01-01T00:00:00Z", + "updated_at": "2026-01-15T10:30:00Z", + "updated_by": "admin" + } +]`, + }, + }, + }, + "401": map[string]any{"description": "Unauthorized - Missing or invalid API key"}, + "403": map[string]any{"description": "Forbidden - Admin scope required"}, + }, + }) + + spec.AddPath("/credentials/{key}", "get", withAuthAndParams( + "Get credential", + `Returns a single credential by key with full unmasked value. + +**Security**: Only admin scope can retrieve unmasked credentials.`, + "Credentials", + "admin", + []param{{Name: "key", In: "path", Description: "Credential key (e.g., GITEA_TOKEN, WOODPECKER_SECRET)", Required: true}}, + )) + + spec.AddPath("/credentials", "post", withAuthAndBody( + "Set credential", + `Creates or updates a single credential. + +Credentials are encrypted at rest in PostgreSQL.`, + "Credentials", + "admin", + `{ + "key": "GITEA_TOKEN", + "value": "abc123xyz", + "description": "Gitea API token for repo operations", + "category": "git" +}`, + `{ + "status": "stored", + "key": "GITEA_TOKEN" +}`, + )) + + spec.AddPath("/credentials/batch", "post", withAuthAndBody( + "Set multiple credentials", + `Creates or updates multiple credentials atomically. + +Useful for bulk credential loading from configuration files.`, + "Credentials", + "admin", + `{ + "credentials": [ + {"key": "GITEA_TOKEN", "value": "abc123", "category": "git"}, + {"key": "WOODPECKER_SECRET", "value": "xyz789", "category": "ci"} + ] +}`, + `{ + "status": "stored", + "count": 2, + "keys": ["GITEA_TOKEN", "WOODPECKER_SECRET"] +}`, + )) + + spec.AddPath("/credentials/{key}", "delete", map[string]any{ + "summary": "Delete credential", + "description": "Removes a credential permanently.\n\n**Required scope**: `admin`", + "tags": []string{"Credentials"}, + "security": []map[string]any{ + {"ApiKeyAuth": []string{}}, + }, + "parameters": []map[string]any{ + { + "name": "key", + "in": "path", + "description": "Credential key to delete", + "required": true, + "schema": map[string]any{"type": "string"}, + }, + }, + "responses": map[string]any{ + "200": map[string]any{ + "description": "Success", + "content": map[string]any{ + "application/json": map[string]any{ + "example": `{"status": "deleted", "key": "GITEA_TOKEN"}`, + }, + }, + }, + "401": map[string]any{"description": "Unauthorized - Missing or invalid API key"}, + "403": map[string]any{"description": "Forbidden - Admin scope required"}, + "404": map[string]any{"description": "Not Found - Credential not found"}, + }, + }) +} + +func registerDiagnosticsPaths(spec *api.OpenAPISpec) { + spec.AddPath("/projects/{projectId}/diagnostics", "get", withAuthAndParams( + "Get project diagnostics", + `Returns comprehensive health and diagnostic information for a project. + +Includes pod status, resource usage, recent errors, external system connectivity, and deployment health.`, + "Diagnostics", + "projects:read", + []param{{Name: "projectId", In: "path", Description: "Project ID", Required: true}}, + )) +} + +func registerVerifyPaths(spec *api.OpenAPISpec) { + spec.AddPath("/verify", "post", withAuthAndBody( + "Submit verification task", + `Submits a visual verification task using Playwright. + +Captures screenshots at multiple viewports and optionally records video. Results include screenshot URLs and AI analysis of visual quality.`, + "Verify", + "verify:write", + `{ + "project_id": "my-landing-page", + "url": "https://my-landing-page.threesix.ai", + "viewports": ["desktop", "tablet", "mobile"], + "wait_for": "networkidle", + "wait_timeout": 30000, + "full_page": true, + "video": true, + "callback_url": "https://myapp.com/webhooks/verify" +}`, + `{ + "task_id": "verify-abc123", + "status_url": "/verify/verify-abc123", + "stream_url": "/verify/verify-abc123/stream" +}`, + )) + + spec.AddPath("/verify/{taskId}", "get", withAuthAndParams( + "Get verification status", + `Returns the status and results of a verification task. + +Includes screenshot URLs, video URL (if recorded), and duration.`, + "Verify", + "verify:read", + []param{{Name: "taskId", In: "path", Description: "Verification task ID", Required: true}}, + )) + + spec.AddPath("/verify/{taskId}/stream", "get", map[string]any{ + "summary": "Stream verification events", + "description": `Streams real-time verification task events via Server-Sent Events (SSE). + +Events include task started, screenshot captured, video recorded, task completed/failed. + +**Required scope**: ` + "`verify:read`", + "tags": []string{"Verify"}, + "security": []map[string]any{ + {"ApiKeyAuth": []string{}}, + }, + "parameters": []map[string]any{ + { + "name": "taskId", + "in": "path", + "description": "Verification task ID", + "required": true, + "schema": map[string]any{"type": "string"}, + }, + }, + "responses": map[string]any{ + "200": map[string]any{ + "description": "Success - SSE stream", + "content": map[string]any{ + "text/event-stream": map[string]any{ + "example": `event: connected +data: {"task_id":"verify-abc123"} + +event: started +data: {"viewport":"desktop","url":"https://my-landing-page.threesix.ai"} + +event: screenshot_captured +data: {"viewport":"desktop","url":"https://storage/verify-abc123-desktop.png"} + +event: completed +data: {"task_id":"verify-abc123","duration_ms":5432}`, + }, + }, + }, + "401": map[string]any{"description": "Unauthorized - Missing or invalid API key"}, + "403": map[string]any{"description": "Forbidden - Insufficient permissions"}, + "404": map[string]any{"description": "Not Found - Task not found"}, + }, + }) + + spec.AddPath("/verify/{taskId}", "delete", withAuthAndParams( + "Cancel verification task", + `Cancels a pending or running verification task. + +Stops Playwright browser and marks task as cancelled.`, + "Verify", + "verify:write", + []param{{Name: "taskId", In: "path", Description: "Verification task ID", Required: true}}, + )) + + spec.AddPath("/projects/{id}/verify", "get", map[string]any{ + "summary": "List project verifications", + "description": `Returns verification tasks for a project with pagination. + +**Required scope**: ` + "`verify:read`", + "tags": []string{"Verify"}, + "security": []map[string]any{ + {"ApiKeyAuth": []string{}}, + }, + "parameters": []map[string]any{ + { + "name": "id", + "in": "path", + "description": "Project ID", + "required": true, + "schema": map[string]any{"type": "string"}, + }, + { + "name": "limit", + "in": "query", + "description": "Maximum number of results (default: 50)", + "required": false, + "schema": map[string]any{"type": "integer", "default": 50}, + }, + { + "name": "offset", + "in": "query", + "description": "Pagination offset (default: 0)", + "required": false, + "schema": map[string]any{"type": "integer", "default": 0}, + }, + }, + "responses": map[string]any{ + "200": map[string]any{"description": "Success"}, + "401": map[string]any{"description": "Unauthorized - Missing or invalid API key"}, + "403": map[string]any{"description": "Forbidden - Insufficient permissions"}, + }, + }) +} + +func registerInfrastructurePaths(spec *api.OpenAPISpec) { + // Git repository endpoints + spec.AddPath("/projects/{id}/repo", "post", withAuthBodyAndParams( + "Create repository", + `Creates a git repository for a project in Gitea. + +Repository is created under the configured git owner (e.g., threesix organization).`, + "Infrastructure", + "projects:execute", + []param{{Name: "id", In: "path", Description: "Project ID (will be repository name)", Required: true}}, + `{ + "description": "Landing page for product launch", + "private": false +}`, + `{ + "id": 123, + "owner": "threesix", + "name": "my-landing-page", + "full_name": "threesix/my-landing-page", + "description": "Landing page for product launch", + "private": false, + "clone_ssh": "git@git.threesix.ai:threesix/my-landing-page.git", + "clone_http": "https://git.threesix.ai/threesix/my-landing-page.git", + "html_url": "https://git.threesix.ai/threesix/my-landing-page" +}`, + )) + + spec.AddPath("/projects/{id}/repo", "get", withAuthAndParams( + "Get repository", + `Returns repository information for a project. + +Includes clone URLs, visibility, and metadata.`, + "Infrastructure", + "projects:read", + []param{{Name: "id", In: "path", Description: "Project ID", Required: true}}, + )) + + spec.AddPath("/projects/{id}/repo", "delete", withAuthAndParams( + "Delete repository", + `Deletes the git repository for a project. + +**Warning**: This permanently deletes all git history and branches.`, + "Infrastructure", + "projects:execute", + []param{{Name: "id", In: "path", Description: "Project ID", Required: true}}, + )) + + // Deployment endpoints + spec.AddPath("/projects/{id}/deploy", "post", withAuthAndParams( + "Deploy project", + `Deploys a project to Kubernetes. + +Creates deployment, service, and ingress resources.`, + "Infrastructure", + "projects:execute", + []param{{Name: "id", In: "path", Description: "Project ID", Required: true}}, + )) + + spec.AddPath("/projects/{id}/deploy/status", "get", withAuthAndParams( + "Get deployment status", + `Returns the current deployment status for a project. + +Includes pod status, replica counts, and health information.`, + "Infrastructure", + "projects:read", + []param{{Name: "id", In: "path", Description: "Project ID", Required: true}}, + )) + + spec.AddPath("/projects/{id}/deploy", "delete", withAuthAndParams( + "Undeploy project", + `Removes deployment, service, and ingress resources for a project. + +Pods are terminated and all Kubernetes resources are deleted.`, + "Infrastructure", + "projects:execute", + []param{{Name: "id", In: "path", Description: "Project ID", Required: true}}, + )) + + spec.AddPath("/projects/{id}/deploy/restart", "post", withAuthAndParams( + "Restart deployment", + `Restarts a project's deployment by recreating all pods. + +Useful for applying configuration changes or recovering from errors.`, + "Infrastructure", + "projects:execute", + []param{{Name: "id", In: "path", Description: "Project ID", Required: true}}, + )) + + spec.AddPath("/projects/{id}/deploy/scale", "post", withAuthBodyAndParams( + "Scale deployment", + `Scales a project's deployment to the specified replica count.`, + "Infrastructure", + "projects:execute", + []param{{Name: "id", In: "path", Description: "Project ID", Required: true}}, + `{"replicas": 3}`, + `{"status": "scaled", "replicas": 3}`, + )) + + spec.AddPath("/projects/{id}/deploy/logs", "get", map[string]any{ + "summary": "Get deployment logs", + "description": `Returns recent logs from a project's deployment pods. + +**Required scope**: ` + "`projects:read`", + "tags": []string{"Infrastructure"}, + "security": []map[string]any{ + {"ApiKeyAuth": []string{}}, + }, + "parameters": []map[string]any{ + { + "name": "id", + "in": "path", + "description": "Project ID", + "required": true, + "schema": map[string]any{"type": "string"}, + }, + { + "name": "tail", + "in": "query", + "description": "Number of recent log lines to retrieve (default: 100)", + "required": false, + "schema": map[string]any{"type": "integer", "default": 100}, + }, + }, + "responses": map[string]any{ + "200": map[string]any{"description": "Success"}, + "401": map[string]any{"description": "Unauthorized - Missing or invalid API key"}, + "403": map[string]any{"description": "Forbidden - Insufficient permissions"}, + "404": map[string]any{"description": "Not Found - Project not found"}, + }, + }) + + // Domain endpoints + spec.AddPath("/projects/{id}/domain", "post", withAuthBodyAndParams( + "Add domain", + `Adds a custom domain to a project. + +Creates DNS A record if domain is a subdomain of the configured base domain (e.g., threesix.ai).`, + "Infrastructure", + "projects:execute", + []param{{Name: "id", In: "path", Description: "Project ID", Required: true}}, + `{"domain": "custom.example.com"}`, + `{ + "project": "my-landing-page", + "domain": "custom.example.com", + "status": "configured", + "note": "External domain configured. Point your DNS to 208.122.204.172" +}`, + )) + + spec.AddPath("/projects/{id}/domain", "delete", map[string]any{ + "summary": "Remove domain", + "description": `Removes a custom domain from a project. + +Deletes DNS A record if domain was a managed subdomain. + +**Required scope**: ` + "`projects:execute`", + "tags": []string{"Infrastructure"}, + "security": []map[string]any{ + {"ApiKeyAuth": []string{}}, + }, + "parameters": []map[string]any{ + { + "name": "id", + "in": "path", + "description": "Project ID", + "required": true, + "schema": map[string]any{"type": "string"}, + }, + { + "name": "domain", + "in": "query", + "description": "Domain to remove (e.g., custom.example.com)", + "required": true, + "schema": map[string]any{"type": "string"}, + }, + }, + "responses": map[string]any{ + "200": map[string]any{"description": "Success"}, + "400": map[string]any{"description": "Bad Request - Missing domain parameter"}, + "401": map[string]any{"description": "Unauthorized - Missing or invalid API key"}, + "403": map[string]any{"description": "Forbidden - Insufficient permissions"}, + }, + }) + + spec.AddPath("/projects/{id}/domains", "get", withAuthAndParams( + "List domains", + `Returns all domains (primary and aliases) for a project.`, + "Infrastructure", + "projects:read", + []param{{Name: "id", In: "path", Description: "Project ID", Required: true}}, + )) + + spec.AddPath("/projects/{id}/domains", "post", withAuthBodyAndParams( + "Add domain alias", + `Adds an additional domain alias to a project. + +Projects can have multiple domains pointing to the same deployment.`, + "Infrastructure", + "projects:execute", + []param{{Name: "id", In: "path", Description: "Project ID", Required: true}}, + `{"domain": "alias.example.com", "proxied": false}`, + `{"status": "added", "domain": "alias.example.com"}`, + )) + + spec.AddPath("/projects/{id}/domains/{domain}", "delete", withAuthAndParams( + "Remove domain alias", + `Removes a domain alias from a project. + +Primary domain cannot be removed via this endpoint.`, + "Infrastructure", + "projects:execute", + []param{ + {Name: "id", In: "path", Description: "Project ID", Required: true}, + {Name: "domain", In: "path", Description: "Domain to remove", Required: true}, + }, + )) + + // CI pipeline endpoints + spec.AddPath("/projects/{id}/pipelines", "get", withAuthAndParams( + "List CI pipelines", + `Returns recent CI pipeline runs for a project. + +Includes build status, branch, commit info, and duration.`, + "Infrastructure", + "projects:read", + []param{{Name: "id", In: "path", Description: "Project ID", Required: true}}, + )) + + spec.AddPath("/projects/{id}/pipelines/{number}", "get", withAuthAndParams( + "Get pipeline details", + `Returns detailed information for a specific pipeline run.`, + "Infrastructure", + "projects:read", + []param{ + {Name: "id", In: "path", Description: "Project ID", Required: true}, + {Name: "number", In: "path", Description: "Pipeline number", Required: true}, + }, + )) + + spec.AddPath("/projects/{id}/pipelines/{number}/steps", "get", withAuthAndParams( + "Get pipeline steps", + `Returns the steps (stages) of a pipeline run with status and logs.`, + "Infrastructure", + "projects:read", + []param{ + {Name: "id", In: "path", Description: "Project ID", Required: true}, + {Name: "number", In: "path", Description: "Pipeline number", Required: true}, + }, + )) + + spec.AddPath("/projects/{id}/pipelines/{number}/retry", "post", withAuthAndParams( + "Retry pipeline", + `Retries a failed pipeline from the failed step. + +Part of the enterprise resilience architecture for handling transient failures.`, + "Infrastructure", + "projects:execute", + []param{ + {Name: "id", In: "path", Description: "Project ID", Required: true}, + {Name: "number", In: "path", Description: "Pipeline number to retry", Required: true}, + }, + )) +} + +func registerWebhookPaths(spec *api.OpenAPISpec) { + spec.AddPath("/webhooks/woodpecker", "post", map[string]any{ + "summary": "Woodpecker CI webhook", + "description": `Receives build event webhooks from Woodpecker CI. + +**Authentication**: Uses HMAC-SHA256 signature verification (X-Woodpecker-Signature header), not API key auth. + +Processes successful builds on main/master branches to trigger deployments and DNS updates.`, + "tags": []string{"Webhooks"}, + "parameters": []map[string]any{ + { + "name": "X-Woodpecker-Signature", + "in": "header", + "description": "HMAC-SHA256 signature of request body", + "required": false, + "schema": map[string]any{"type": "string"}, + }, + }, + "requestBody": map[string]any{ + "required": true, + "content": map[string]any{ + "application/json": map[string]any{ + "example": `{ + "event": "push", + "repo": { + "owner": "threesix", + "name": "my-landing-page", + "full_name": "threesix/my-landing-page" + }, + "build": { + "number": 42, + "status": "success", + "branch": "main", + "commit": "abc123def456", + "message": "Add feature X", + "author": "jordan" + } +}`, + }, + }, + }, + "responses": map[string]any{ + "200": map[string]any{ + "description": "Success - Event processed", + "content": map[string]any{ + "application/json": map[string]any{ + "example": `{ + "status": "success", + "project": "my-landing-page", + "image": "registry.threesix.ai/my-landing-page:abc123de", + "commit": "abc123def456", + "note": "component deployments managed by CI pipeline" +}`, + }, + }, + }, + "400": map[string]any{"description": "Bad Request - Invalid payload"}, + "401": map[string]any{"description": "Unauthorized - Invalid signature"}, + }, + }) +} + +func registerSagaPaths(spec *api.OpenAPISpec) { + spec.AddPath("/sagas", "post", withAuthAndBody( + "Create and start saga", + `Creates a new saga workflow and starts execution. + +Sagas are distributed workflows with automatic compensation on failure. Useful for multi-step operations like project creation with rollback support.`, + "Sagas", + "projects:execute", + `{ + "name": "create-full-stack-project", + "max_retries": 3, + "steps": [ + { + "name": "create-repo", + "action": "api", + "compensate": "delete-repo", + "config": {"method": "POST", "endpoint": "/projects/{id}/repo"} + }, + { + "name": "create-db", + "action": "api", + "depends_on": ["create-repo"], + "compensate": "delete-db", + "config": {"method": "POST", "endpoint": "/projects/{id}/database"} + } + ] +}`, + `{ + "id": "saga-abc123", + "name": "create-full-stack-project", + "status": "running", + "current_step": "create-repo", + "retry_count": 0, + "max_retries": 3, + "created_at": "2026-02-08T10:00:00Z", + "updated_at": "2026-02-08T10:00:00Z" +}`, + )) + + spec.AddPath("/sagas", "get", map[string]any{ + "summary": "List sagas", + "description": `Returns sagas with optional filtering. + +**Query parameters**: ?name=&status= + +**Required scope**: ` + "`projects:read`", + "tags": []string{"Sagas"}, + "security": []map[string]any{ + {"ApiKeyAuth": []string{}}, + }, + "parameters": []map[string]any{ + { + "name": "name", + "in": "query", + "description": "Filter by saga name", + "required": false, + "schema": map[string]any{"type": "string"}, + }, + { + "name": "status", + "in": "query", + "description": "Filter by status (pending, running, completed, failed, compensating, compensated)", + "required": false, + "schema": map[string]any{"type": "string"}, + }, + }, + "responses": map[string]any{ + "200": map[string]any{"description": "Success"}, + "401": map[string]any{"description": "Unauthorized - Missing or invalid API key"}, + "403": map[string]any{"description": "Forbidden - Insufficient permissions"}, + }, + }) + + spec.AddPath("/sagas/{id}", "get", withAuthAndParams( + "Get saga", + `Returns detailed saga execution state including all steps, outputs, and errors.`, + "Sagas", + "projects:read", + []param{{Name: "id", In: "path", Description: "Saga ID", Required: true}}, + )) + + spec.AddPath("/sagas/{id}", "delete", withAuthAndParams( + "Delete saga", + `Deletes a saga and all associated state. + +Running sagas should be cancelled before deletion.`, + "Sagas", + "projects:execute", + []param{{Name: "id", In: "path", Description: "Saga ID", Required: true}}, + )) + + spec.AddPath("/sagas/{id}/retry", "post", withAuthAndParams( + "Retry failed saga", + `Resumes a failed saga from the last failed step. + +Increments retry count and attempts to continue execution.`, + "Sagas", + "projects:execute", + []param{{Name: "id", In: "path", Description: "Saga ID", Required: true}}, + )) + + spec.AddPath("/sagas/{id}/rollback", "post", withAuthAndParams( + "Rollback saga", + `Triggers compensation for a failed saga. + +Runs compensation actions in reverse order to undo completed steps.`, + "Sagas", + "projects:execute", + []param{{Name: "id", In: "path", Description: "Saga ID", Required: true}}, + )) + + spec.AddPath("/sagas/{id}/steps/{step}/retry", "post", withAuthAndParams( + "Retry specific step", + `Retries a specific failed step within a saga. + +Does not increment saga-level retry count.`, + "Sagas", + "projects:execute", + []param{ + {Name: "id", In: "path", Description: "Saga ID", Required: true}, + {Name: "step", In: "path", Description: "Step name", Required: true}, + }, + )) + + spec.AddPath("/sagas/{id}/steps/{step}/skip", "post", withAuthAndParams( + "Skip step", + `Skips a failed step and continues saga execution. + +Marks step as skipped and allows dependent steps to proceed.`, + "Sagas", + "projects:execute", + []param{ + {Name: "id", In: "path", Description: "Saga ID", Required: true}, + {Name: "step", In: "path", Description: "Step name to skip", Required: true}, + }, + )) +} diff --git a/cookbooks/trees/slackpath-5-full-lifecycle.yaml b/cookbooks/trees/slackpath-5-full-lifecycle.yaml index 26bbb43..c50ddd6 100644 --- a/cookbooks/trees/slackpath-5-full-lifecycle.yaml +++ b/cookbooks/trees/slackpath-5-full-lifecycle.yaml @@ -233,11 +233,20 @@ steps: # ============================================================ # PHASE 4: PLANNED → READY - # No new artifacts needed, just transition + # Create git branch, then transition # ============================================================ + create-branch: + description: "Create feature branch" + depends_on: [transition-to-planned] + action: api + method: POST + endpoint: "/projects/{{ .outputs.create-project.project_id }}/sdlc/features/{{ .vars.feature_slug }}/branch" + outputs: + - branch_name: .data.name + transition-to-ready: description: "Transition from planned to ready" - depends_on: [transition-to-planned] + depends_on: [create-branch] action: api method: POST endpoint: "/projects/{{ .outputs.create-project.project_id }}/sdlc/features/{{ .vars.feature_slug }}/transition" @@ -314,18 +323,16 @@ steps: max_attempts: 720 poll_interval: 5 - approve-review: - description: "API approves the review" + pass-review: + description: "Mark review as passed" depends_on: [wait-review] action: api method: POST - endpoint: "/projects/{{ .outputs.create-project.project_id }}/sdlc/features/{{ .vars.feature_slug }}/artifacts/review/approve" - body: - comment: "Review approved by automation" + endpoint: "/projects/{{ .outputs.create-project.project_id }}/sdlc/features/{{ .vars.feature_slug }}/artifacts/review/pass" transition-to-review: description: "Transition to review phase" - depends_on: [approve-review] + depends_on: [pass-review] action: api method: POST endpoint: "/projects/{{ .outputs.create-project.project_id }}/sdlc/features/{{ .vars.feature_slug }}/transition" @@ -359,18 +366,16 @@ steps: max_attempts: 720 poll_interval: 5 - approve-audit: - description: "API approves the audit" + pass-audit: + description: "Mark audit as passed" depends_on: [wait-audit] action: api method: POST - endpoint: "/projects/{{ .outputs.create-project.project_id }}/sdlc/features/{{ .vars.feature_slug }}/artifacts/audit/approve" - body: - comment: "Audit approved by automation" + endpoint: "/projects/{{ .outputs.create-project.project_id }}/sdlc/features/{{ .vars.feature_slug }}/artifacts/audit/pass" transition-to-audit: description: "Transition to audit phase" - depends_on: [approve-audit] + depends_on: [pass-audit] action: api method: POST endpoint: "/projects/{{ .outputs.create-project.project_id }}/sdlc/features/{{ .vars.feature_slug }}/transition" @@ -404,9 +409,16 @@ steps: max_attempts: 720 poll_interval: 5 + pass-qa: + description: "Mark QA as passed" + depends_on: [wait-qa] + action: api + method: POST + endpoint: "/projects/{{ .outputs.create-project.project_id }}/sdlc/features/{{ .vars.feature_slug }}/artifacts/qa_results/pass" + transition-to-qa: description: "Transition to QA phase" - depends_on: [wait-qa] + depends_on: [pass-qa] action: api method: POST endpoint: "/projects/{{ .outputs.create-project.project_id }}/sdlc/features/{{ .vars.feature_slug }}/transition" @@ -417,22 +429,11 @@ steps: # ============================================================ # PHASE 9: QA → MERGE - # Merge feature branch to main + # Transition to merge phase, then merge feature branch to main # ============================================================ - merge-feature: - description: "Merge feature branch to main" - depends_on: [transition-to-qa] - action: api - method: POST - endpoint: "/projects/{{ .outputs.create-project.project_id }}/sdlc/features/{{ .vars.feature_slug }}/merge" - body: - strategy: "squash" - outputs: - - merge_commit: .data.commit_sha - transition-to-merge: description: "Transition to merge phase" - depends_on: [merge-feature] + depends_on: [transition-to-qa] action: api method: POST endpoint: "/projects/{{ .outputs.create-project.project_id }}/sdlc/features/{{ .vars.feature_slug }}/transition" @@ -441,13 +442,24 @@ steps: outputs: - new_phase: .data.phase + merge-feature: + description: "Merge feature branch to main" + depends_on: [transition-to-merge] + action: api + method: POST + endpoint: "/projects/{{ .outputs.create-project.project_id }}/sdlc/features/{{ .vars.feature_slug }}/merge" + body: + strategy: "squash" + outputs: + - merge_commit: .data.commit_sha + # ============================================================ # PHASE 10: MERGE → RELEASED # Archive the feature # ============================================================ wait-final-deploy: description: "Wait for merged code to deploy" - depends_on: [transition-to-merge] + depends_on: [merge-feature] action: wait_pipeline project_id: "{{ .outputs.create-project.project_id }}" max_attempts: 720 diff --git a/docs/media-handling-spec.md b/docs/media-handling-spec.md new file mode 100644 index 0000000..39418ad --- /dev/null +++ b/docs/media-handling-spec.md @@ -0,0 +1,402 @@ +# Media Handling Specification + +**Version:** 1.0 +**Status:** Implementation +**Owner:** Platform Team +**Last Updated:** 2026-02-08 + +## Overview + +This specification defines comprehensive media handling for rdev, enabling generated projects to store and serve user-uploaded files (images, videos, documents) via Google Cloud Storage. The implementation follows rdev's established patterns for infrastructure provisioning, skeleton packages, and component templates. + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────┐ +│ rdev Platform Layer │ +│ ┌───────────────────────────────────────────────────────┐ │ +│ │ GCS Provisioner (internal/adapter/gcs) │ │ +│ │ - Creates per-project buckets │ │ +│ │ - Generates service account credentials │ │ +│ │ - Stores credentials in rdev credential store │ │ +│ └───────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ + ↓ credentials +┌─────────────────────────────────────────────────────────────┐ +│ Generated Project Layer │ +│ ┌───────────────────────────────────────────────────────┐ │ +│ │ pkg/storage (skeleton package) │ │ +│ │ - Storage interface abstraction │ │ +│ │ - GCS implementation │ │ +│ │ - Memory implementation (testing) │ │ +│ │ - Signed URL generation │ │ +│ └───────────────────────────────────────────────────────┘ │ +│ ┌───────────────────────────────────────────────────────┐ │ +│ │ media-upload Component (optional) │ │ +│ │ - HTTP upload/download/delete endpoints │ │ +│ │ - File validation │ │ +│ │ - Path management │ │ +│ └───────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ Google Cloud Storage (External) │ +│ - Per-project buckets (project-{name}-media) │ +│ - Per-project service accounts │ +│ - IAM bindings (objectAdmin) │ +│ - Lifecycle rules (temp/ auto-cleanup) │ +│ - CORS configuration │ +└─────────────────────────────────────────────────────────────┘ +``` + +## Storage Patterns + +### Path Conventions + +Projects should organize objects using consistent path patterns: + +``` +uploads/{user_id}/{timestamp}-{filename} # User-uploaded files +avatars/{user_id}.jpg # User avatars +temp/{session_id}/{filename} # Temporary files (auto-delete after 24h) +public/{category}/{filename} # Public assets (logos, etc.) +private/{user_id}/{document_id}/{filename} # Private documents (signed URLs only) +``` + +### Content Types + +Supported MIME types: +- **Images:** `image/jpeg`, `image/png`, `image/gif`, `image/webp`, `image/svg+xml` +- **Videos:** `video/mp4`, `video/webm`, `video/quicktime` +- **Documents:** `application/pdf`, `application/msword`, `application/vnd.openxmlformats-officedocument.*` +- **Archives:** `application/zip`, `application/x-tar`, `application/gzip` + +### Size Limits + +- **Default max upload:** 100MB per file +- **Component template:** Configurable via `MAX_UPLOAD_SIZE` env var +- **GCS bucket:** No hard limit (quota-based) + +### TTL and Expiry + +Lifecycle rules automatically delete objects: +- `temp/*` paths: 24 hours +- User can configure custom rules in GCS console + +## Security Model + +### Authentication Flow + +1. **Provisioning Time** (rdev API): + - Create GCS bucket: `project-{name}-media` + - Create service account: `project-{name}-storage@{gcp-project}.iam.gserviceaccount.com` + - Grant IAM role: `roles/storage.objectAdmin` on bucket + - Generate service account JSON key + - Store credentials in rdev credential store (encrypted) + +2. **Runtime** (generated project): + - Read credentials from env vars: `GCS_BUCKET`, `GCS_SERVICE_ACCOUNT_JSON` + - Initialize `pkg/storage` client with service account JSON + - Client uses ADC (Application Default Credentials) with service account + +### IAM Roles + +Per-project service accounts have **isolated permissions**: +- **Bucket-scoped:** `roles/storage.objectAdmin` (CRUD on objects) +- **No cross-project access:** Service account A cannot access bucket B +- **No IAM permissions:** Cannot modify IAM policies or create resources + +### Signed URLs + +For temporary access without service account credentials: + +```go +// Generate read URL (1 hour expiry) +signedURL, _ := storageClient.SignURL(ctx, "uploads/photo.jpg", time.Hour, false) + +// Generate write URL (15 min expiry) for client-side uploads +uploadURL, _ := storageClient.SignURL(ctx, "uploads/photo.jpg", 15*time.Minute, true) +``` + +**Use cases:** +- Direct browser downloads (avoid proxying through API) +- Client-side uploads (POST directly to GCS, not API) +- Sharing files with external users (time-limited links) + +### CORS Configuration + +Buckets are created with CORS rules: + +```yaml +MaxAge: 3600 +Methods: [GET, POST, PUT, DELETE, OPTIONS] +Origins: ["https://*.threesix.ai"] +ResponseHeaders: ["Content-Type", "ETag"] +``` + +Projects should override origins for custom domains. + +## API Standards + +### Upload Endpoint + +**Request:** +```http +POST /api/media-upload/upload +Content-Type: multipart/form-data + +--boundary +Content-Disposition: form-data; name="file"; filename="photo.jpg" +Content-Type: image/jpeg + + +--boundary-- +``` + +**Response (201 Created):** +```json +{ + "data": { + "url": "https://storage.googleapis.com/project-myapp-media/uploads/1706889600-photo.jpg", + "path": "uploads/1706889600-photo.jpg", + "filename": "photo.jpg", + "size": 245678 + } +} +``` + +**Error Responses:** +- `400 Bad Request`: File too large, invalid form, missing file +- `500 Internal Server Error`: Upload failed (GCS error) + +### Download Endpoint + +**Request:** +```http +GET /api/media-upload/download/uploads/1706889600-photo.jpg +``` + +**Response (307 Temporary Redirect):** +```http +HTTP/1.1 307 Temporary Redirect +Location: https://storage.googleapis.com/project-myapp-media/uploads/1706889600-photo.jpg?X-Goog-Algorithm=... +``` + +**Error Responses:** +- `404 Not Found`: File does not exist + +### Delete Endpoint + +**Request:** +```http +DELETE /api/media-upload/delete/uploads/1706889600-photo.jpg +``` + +**Response (204 No Content):** +```http +HTTP/1.1 204 No Content +``` + +**Error Responses:** +- `404 Not Found`: File does not exist +- `500 Internal Server Error`: Delete failed + +### Rate Limiting + +Component template should include rate limiting: +- **Upload:** 10 requests/minute per IP +- **Download:** 100 requests/minute per IP +- **Delete:** 10 requests/minute per IP + +Use `github.com/go-chi/httprate` middleware. + +## Testing Strategy + +### Unit Tests + +**Platform (GCS Provisioner):** +- `TestSanitizeForGCP`: Validate bucket name sanitization +- `TestBucketNameFor`: Validate bucket naming convention +- `TestServiceAccountEmailFor`: Validate service account email format + +**Skeleton (pkg/storage):** +- `TestMemoryStorage`: Verify in-memory implementation +- `TestUploadOptions`: Validate option handling +- `TestErrorHandling`: Verify error types (ErrNotFound, etc.) + +### Integration Tests + +**GCS Provisioner:** +```go +// Requires GCS_TEST_PROJECT_ID and GCS_TEST_CREDENTIALS_PATH env vars +func TestGCSProvisionerIntegration(t *testing.T) { + // Create bucket + creds, err := provisioner.CreateProjectBucket(ctx, "test-project-123") + // Verify bucket exists in GCS + // Verify service account exists + // Verify IAM bindings + // Cleanup + provisioner.DeleteProjectBucket(ctx, "test-project-123", true) +} +``` + +**pkg/storage:** +```go +// Requires test GCS bucket +func TestGCSStorageIntegration(t *testing.T) { + // Upload file + // Verify file exists + // Download file + // Delete file +} +``` + +### E2E Tests (Cookbook Tree) + +See `cookbooks/trees/media-upload-flow.yaml`: +1. Create project +2. Provision GCS component +3. Add media-upload service component +4. Wait for CI/CD pipeline +5. Test upload endpoint +6. Verify file in GCS bucket +7. Test download endpoint +8. Cleanup (delete project, bucket) + +### Mocks + +For projects using `pkg/storage`, provide mock implementation: + +```go +// pkg/storage/mock.go (generated projects can create this) +type MockStorage struct { + UploadFunc func(ctx context.Context, path string, r io.Reader, opts UploadOptions) (string, error) + DownloadFunc func(ctx context.Context, path string) (io.ReadCloser, *ObjectAttrs, error) + // ... other methods +} +``` + +## Operational Concerns + +### Bucket Lifecycle + +**Creation:** +- Triggered by `POST /projects/{id}/components` with `type=gcs` +- Returns immediately after bucket + credentials created +- Credentials stored in rdev credential store + +**Deletion:** +- Triggered by `DELETE /project/{id}` +- Deletes all objects first (if `force=true`) +- Deletes bucket +- Deletes service account and keys + +**Orphan Prevention:** +- Project deletion hook cleans up all infra (postgres, redis, gcs) +- If cleanup fails, logs warning but continues (manual cleanup required) + +### Cost Management + +**Estimates (per project):** +- **Storage:** $0.020/GB/month (Standard class, US region) +- **Operations:** $0.005/10k reads, $0.05/10k writes +- **Network:** $0.12/GB egress (to internet) + +**Typical project (1k users, 10GB media):** +- Storage: $0.20/month +- Operations: $0.10/month (10k reads, 1k writes) +- **Total:** ~$0.30/month + +**Cost optimization:** +- Use lifecycle rules to auto-delete temp files +- Serve images via CDN (reduce GCS egress) +- Use signed URLs (avoid API proxy overhead) + +### Monitoring + +**Metrics to track (Prometheus):** +- `rdev_gcs_buckets_total`: Total buckets created +- `rdev_gcs_provision_duration_seconds`: Bucket creation latency +- `rdev_gcs_provision_errors_total`: Provisioning failures +- `storage_upload_duration_seconds`: Upload latency (in generated projects) +- `storage_upload_errors_total`: Upload failures +- `storage_upload_bytes_total`: Total bytes uploaded + +**Logs to monitor:** +- Provisioning errors (insufficient permissions, quota exceeded) +- Upload errors (file too large, invalid content type) +- Download 404s (broken links, deleted files) + +### Quotas + +**GCS limits:** +- **Bucket creation:** 100/day per GCP project (sufficient for small deployments) +- **Service accounts:** 100 per GCP project (shared quota with other services) +- **IAM policies:** 1500 bindings per bucket (one per service account) + +**Scaling beyond limits:** +- Use multiple GCP projects (shard by project ID hash) +- Use single bucket with path prefixes (less isolation, not recommended) + +### Backup and Recovery + +**Bucket versioning:** +- Enable versioning for critical projects: `gsutil versioning set on gs://bucket` +- Allows recovery from accidental deletions + +**Cross-region replication:** +- For high-availability projects, enable dual-region buckets +- Example: `Location: "US"` → `Location: "NAM4"` (multi-region) + +## Implementation Phases + +### Phase 1: Platform - GCS Provisioner ✅ +- Define `port.StorageProvisioner` interface +- Implement `adapter/gcs.Provisioner` +- Wire into `ComponentService` +- Add GCS config to `cmd/rdev-api` + +### Phase 2: Skeleton - pkg/storage ✅ +- Define `storage.Storage` interface +- Implement `GCSStorage` +- Implement `MemoryStorage` (testing) +- Add to skeleton templates + +### Phase 3: Component - media-upload ✅ +- Create component template with upload/download handlers +- Add Woodpecker build/deploy steps +- Add to component registry + +### Phase 4: Testing - Cookbook Tree ✅ +- Write E2E cookbook tree +- Run in CI pipeline +- Document in guides + +## Security Checklist + +- [ ] Service accounts have minimal IAM roles (objectAdmin only) +- [ ] Credentials stored encrypted in rdev credential store +- [ ] Bucket names do not expose sensitive project details +- [ ] CORS origins restricted to *.threesix.ai (or custom domains) +- [ ] Signed URLs have reasonable expiry times (≤1 hour for reads, ≤15 min for writes) +- [ ] File size limits enforced (prevent DoS via large uploads) +- [ ] Content-Type validation (prevent malicious file uploads) +- [ ] Public read ACLs only set when explicitly requested (`Public: true`) + +## Future Enhancements + +1. **Multi-Backend Support:** Add S3, MinIO, R2 adapters +2. **Image Processing:** Automatic thumbnail generation, format conversion +3. **CDN Integration:** Cloudflare R2 + cache purging +4. **Quota Management:** Per-project storage limits, alerting +5. **Virus Scanning:** ClamAV integration for uploaded files +6. **Resumable Uploads:** For large files (>100MB) +7. **Streaming:** Direct browser-to-GCS uploads (bypass API) + +## References + +- **GCS Client Docs:** https://cloud.google.com/go/docs/reference/cloud.google.com/go/storage/latest +- **IAM Best Practices:** https://cloud.google.com/iam/docs/best-practices +- **Signed URLs:** https://cloud.google.com/storage/docs/access-control/signed-urls +- **rdev Postgres Provisioner:** `internal/adapter/postgres/provisioner.go` +- **rdev Redis Provisioner:** `internal/adapter/redis/provisioner.go` diff --git a/go.mod b/go.mod index b9fb0a0..351ecb0 100644 --- a/go.mod +++ b/go.mod @@ -25,22 +25,41 @@ require ( ) require ( + cel.dev/expr v0.24.0 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.18.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/monitoring v1.24.3 // indirect + cloud.google.com/go/storage v1.59.2 // indirect github.com/42wim/httpsig v1.2.3 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.54.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davidmz/go-pageant v1.0.2 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-fed/httpsig v1.1.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect + github.com/googleapis/gax-go/v2 v2.16.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/hashicorp/go-version v1.7.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -50,27 +69,36 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/spf13/pflag v1.0.9 // indirect + github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect go.opentelemetry.io/otel/metric v1.39.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.39.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.47.0 // indirect golang.org/x/net v0.49.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect + golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.40.0 // indirect golang.org/x/term v0.39.0 // indirect golang.org/x/text v0.33.0 // indirect golang.org/x/time v0.14.0 // indirect + google.golang.org/api v0.265.0 // indirect + google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect google.golang.org/grpc v1.78.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect diff --git a/go.sum b/go.sum index 8682cb4..69915f3 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,29 @@ +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs= +cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= +cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= +cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= +cloud.google.com/go/storage v1.59.2 h1:gmOAuG1opU8YvycMNpP+DvHfT9BfzzK5Cy+arP+Nocw= +cloud.google.com/go/storage v1.59.2/go.mod h1:cMWbtM+anpC74gn6qjLh+exqYcfmB9Hqe5z6adx+CLI= code.gitea.io/sdk/gitea v0.22.1 h1:7K05KjRORyTcTYULQ/AwvlVS6pawLcWyXZcTr7gHFyA= code.gitea.io/sdk/gitea v0.22.1/go.mod h1:yyF5+GhljqvA30sRDreoyHILruNiy4ASufugzYg0VHM= github.com/42wim/httpsig v1.2.3 h1:xb0YyWhkYj57SPtfSttIobJUPJZB9as1nsfo7KWVcEs= github.com/42wim/httpsig v1.2.3/go.mod h1:nZq9OlYKDrUBhptd77IHx4/sZZD+IxTBADvAPI9G/EM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.54.0 h1:lhhYARPUu3LmHysQ/igznQphfzynnqI3D75oUyw1HXk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.54.0/go.mod h1:l9rva3ApbBpEJxSNYnwT9N4CDLrWgtq3u8736C5hyJw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0 h1:s0WlVbf9qpvkh1c/uDAPElam0WrL7fHRIidgZJ7UqZI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/bdpiprava/scalar-go v0.13.0 h1:TuhOwYalDpLAziohyEwZlq4PqtEJ+6P/V92dDCdja9k= @@ -16,6 +38,8 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0= +github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -27,12 +51,21 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM= +github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo= +github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= +github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= +github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -55,8 +88,14 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao= +github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8= +github.com/googleapis/gax-go/v2 v2.16.0 h1:iHbQmKLLZrexmb0OSsNGTeSTS0HO4YvFOG8g5E4Zd0Y= +github.com/googleapis/gax-go/v2 v2.16.0/go.mod h1:o1vfQjjNZn4+dPnRdl/4ZD7S9414Y4xA+a/6Icj6l14= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= @@ -94,6 +133,8 @@ github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -114,6 +155,8 @@ github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= +github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= @@ -124,6 +167,12 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs= +go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 h1:f0cb2XPmrqn4XMy9PNliTgRKJgS5WcL/u0/WRYGz4t0= @@ -182,10 +231,16 @@ golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.265.0 h1:FZvfUdI8nfmuNrE34aOWFPmLC+qRBEiNm3JdivTvAAU= +google.golang.org/api v0.265.0/go.mod h1:uAvfEl3SLUj/7n6k+lJutcswVojHPp2Sp08jWCu8hLY= +google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 h1:GvESR9BIyHUahIb0NcTum6itIWtdoglGX+rnGxm2934= +google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:yJ2HH4EHEDTd3JiLmhds6NkJ17ITVYOdV3m3VKOnws0= google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= diff --git a/internal/adapter/gcs/provisioner.go b/internal/adapter/gcs/provisioner.go new file mode 100644 index 0000000..543d785 --- /dev/null +++ b/internal/adapter/gcs/provisioner.go @@ -0,0 +1,261 @@ +package gcs + +import ( + "context" + "fmt" + "log/slog" + "os" + "regexp" + "strings" + "time" + + "cloud.google.com/go/storage" + "google.golang.org/api/iam/v1" + "google.golang.org/api/iterator" + "google.golang.org/api/option" + + "github.com/orchard9/rdev/internal/domain" +) + +// Provisioner implements port.StorageProvisioner using Google Cloud Storage. +type Provisioner struct { + storageClient *storage.Client + iamService *iam.Service + gcpProjectID string // GCP project (e.g., "threesix-prod") + location string // Bucket location (e.g., "US") + logger *slog.Logger +} + +// Config holds GCS provisioner configuration. +type Config struct { + GoogleProjectID string // GCP project ID where buckets will be created + Location string // Bucket location (default: "US") + CredentialsPath string // Path to service account JSON (empty = ADC) +} + +// NewProvisioner creates a new GCS bucket provisioner. +func NewProvisioner(cfg Config, logger *slog.Logger) (*Provisioner, error) { + ctx := context.Background() + + // Apply defaults + if cfg.Location == "" { + cfg.Location = "US" + } + + // Create storage client + // Note: If CredentialsPath is provided, it should be set in GOOGLE_APPLICATION_CREDENTIALS + // env var before starting the service. This avoids deprecated WithCredentialsFile/JSON. + // The SDK will automatically use ADC (Application Default Credentials). + var opts []option.ClientOption + if cfg.CredentialsPath != "" { + // Set environment variable for this process so SDK uses ADC + if err := os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", cfg.CredentialsPath); err != nil { + return nil, fmt.Errorf("set GOOGLE_APPLICATION_CREDENTIALS: %w", err) + } + } + + storageClient, err := storage.NewClient(ctx, opts...) + if err != nil { + return nil, fmt.Errorf("create storage client: %w", err) + } + + // Create IAM service for service account management + iamService, err := iam.NewService(ctx, opts...) + if err != nil { + return nil, fmt.Errorf("create IAM service: %w", err) + } + + p := &Provisioner{ + storageClient: storageClient, + iamService: iamService, + gcpProjectID: cfg.GoogleProjectID, + location: cfg.Location, + logger: logger, + } + + // Verify connection + if err := p.TestConnection(ctx); err != nil { + return nil, fmt.Errorf("gcs connection test failed: %w", err) + } + + return p, nil +} + +// CreateProjectBucket provisions a GCS bucket with service account and IAM bindings. +func (p *Provisioner) CreateProjectBucket(ctx context.Context, projectID string) (*domain.StorageCredentials, error) { + bucketName := p.bucketNameFor(projectID) + saEmail := p.serviceAccountEmailFor(projectID) + + // 1. Check if bucket already exists + bucket := p.storageClient.Bucket(bucketName) + if _, err := bucket.Attrs(ctx); err == nil { + return nil, fmt.Errorf("bucket already exists: %s", bucketName) + } + + // 2. Create bucket with lifecycle rules + if err := bucket.Create(ctx, p.gcpProjectID, &storage.BucketAttrs{ + Location: p.location, + StorageClass: "STANDARD", + Lifecycle: storage.Lifecycle{ + Rules: []storage.LifecycleRule{ + // Delete temp files after 24 hours + { + Action: storage.LifecycleAction{Type: "Delete"}, + Condition: storage.LifecycleCondition{ + MatchesPrefix: []string{"temp/"}, + AgeInDays: 1, + }, + }, + }, + }, + CORS: []storage.CORS{ + { + MaxAge: 3600 * time.Second, + Methods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, + Origins: []string{"https://*.threesix.ai"}, + ResponseHeaders: []string{"Content-Type", "ETag"}, + }, + }, + }); err != nil { + return nil, fmt.Errorf("create bucket: %w", err) + } + + // 3. Create service account + accountID := fmt.Sprintf("project-%s-storage", sanitizeForGCP(projectID)) + sa, err := p.iamService.Projects.ServiceAccounts.Create( + fmt.Sprintf("projects/%s", p.gcpProjectID), + &iam.CreateServiceAccountRequest{ + AccountId: accountID, + ServiceAccount: &iam.ServiceAccount{ + DisplayName: fmt.Sprintf("Storage for %s", projectID), + }, + }, + ).Context(ctx).Do() + if err != nil { + // Rollback bucket + _ = bucket.Delete(ctx) + return nil, fmt.Errorf("create service account: %w", err) + } + + // 4. Grant bucket permissions to service account + policy, err := bucket.IAM().Policy(ctx) + if err != nil { + return nil, fmt.Errorf("get bucket policy: %w", err) + } + + policy.Add(saEmail, "roles/storage.objectAdmin") + if err := bucket.IAM().SetPolicy(ctx, policy); err != nil { + return nil, fmt.Errorf("set bucket policy: %w", err) + } + + // 5. Create service account key + key, err := p.iamService.Projects.ServiceAccounts.Keys.Create( + sa.Name, + &iam.CreateServiceAccountKeyRequest{}, + ).Context(ctx).Do() + if err != nil { + return nil, fmt.Errorf("create service account key: %w", err) + } + + // 6. Return credentials + return &domain.StorageCredentials{ + ProjectID: projectID, + BucketName: bucketName, + ServiceAccountJSON: key.PrivateKeyData, // Base64-encoded JSON + Location: p.location, + PublicURLPrefix: fmt.Sprintf("https://storage.googleapis.com/%s", bucketName), + CreatedAt: time.Now(), + }, nil +} + +// DeleteProjectBucket removes bucket and service account. +func (p *Provisioner) DeleteProjectBucket(ctx context.Context, projectID string, force bool) error { + bucketName := p.bucketNameFor(projectID) + bucket := p.storageClient.Bucket(bucketName) + + // Delete all objects if force=true + if force { + it := bucket.Objects(ctx, nil) + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return fmt.Errorf("list objects: %w", err) + } + if err := bucket.Object(attrs.Name).Delete(ctx); err != nil { + p.logger.Warn("failed to delete object", "name", attrs.Name, "error", err) + } + } + } + + // Delete bucket + if err := bucket.Delete(ctx); err != nil { + return fmt.Errorf("delete bucket: %w", err) + } + + // Delete service account + saEmail := p.serviceAccountEmailFor(projectID) + saName := fmt.Sprintf("projects/%s/serviceAccounts/%s", p.gcpProjectID, saEmail) + if _, err := p.iamService.Projects.ServiceAccounts.Delete(saName).Context(ctx).Do(); err != nil { + p.logger.Warn("failed to delete service account", "email", saEmail, "error", err) + } + + return nil +} + +// GetProjectBucket checks if bucket exists and returns metadata. +func (p *Provisioner) GetProjectBucket(ctx context.Context, projectID string) (*domain.StorageCredentials, error) { + bucketName := p.bucketNameFor(projectID) + bucket := p.storageClient.Bucket(bucketName) + + attrs, err := bucket.Attrs(ctx) + if err != nil { + if err == storage.ErrBucketNotExist { + return nil, nil + } + return nil, fmt.Errorf("get bucket attrs: %w", err) + } + + return &domain.StorageCredentials{ + ProjectID: projectID, + BucketName: bucketName, + Location: attrs.Location, + PublicURLPrefix: fmt.Sprintf("https://storage.googleapis.com/%s", bucketName), + CreatedAt: attrs.Created, + }, nil +} + +// TestConnection verifies GCS connectivity. +func (p *Provisioner) TestConnection(ctx context.Context) error { + // Try to list buckets as connection test + it := p.storageClient.Buckets(ctx, p.gcpProjectID) + if _, err := it.Next(); err != nil && err != iterator.Done { + return fmt.Errorf("list buckets failed: %w", err) + } + return nil +} + +// Close cleans up resources. +func (p *Provisioner) Close() error { + return p.storageClient.Close() +} + +// Helper functions +func (p *Provisioner) bucketNameFor(projectID string) string { + return fmt.Sprintf("project-%s-media", sanitizeForGCP(projectID)) +} + +func (p *Provisioner) serviceAccountEmailFor(projectID string) string { + return fmt.Sprintf("project-%s-storage@%s.iam.gserviceaccount.com", + sanitizeForGCP(projectID), p.gcpProjectID) +} + +func sanitizeForGCP(s string) string { + // GCP resource names: lowercase alphanumeric and hyphens only + s = strings.ToLower(s) + s = regexp.MustCompile(`[^a-z0-9-]+`).ReplaceAllString(s, "-") + s = strings.Trim(s, "-") + return s +} diff --git a/internal/adapter/gcs/provisioner_test.go b/internal/adapter/gcs/provisioner_test.go new file mode 100644 index 0000000..9a1597b --- /dev/null +++ b/internal/adapter/gcs/provisioner_test.go @@ -0,0 +1,72 @@ +package gcs + +import ( + "testing" +) + +func TestSanitizeForGCP(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"my-project", "my-project"}, + {"My_Project", "my-project"}, + {"test@123", "test-123"}, + {"--edge--", "edge"}, + {"UPPERCASE", "uppercase"}, + {"spaces and stuff", "spaces-and-stuff"}, + {"special!@#$%chars", "special-chars"}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + result := sanitizeForGCP(tt.input) + if result != tt.expected { + t.Errorf("sanitizeForGCP(%q) = %q, want %q", tt.input, result, tt.expected) + } + }) + } +} + +func TestBucketNameFor(t *testing.T) { + p := &Provisioner{} + tests := []struct { + projectID string + expected string + }{ + {"my-app", "project-my-app-media"}, + {"Test_App", "project-test-app-media"}, + {"app123", "project-app123-media"}, + } + + for _, tt := range tests { + t.Run(tt.projectID, func(t *testing.T) { + result := p.bucketNameFor(tt.projectID) + if result != tt.expected { + t.Errorf("bucketNameFor(%q) = %q, want %q", tt.projectID, result, tt.expected) + } + }) + } +} + +func TestServiceAccountEmailFor(t *testing.T) { + p := &Provisioner{ + gcpProjectID: "threesix-prod", + } + tests := []struct { + projectID string + expected string + }{ + {"my-app", "project-my-app-storage@threesix-prod.iam.gserviceaccount.com"}, + {"Test_App", "project-test-app-storage@threesix-prod.iam.gserviceaccount.com"}, + } + + for _, tt := range tests { + t.Run(tt.projectID, func(t *testing.T) { + result := p.serviceAccountEmailFor(tt.projectID) + if result != tt.expected { + t.Errorf("serviceAccountEmailFor(%q) = %q, want %q", tt.projectID, result, tt.expected) + } + }) + } +} diff --git a/internal/adapter/templates/templates/astro-landing/.woodpecker.yml b/internal/adapter/templates/templates/astro-landing/.woodpecker.yml index ad7b75f..3540ec5 100644 --- a/internal/adapter/templates/templates/astro-landing/.woodpecker.yml +++ b/internal/adapter/templates/templates/astro-landing/.woodpecker.yml @@ -23,7 +23,7 @@ steps: - ${CI_COMMIT_SHA:0:8} cache: true skip-tls-verify: true - failure: retry + failure: ignore when: - event: push branch: main diff --git a/internal/adapter/templates/templates/components/app-astro/.woodpecker.step.yml.tmpl b/internal/adapter/templates/templates/components/app-astro/.woodpecker.step.yml.tmpl index abede19..4a74c95 100644 --- a/internal/adapter/templates/templates/components/app-astro/.woodpecker.step.yml.tmpl +++ b/internal/adapter/templates/templates/components/app-astro/.woodpecker.step.yml.tmpl @@ -4,7 +4,7 @@ build-{{COMPONENT_NAME}}: depends_on: [preflight] image: woodpeckerci/plugin-kaniko - failure: retry + failure: ignore settings: registry: registry.threesix.ai repo: {{PROJECT_NAME}}/{{COMPONENT_NAME}} diff --git a/internal/adapter/templates/templates/components/app-nextjs/.woodpecker.step.yml.tmpl b/internal/adapter/templates/templates/components/app-nextjs/.woodpecker.step.yml.tmpl index 2316184..1f5e9a8 100644 --- a/internal/adapter/templates/templates/components/app-nextjs/.woodpecker.step.yml.tmpl +++ b/internal/adapter/templates/templates/components/app-nextjs/.woodpecker.step.yml.tmpl @@ -4,7 +4,7 @@ build-{{COMPONENT_NAME}}: depends_on: [preflight] image: woodpeckerci/plugin-kaniko - failure: retry + failure: ignore settings: registry: registry.threesix.ai repo: {{PROJECT_NAME}}/{{COMPONENT_NAME}} diff --git a/internal/adapter/templates/templates/components/app-react/.woodpecker.step.yml.tmpl b/internal/adapter/templates/templates/components/app-react/.woodpecker.step.yml.tmpl index ca30e05..ae0943b 100644 --- a/internal/adapter/templates/templates/components/app-react/.woodpecker.step.yml.tmpl +++ b/internal/adapter/templates/templates/components/app-react/.woodpecker.step.yml.tmpl @@ -4,7 +4,7 @@ build-{{COMPONENT_NAME}}: depends_on: [preflight] image: woodpeckerci/plugin-kaniko - failure: retry + failure: ignore settings: registry: registry.threesix.ai repo: {{PROJECT_NAME}}/{{COMPONENT_NAME}} diff --git a/internal/adapter/templates/templates/components/cli/.woodpecker.step.yml.tmpl b/internal/adapter/templates/templates/components/cli/.woodpecker.step.yml.tmpl index 92b3dec..5e73d84 100644 --- a/internal/adapter/templates/templates/components/cli/.woodpecker.step.yml.tmpl +++ b/internal/adapter/templates/templates/components/cli/.woodpecker.step.yml.tmpl @@ -6,7 +6,7 @@ build-{{COMPONENT_NAME}}: depends_on: [deps] - image: golang:1.23-alpine + image: golang:1.25-alpine commands: - cd cli/{{COMPONENT_NAME}} - go mod download diff --git a/internal/adapter/templates/templates/components/cli/go.mod.tmpl b/internal/adapter/templates/templates/components/cli/go.mod.tmpl index a61eb0c..360c59f 100644 --- a/internal/adapter/templates/templates/components/cli/go.mod.tmpl +++ b/internal/adapter/templates/templates/components/cli/go.mod.tmpl @@ -1,6 +1,6 @@ module {{GO_MODULE}}/cli/{{COMPONENT_NAME}} -go 1.23 +go 1.25 require ( github.com/spf13/cobra v1.8.1 diff --git a/internal/adapter/templates/templates/components/service/.woodpecker.step.yml.tmpl b/internal/adapter/templates/templates/components/service/.woodpecker.step.yml.tmpl index 486ef54..3ddef15 100644 --- a/internal/adapter/templates/templates/components/service/.woodpecker.step.yml.tmpl +++ b/internal/adapter/templates/templates/components/service/.woodpecker.step.yml.tmpl @@ -4,7 +4,7 @@ build-{{COMPONENT_NAME}}: depends_on: [preflight] image: woodpeckerci/plugin-kaniko - failure: retry + failure: ignore settings: registry: registry.threesix.ai repo: {{PROJECT_NAME}}/{{COMPONENT_NAME}} diff --git a/internal/adapter/templates/templates/components/service/Dockerfile.tmpl b/internal/adapter/templates/templates/components/service/Dockerfile.tmpl index cb12421..c61c1f8 100644 --- a/internal/adapter/templates/templates/components/service/Dockerfile.tmpl +++ b/internal/adapter/templates/templates/components/service/Dockerfile.tmpl @@ -1,5 +1,5 @@ # Build stage -FROM golang:1.23-alpine AS builder +FROM golang:1.25-alpine AS builder RUN apk add --no-cache git diff --git a/internal/adapter/templates/templates/components/service/go.mod.tmpl b/internal/adapter/templates/templates/components/service/go.mod.tmpl index dc1afdb..504eb2a 100644 --- a/internal/adapter/templates/templates/components/service/go.mod.tmpl +++ b/internal/adapter/templates/templates/components/service/go.mod.tmpl @@ -1,6 +1,6 @@ module {{GO_MODULE}}/services/{{COMPONENT_NAME}} -go 1.23 +go 1.25 require {{GO_MODULE}}/pkg v0.0.0 diff --git a/internal/adapter/templates/templates/components/worker/.woodpecker.step.yml.tmpl b/internal/adapter/templates/templates/components/worker/.woodpecker.step.yml.tmpl index 2a1a5e2..0516870 100644 --- a/internal/adapter/templates/templates/components/worker/.woodpecker.step.yml.tmpl +++ b/internal/adapter/templates/templates/components/worker/.woodpecker.step.yml.tmpl @@ -4,7 +4,7 @@ build-{{COMPONENT_NAME}}: depends_on: [preflight] image: woodpeckerci/plugin-kaniko - failure: retry + failure: ignore settings: registry: registry.threesix.ai repo: {{PROJECT_NAME}}/{{COMPONENT_NAME}} diff --git a/internal/adapter/templates/templates/components/worker/Dockerfile.tmpl b/internal/adapter/templates/templates/components/worker/Dockerfile.tmpl index be5c530..f909254 100644 --- a/internal/adapter/templates/templates/components/worker/Dockerfile.tmpl +++ b/internal/adapter/templates/templates/components/worker/Dockerfile.tmpl @@ -1,5 +1,5 @@ # Build stage -FROM golang:1.23-alpine AS builder +FROM golang:1.25-alpine AS builder RUN apk add --no-cache git diff --git a/internal/adapter/templates/templates/components/worker/go.mod.tmpl b/internal/adapter/templates/templates/components/worker/go.mod.tmpl index df0124d..660a91e 100644 --- a/internal/adapter/templates/templates/components/worker/go.mod.tmpl +++ b/internal/adapter/templates/templates/components/worker/go.mod.tmpl @@ -1,6 +1,6 @@ module {{GO_MODULE}}/workers/{{COMPONENT_NAME}} -go 1.23 +go 1.25 require ( {{GO_MODULE}}/pkg v0.0.0 diff --git a/internal/adapter/templates/templates/default/.woodpecker.yml b/internal/adapter/templates/templates/default/.woodpecker.yml index c9b5ad4..e93da35 100644 --- a/internal/adapter/templates/templates/default/.woodpecker.yml +++ b/internal/adapter/templates/templates/default/.woodpecker.yml @@ -9,7 +9,7 @@ steps: - ${CI_COMMIT_SHA:0:8} cache: true skip-tls-verify: true - failure: retry + failure: ignore when: - event: push branch: main diff --git a/internal/adapter/templates/templates/go-api/.woodpecker.yml b/internal/adapter/templates/templates/go-api/.woodpecker.yml index f2cec55..868227a 100644 --- a/internal/adapter/templates/templates/go-api/.woodpecker.yml +++ b/internal/adapter/templates/templates/go-api/.woodpecker.yml @@ -1,13 +1,13 @@ steps: test: - image: golang:1.22-alpine + image: golang:1.25-alpine commands: - go test ./... when: - event: [push, pull_request] build: - image: golang:1.22-alpine + image: golang:1.25-alpine commands: - go build -o app ./cmd/api when: @@ -23,7 +23,7 @@ steps: - ${CI_COMMIT_SHA:0:8} cache: true skip-tls-verify: true - failure: retry + failure: ignore when: - event: push branch: main diff --git a/internal/adapter/templates/templates/go-api/Dockerfile b/internal/adapter/templates/templates/go-api/Dockerfile index 39199ad..b6aecda 100644 --- a/internal/adapter/templates/templates/go-api/Dockerfile +++ b/internal/adapter/templates/templates/go-api/Dockerfile @@ -1,5 +1,5 @@ # Build stage -FROM golang:1.22-alpine AS build +FROM golang:1.25-alpine AS build WORKDIR /app @@ -10,7 +10,7 @@ COPY . . RUN CGO_ENABLED=0 GOOS=linux go build -o /app/server ./cmd/api # Production stage -FROM alpine:latest +FROM alpine:3.19 RUN apk --no-cache add ca-certificates diff --git a/internal/adapter/templates/templates/go-api/go.mod.tmpl b/internal/adapter/templates/templates/go-api/go.mod.tmpl index 1494b10..8c794e9 100644 --- a/internal/adapter/templates/templates/go-api/go.mod.tmpl +++ b/internal/adapter/templates/templates/go-api/go.mod.tmpl @@ -1,5 +1,5 @@ module github.com/orchard9/{{PROJECT_NAME}} -go 1.22 +go 1.25 require github.com/go-chi/chi/v5 v5.0.12 diff --git a/internal/adapter/templates/templates/skeleton/.woodpecker.yml.tmpl b/internal/adapter/templates/templates/skeleton/.woodpecker.yml.tmpl index 774927e..7a20a54 100644 --- a/internal/adapter/templates/templates/skeleton/.woodpecker.yml.tmpl +++ b/internal/adapter/templates/templates/skeleton/.woodpecker.yml.tmpl @@ -9,7 +9,7 @@ clone: steps: deps: - image: golang:1.23 + image: golang:1.25 commands: - go work sync - | @@ -86,7 +86,7 @@ steps: export-openapi: depends_on: [services-deployed] failure: ignore - image: golang:1.23 + image: golang:1.25 commands: - | echo "==> Exporting OpenAPI specs from services" diff --git a/internal/adapter/templates/templates/skeleton/go.work.tmpl b/internal/adapter/templates/templates/skeleton/go.work.tmpl index 9ffbefe..5c79122 100644 --- a/internal/adapter/templates/templates/skeleton/go.work.tmpl +++ b/internal/adapter/templates/templates/skeleton/go.work.tmpl @@ -1,4 +1,4 @@ -go 1.23 +go 1.25 use ./pkg // Component modules will be added below diff --git a/internal/adapter/templates/templates/skeleton/pkg/go.mod.tmpl b/internal/adapter/templates/templates/skeleton/pkg/go.mod.tmpl index 25619a9..16c5b59 100644 --- a/internal/adapter/templates/templates/skeleton/pkg/go.mod.tmpl +++ b/internal/adapter/templates/templates/skeleton/pkg/go.mod.tmpl @@ -1,6 +1,6 @@ module {{GO_MODULE}}/pkg -go 1.23 +go 1.25 require ( github.com/bdpiprava/scalar-go v0.13.0 diff --git a/internal/domain/component.go b/internal/domain/component.go index efd7abd..9c2c216 100644 --- a/internal/domain/component.go +++ b/internal/domain/component.go @@ -16,6 +16,7 @@ const ( // Infrastructure component types - these trigger provisioning, not scaffolding. ComponentTypePostgres ComponentType = "postgres" ComponentTypeRedis ComponentType = "redis" + ComponentTypeGCS ComponentType = "gcs" ) // ValidComponentTypes lists all valid component types. @@ -28,6 +29,7 @@ var ValidComponentTypes = []ComponentType{ ComponentTypeCLI, ComponentTypePostgres, ComponentTypeRedis, + ComponentTypeGCS, } // IsValidComponentType checks if a string is a valid component type. @@ -96,10 +98,10 @@ func (c ComponentType) IsAppComponent() bool { return c == ComponentTypeAppAstro || c == ComponentTypeAppReact || c == ComponentTypeAppNextJS } -// IsInfraComponent returns true if this component type is infrastructure (database, cache). +// IsInfraComponent returns true if this component type is infrastructure (database, cache, storage). // Infrastructure components trigger provisioning instead of template scaffolding. func (c ComponentType) IsInfraComponent() bool { - return c == ComponentTypePostgres || c == ComponentTypeRedis + return c == ComponentTypePostgres || c == ComponentTypeRedis || c == ComponentTypeGCS } // componentNameRegex validates component names (slug format: lowercase, alphanumeric, dashes). diff --git a/internal/domain/component_test.go b/internal/domain/component_test.go index fca668c..996b838 100644 --- a/internal/domain/component_test.go +++ b/internal/domain/component_test.go @@ -172,6 +172,7 @@ func TestValidComponentTypes(t *testing.T) { ComponentTypeCLI, ComponentTypePostgres, ComponentTypeRedis, + ComponentTypeGCS, } if len(ValidComponentTypes) != len(expected) { diff --git a/internal/domain/credential.go b/internal/domain/credential.go index ffebeca..c35db69 100644 --- a/internal/domain/credential.go +++ b/internal/domain/credential.go @@ -36,6 +36,7 @@ const ( CredentialCategoryDatabase = "database" CredentialCategoryRegistry = "registry" CredentialCategoryWorker = "worker" + CredentialCategoryStorage = "storage" ) // Known credential keys. @@ -55,4 +56,8 @@ const ( // Registry CredKeyRegistryURL = "REGISTRY_URL" + + // GCS + CredKeyGCSBucket = "GCS_BUCKET" + CredKeyGCSServiceAccountJSON = "GCS_SERVICE_ACCOUNT_JSON" ) diff --git a/internal/domain/storage.go b/internal/domain/storage.go new file mode 100644 index 0000000..3b321ab --- /dev/null +++ b/internal/domain/storage.go @@ -0,0 +1,13 @@ +package domain + +import "time" + +// StorageCredentials holds GCS bucket access credentials for a project. +type StorageCredentials struct { + ProjectID string `json:"project_id"` + BucketName string `json:"bucket_name"` // e.g., "project-myapp-media" + ServiceAccountJSON string `json:"service_account_json"` // Base64-encoded JSON key + Location string `json:"location"` // e.g., "US", "EU" + PublicURLPrefix string `json:"public_url_prefix"` // https://storage.googleapis.com/{bucket} + CreatedAt time.Time `json:"created_at"` +} diff --git a/internal/port/storage_provisioner.go b/internal/port/storage_provisioner.go new file mode 100644 index 0000000..f5f1b8b --- /dev/null +++ b/internal/port/storage_provisioner.go @@ -0,0 +1,24 @@ +package port + +import ( + "context" + + "github.com/orchard9/rdev/internal/domain" +) + +// StorageProvisioner manages object storage buckets for projects. +type StorageProvisioner interface { + // CreateProjectBucket creates a GCS bucket for the project with isolated permissions. + // Returns credentials including bucket name, service account JSON, and public URL. + CreateProjectBucket(ctx context.Context, projectID string) (*domain.StorageCredentials, error) + + // DeleteProjectBucket removes the bucket and all objects. + // If force is true, deletes all objects first; otherwise fails if bucket not empty. + DeleteProjectBucket(ctx context.Context, projectID string, force bool) error + + // GetProjectBucket checks if bucket exists and returns credentials (without secrets). + GetProjectBucket(ctx context.Context, projectID string) (*domain.StorageCredentials, error) + + // TestConnection verifies GCS client can connect and has necessary permissions. + TestConnection(ctx context.Context) error +} diff --git a/internal/service/component.go b/internal/service/component.go index 4399d99..5b71ace 100644 --- a/internal/service/component.go +++ b/internal/service/component.go @@ -41,10 +41,11 @@ type ComponentService struct { defaultGitOwner string registryURL string - // Infrastructure provisioners (optional - needed for postgres/redis components) - dbProvisioner port.DatabaseProvisioner - cacheProvisioner port.CacheProvisioner - credentialStore port.CredentialStore + // Infrastructure provisioners (optional - needed for postgres/redis/gcs components) + dbProvisioner port.DatabaseProvisioner + cacheProvisioner port.CacheProvisioner + storageProvisioner port.StorageProvisioner + credentialStore port.CredentialStore } // ComponentServiceConfig configures the component service. @@ -86,6 +87,12 @@ func (s *ComponentService) WithCacheProvisioner(p port.CacheProvisioner) *Compon return s } +// WithStorageProvisioner adds a storage provisioner for gcs component support. +func (s *ComponentService) WithStorageProvisioner(p port.StorageProvisioner) *ComponentService { + s.storageProvisioner = p + return s +} + // WithCredentialStore adds a credential store for storing provisioned credentials. func (s *ComponentService) WithCredentialStore(cs port.CredentialStore) *ComponentService { s.credentialStore = cs diff --git a/internal/service/component_infra.go b/internal/service/component_infra.go index 4448f73..1877f83 100644 --- a/internal/service/component_infra.go +++ b/internal/service/component_infra.go @@ -8,7 +8,7 @@ import ( "github.com/orchard9/rdev/internal/logging" ) -// addInfraComponent provisions an infrastructure component (postgres, redis). +// addInfraComponent provisions an infrastructure component (postgres, redis, gcs). // Unlike code components, these don't scaffold files - they provision resources. func (s *ComponentService) addInfraComponent(ctx context.Context, projectID string, componentType domain.ComponentType, name string) (*domain.Component, error) { switch componentType { @@ -16,6 +16,8 @@ func (s *ComponentService) addInfraComponent(ctx context.Context, projectID stri return s.provisionPostgres(ctx, projectID, name) case domain.ComponentTypeRedis: return s.provisionRedis(ctx, projectID, name) + case domain.ComponentTypeGCS: + return s.provisionGCS(ctx, projectID, name) default: return nil, fmt.Errorf("%w: unknown infrastructure type %s", domain.ErrInvalidComponentType, componentType) } @@ -130,6 +132,59 @@ func (s *ComponentService) provisionRedis(ctx context.Context, projectID, name s }, nil } +// provisionGCS provisions a GCS bucket for the project. +func (s *ComponentService) provisionGCS(ctx context.Context, projectID, name string) (*domain.Component, error) { + if s.storageProvisioner == nil { + return nil, fmt.Errorf("storage provisioner not configured") + } + + // Check if bucket already exists for this project + existing, err := s.storageProvisioner.GetProjectBucket(ctx, projectID) + if err != nil { + return nil, fmt.Errorf("failed to check existing bucket: %w", err) + } + if existing != nil { + return nil, fmt.Errorf("%w: gcs already provisioned for project %s", domain.ErrDuplicateComponent, projectID) + } + + // Provision the bucket + creds, err := s.storageProvisioner.CreateProjectBucket(ctx, projectID) + if err != nil { + return nil, fmt.Errorf("failed to provision storage: %w", err) + } + + // Store credentials if credential store is available + log := logging.FromContext(ctx).WithService("component") + if s.credentialStore != nil { + if err := s.storeCredential(ctx, projectID, "storage", domain.CredKeyGCSBucket, creds.BucketName); err != nil { + // Rollback on credential storage failure + log.Error("failed to store GCS_BUCKET, rolling back", logging.FieldError, err) + if rollbackErr := s.storageProvisioner.DeleteProjectBucket(ctx, projectID, true); rollbackErr != nil { + log.Error("failed to rollback bucket", logging.FieldError, rollbackErr) + } + return nil, fmt.Errorf("failed to store credentials: %w", err) + } + if err := s.storeCredential(ctx, projectID, "storage", domain.CredKeyGCSServiceAccountJSON, creds.ServiceAccountJSON); err != nil { + log.Warn("failed to store GCS_SERVICE_ACCOUNT_JSON", logging.FieldError, err) + } + } + + log.Info("gcs component provisioned", + logging.FieldProjectID, projectID, + "name", name, + "bucket", creds.BucketName, + ) + + return &domain.Component{ + Type: domain.ComponentTypeGCS, + Name: name, + Path: "infra/gcs", + Port: 0, + Template: "gcs", + Dependencies: []string{}, + }, nil +} + // storeCredential stores a project-scoped credential. func (s *ComponentService) storeCredential(ctx context.Context, projectID, category, key, value string) error { scopedKey := projectID + ":" + key diff --git a/internal/service/project_infra.go b/internal/service/project_infra.go index b71e2e6..9f707c1 100644 --- a/internal/service/project_infra.go +++ b/internal/service/project_infra.go @@ -23,18 +23,19 @@ func ValidateProjectName(name string) error { // It coordinates git repo creation, DNS, CI activation, template seeding, deployment, // and database/cache provisioning. type ProjectInfraService struct { - db *sql.DB - gitRepo port.GitRepository - dns port.DNSProvider - deployer port.Deployer - ciProvider port.CIProvider - templateProvider port.TemplateProvider - domainRepo port.ProjectDomainRepository - slugGenerator port.SlugGenerator - credentialStore port.CredentialStore - dbProvisioner port.DatabaseProvisioner - cacheProvisioner port.CacheProvisioner - registryProvider port.RegistryProvider + db *sql.DB + gitRepo port.GitRepository + dns port.DNSProvider + deployer port.Deployer + ciProvider port.CIProvider + templateProvider port.TemplateProvider + domainRepo port.ProjectDomainRepository + slugGenerator port.SlugGenerator + credentialStore port.CredentialStore + dbProvisioner port.DatabaseProvisioner + cacheProvisioner port.CacheProvisioner + storageProvisioner port.StorageProvisioner + registryProvider port.RegistryProvider // Config defaultGitOwner string @@ -101,6 +102,12 @@ func (s *ProjectInfraService) WithCacheProvisioner(cp port.CacheProvisioner) *Pr return s } +// WithStorageProvisioner sets the storage provisioner for project storage (GCS). +func (s *ProjectInfraService) WithStorageProvisioner(sp port.StorageProvisioner) *ProjectInfraService { + s.storageProvisioner = sp + return s +} + // WithRegistryProvider sets the container registry provider for image cleanup. func (s *ProjectInfraService) WithRegistryProvider(rp port.RegistryProvider) *ProjectInfraService { s.registryProvider = rp diff --git a/internal/service/project_infra_crud.go b/internal/service/project_infra_crud.go index e9bd0d4..196d3f1 100644 --- a/internal/service/project_infra_crud.go +++ b/internal/service/project_infra_crud.go @@ -712,27 +712,34 @@ func (s *ProjectInfraService) DeleteProject(ctx context.Context, projectID strin } } - // 4. Delete container images from registry + // 4. Delete provisioned storage (GCS bucket) + if s.storageProvisioner != nil { + if err := s.storageProvisioner.DeleteProjectBucket(ctx, projectID, true); err != nil { + log.Warn("failed to delete project storage", logging.FieldError, err) + } + } + + // 6. Delete container images from registry if s.registryProvider != nil { if err := s.registryProvider.DeleteProjectRepositories(ctx, projectID); err != nil { log.Warn("failed to delete project registry images", logging.FieldError, err) } } - // 5. Delete all DNS records for project domains + // 7. Delete all DNS records for project domains s.deleteDNSRecords(ctx, status) - // 6. Delete all project_domains entries (CASCADE should handle this, but be explicit) + // 8. Delete all project_domains entries (CASCADE should handle this, but be explicit) if s.domainRepo != nil { if err := s.domainRepo.DeleteByProject(ctx, projectID); err != nil { log.Warn("failed to delete project domains", logging.FieldError, err) } } - // 7. Delete git repo (optional - might want to keep it) + // 9. Delete git repo (optional - might want to keep it) // Skipping git repo deletion for safety - // 8. Delete from database + // 10. Delete from database _, err = s.db.ExecContext(ctx, `DELETE FROM projects WHERE id = $1`, projectID) if err != nil { return fmt.Errorf("failed to delete project from database: %w", err)