From 592b2d5ec0c45ab1b4ab944c4b2aed68e4aafdef Mon Sep 17 00:00:00 2001 From: jordan Date: Thu, 19 Feb 2026 23:13:21 -0700 Subject: [PATCH] fix: clarify database types across docs and fix video storage persistence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two distinct fixes: 1. Database terminology: Make it crystal clear that generated projects use CockroachDB in production and PostgreSQL for local dev, while the rdev platform itself uses PostgreSQL. Updated 15 files across skeleton agents, component templates, cookbook trees, and platform docs. 2. Video storage: VideoHandler was ignoring vid.Data bytes (already downloaded by the Gemini adapter with auth) and re-downloading from the provider URL with a plain GET — which fails because Gemini URLs require API key auth. Now uses vid.Data first, falls back to downloadURL only for public URLs. Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 10 ++++++++ cookbooks/trees/aeries-1-genesis.yaml | 2 +- cookbooks/trees/foundary.yaml | 8 +++---- .../slackpath-1-authenticated-service.yaml | 2 +- docs/media-handling-spec.md | 4 ++-- .../components/service/.env.example.tmpl | 2 ++ .../components/worker/.env.example.tmpl | 2 ++ .../.claude/agents/database-architect.md | 11 ++++++--- .../.claude/agents/queue-specialist.md | 2 +- .../.claude/agents/worker-specialist.md | 2 +- .../skills/knowledge-librarian/SKILL.md | 2 +- .../templates/skeleton/CLAUDE.md.tmpl | 1 + .../skeleton/docker-compose.yml.tmpl | 3 +++ .../skeleton/pkg/database/db.go.tmpl | 10 +++++--- .../skeleton/pkg/generation/handlers.go.tmpl | 24 ++++++++++++++----- 15 files changed, 62 insertions(+), 23 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a930e1a..e6e08c5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,6 +14,16 @@ Run Claude Code instances in isolated Kubernetes pods with REST API control. Ena When discussing code: "add to **platform**" = edit rdev; "add to **skeleton**" = edit project templates. +### Database Rule + +| Context | Database | Details | +|---------|----------|---------| +| **rdev platform** | PostgreSQL | API keys, audit logs, work queue, credentials (`internal/adapter/postgres/`) | +| **Generated projects (production)** | CockroachDB | Provisioned per-project by rdev (`internal/adapter/cockroach/`) | +| **Generated projects (local dev)** | PostgreSQL | Via docker-compose, wire-compatible with CockroachDB | + +Both use `lib/pq` driver. The `type: postgres` component API provisions **CockroachDB** in production — the name is a legacy artifact. Skeleton SQL must be compatible with both PostgreSQL and CockroachDB. + ## Find Your Guide | If you need to... | Read this | diff --git a/cookbooks/trees/aeries-1-genesis.yaml b/cookbooks/trees/aeries-1-genesis.yaml index 5e1a21e..de99da9 100644 --- a/cookbooks/trees/aeries-1-genesis.yaml +++ b/cookbooks/trees/aeries-1-genesis.yaml @@ -22,7 +22,7 @@ steps: - domain: .data.domain add-db: - description: Add Postgres + description: Add CockroachDB depends_on: [create-project] action: api method: POST diff --git a/cookbooks/trees/foundary.yaml b/cookbooks/trees/foundary.yaml index 3bc65cc..7caa44a 100644 --- a/cookbooks/trees/foundary.yaml +++ b/cookbooks/trees/foundary.yaml @@ -19,7 +19,7 @@ steps: name: "{{ .vars.project_name }}" description: "Foundary Studio: Task management with Kanban board" template: "skeleton" - prompt: "Set up the monorepo workspace. Ensure the root README describes a task management studio with Kanban board, REST API, and Postgres persistence." + prompt: "Set up the monorepo workspace. Ensure the root README describes a task management studio with Kanban board, REST API, and CockroachDB persistence." auto_commit: true auto_push: true outputs: @@ -59,7 +59,7 @@ steps: poll_interval: 5 add-components: - description: "Add React frontend, API service, and Postgres database" + description: "Add React frontend, API service, and CockroachDB database" depends_on: [wait-setup-hooks] action: api method: POST @@ -101,7 +101,7 @@ steps: method: POST endpoint: "/projects/{{ .outputs.create-project.project_id }}/architect/start" body: - prompt: "I want to build a task management studio. The product needs: 1) Core data models for Task, Project, Label, and Assignment entities with full CRUD stored in Postgres via studio-db, exposed as REST endpoints on studio-api. 2) A React frontend in studio-ui with a Kanban board (drag-and-drop columns: To Do, In Progress, Done), task creation/edit modals, and filtering by label and assignee. Propose the architecture and identify the two MVP features we should build." + prompt: "I want to build a task management studio. The product needs: 1) Core data models for Task, Project, Label, and Assignment entities with full CRUD stored in CockroachDB via studio-db, exposed as REST endpoints on studio-api. 2) A React frontend in studio-ui with a Kanban board (drag-and-drop columns: To Do, In Progress, Done), task creation/edit modals, and filtering by label and assignee. Propose the architecture and identify the two MVP features we should build." outputs: - conversation_id: .data.id @@ -143,7 +143,7 @@ steps: - Assignment: id, task_id, label_id (many-to-many join) 2. Database: Create SQL migrations for all tables with foreign keys and indexes. - Use the studio-db Postgres connection (DATABASE_URL env var). + Use the studio-db CockroachDB connection (DATABASE_URL env var). 3. Repository layer: Implement CRUD operations for each entity using sqlx. diff --git a/cookbooks/trees/slackpath-1-authenticated-service.yaml b/cookbooks/trees/slackpath-1-authenticated-service.yaml index 7df10b9..853b1ce 100644 --- a/cookbooks/trees/slackpath-1-authenticated-service.yaml +++ b/cookbooks/trees/slackpath-1-authenticated-service.yaml @@ -21,7 +21,7 @@ steps: - domain: .data.domain add-db: - description: Add PostgreSQL for user storage + description: Add CockroachDB for user storage depends_on: [create-project] action: api method: POST diff --git a/docs/media-handling-spec.md b/docs/media-handling-spec.md index 39418ad..18a2cd1 100644 --- a/docs/media-handling-spec.md +++ b/docs/media-handling-spec.md @@ -292,7 +292,7 @@ type MockStorage struct { - Deletes service account and keys **Orphan Prevention:** -- Project deletion hook cleans up all infra (postgres, redis, gcs) +- Project deletion hook cleans up all infra (CockroachDB, Redis, GCS) - If cleanup fails, logs warning but continues (manual cleanup required) ### Cost Management @@ -398,5 +398,5 @@ type MockStorage struct { - **GCS Client Docs:** https://cloud.google.com/go/docs/reference/cloud.google.com/go/storage/latest - **IAM Best Practices:** https://cloud.google.com/iam/docs/best-practices - **Signed URLs:** https://cloud.google.com/storage/docs/access-control/signed-urls -- **rdev Postgres Provisioner:** `internal/adapter/postgres/provisioner.go` +- **rdev CockroachDB Provisioner:** `internal/adapter/cockroach/provisioner.go` - **rdev Redis Provisioner:** `internal/adapter/redis/provisioner.go` diff --git a/internal/adapter/templates/templates/components/service/.env.example.tmpl b/internal/adapter/templates/templates/components/service/.env.example.tmpl index f01de5f..40a9052 100644 --- a/internal/adapter/templates/templates/components/service/.env.example.tmpl +++ b/internal/adapter/templates/templates/components/service/.env.example.tmpl @@ -18,4 +18,6 @@ AUTH_ENABLED=false JWT_SECRET=dev-secret-change-in-production # Database (if needed) +# Local dev: PostgreSQL via docker-compose. Production: CockroachDB (platform-provisioned). +# The postgres:// scheme works for both — CockroachDB is wire-compatible. DATABASE_URL=postgres://dev:dev@localhost:5432/{{PROJECT_NAME}}?sslmode=disable diff --git a/internal/adapter/templates/templates/components/worker/.env.example.tmpl b/internal/adapter/templates/templates/components/worker/.env.example.tmpl index 4041ac0..be3c4f9 100644 --- a/internal/adapter/templates/templates/components/worker/.env.example.tmpl +++ b/internal/adapter/templates/templates/components/worker/.env.example.tmpl @@ -10,6 +10,8 @@ LOG_LEVEL=debug LOG_FORMAT=text # Database (required for job queue) +# Local dev: PostgreSQL via docker-compose. Production: CockroachDB (platform-provisioned). +# The postgres:// scheme works for both — CockroachDB is wire-compatible. DATABASE_URL=postgres://dev:dev@localhost:5432/{{PROJECT_NAME}}?sslmode=disable # Worker diff --git a/internal/adapter/templates/templates/skeleton/.claude/agents/database-architect.md b/internal/adapter/templates/templates/skeleton/.claude/agents/database-architect.md index 607c2d7..abae22c 100644 --- a/internal/adapter/templates/templates/skeleton/.claude/agents/database-architect.md +++ b/internal/adapter/templates/templates/skeleton/.claude/agents/database-architect.md @@ -1,6 +1,6 @@ --- name: database-architect -description: Database schema design and query optimization for {{PROJECT_NAME}} - PostgreSQL, migrations, indexing +description: Database schema design and query optimization for {{PROJECT_NAME}} - CockroachDB (production), PostgreSQL (local dev), migrations, indexing color: yellow --- @@ -10,11 +10,16 @@ You design database schemas and optimize queries for {{PROJECT_NAME}}. Every ser ## Stack -- **Primary:** PostgreSQL -- **Driver:** sqlx (no GORM) +- **Production:** CockroachDB (distributed SQL, provisioned by the platform) +- **Local dev:** PostgreSQL via docker-compose (wire-compatible with CockroachDB) +- **Driver:** sqlx with lib/pq (no GORM) — works with both PostgreSQL and CockroachDB - **Migrations:** Per-service in `services/{name}/migrations/` - **Naming:** snake_case for tables and columns +> **Important:** Write SQL that is compatible with both PostgreSQL and CockroachDB. +> Avoid PostgreSQL-specific features not supported by CockroachDB (e.g., advisory locks, listen/notify, full-text search with tsvector). +> Use `UUID` primary keys (CockroachDB handles these efficiently with no hotspotting). + ## Schema Conventions ### Tables diff --git a/internal/adapter/templates/templates/skeleton/.claude/agents/queue-specialist.md b/internal/adapter/templates/templates/skeleton/.claude/agents/queue-specialist.md index af4b309..dff0d9b 100644 --- a/internal/adapter/templates/templates/skeleton/.claude/agents/queue-specialist.md +++ b/internal/adapter/templates/templates/skeleton/.claude/agents/queue-specialist.md @@ -1,6 +1,6 @@ --- name: queue-specialist -description: Async job processing patterns for {{PROJECT_NAME}} - PostgreSQL queues, producer/consumer, retry logic, idempotency +description: Async job processing patterns for {{PROJECT_NAME}} - SQL queues (CockroachDB/PostgreSQL), producer/consumer, retry logic, idempotency color: purple --- diff --git a/internal/adapter/templates/templates/skeleton/.claude/agents/worker-specialist.md b/internal/adapter/templates/templates/skeleton/.claude/agents/worker-specialist.md index c942d76..d0d004d 100644 --- a/internal/adapter/templates/templates/skeleton/.claude/agents/worker-specialist.md +++ b/internal/adapter/templates/templates/skeleton/.claude/agents/worker-specialist.md @@ -11,7 +11,7 @@ You design and implement background workers for {{PROJECT_NAME}}. Workers are re ## Worker Types ### Queue Consumer -Processes jobs from a queue (PostgreSQL SKIP LOCKED, Redis, etc.): +Processes jobs from a queue (CockroachDB/PostgreSQL SKIP LOCKED, Redis, etc.): ```go func (w *Worker) Run(ctx context.Context) error { for { diff --git a/internal/adapter/templates/templates/skeleton/.claude/skills/knowledge-librarian/SKILL.md b/internal/adapter/templates/templates/skeleton/.claude/skills/knowledge-librarian/SKILL.md index 21691e7..d8db560 100644 --- a/internal/adapter/templates/templates/skeleton/.claude/skills/knowledge-librarian/SKILL.md +++ b/internal/adapter/templates/templates/skeleton/.claude/skills/knowledge-librarian/SKILL.md @@ -29,7 +29,7 @@ You are a librarian who transforms ephemeral conversation knowledge into permane | `architecture/` | System design facts | "How the work queue flows" | | `debugging/` | How to diagnose issues | "How to debug pod execution" | | `conventions/` | Naming, style, standards | "Error type naming convention" | -| `integrations/` | External system knowledge | "How we talk to PostgreSQL" | +| `integrations/` | External system knowledge | "How we talk to CockroachDB" | ## Storage Structure diff --git a/internal/adapter/templates/templates/skeleton/CLAUDE.md.tmpl b/internal/adapter/templates/templates/skeleton/CLAUDE.md.tmpl index 50d363a..6f79b36 100644 --- a/internal/adapter/templates/templates/skeleton/CLAUDE.md.tmpl +++ b/internal/adapter/templates/templates/skeleton/CLAUDE.md.tmpl @@ -38,6 +38,7 @@ - **OpenAPI first:** Document endpoints in `spec.go` using `openapi.*` helpers. Mount with `application.EnableDocs(spec)`. - **CSS variables:** All UI components use CSS custom properties (`var(--background)`, `var(--accent)`, etc.). Never hardcode colors. - **Monorepo imports:** Go packages from `{{GO_MODULE}}/pkg/*`, TypeScript from `@{{PROJECT_NAME}}/*`. +- **Database:** Production uses **CockroachDB** (provisioned by the platform). Local dev uses **PostgreSQL** via docker-compose. Both are wire-compatible via `lib/pq`. Write SQL compatible with both — avoid PostgreSQL-only features (advisory locks, listen/notify, tsvector). - **NO WEBSOCKETS. EVER.** All real-time communication uses HTTP2 + SSE. User → server is HTTP2 POST. Server → user is SSE. This includes chat, notifications, progress, everything. - **Event flow:** `POST → Service (enqueue) → Queue → Worker (generate) → Redis pub/sub → Service SSE subscriber → User`. Service is thin, worker does AI work. - **Channel naming:** `user:` = events for a specific user. `channel:` = events for a topic/room/resource. Document all channels in `./docs/channels.md`. diff --git a/internal/adapter/templates/templates/skeleton/docker-compose.yml.tmpl b/internal/adapter/templates/templates/skeleton/docker-compose.yml.tmpl index 864fa98..b8aa609 100644 --- a/internal/adapter/templates/templates/skeleton/docker-compose.yml.tmpl +++ b/internal/adapter/templates/templates/skeleton/docker-compose.yml.tmpl @@ -1,5 +1,8 @@ version: '3.8' +# Local development uses PostgreSQL for convenience. +# Production uses CockroachDB (provisioned by the platform). +# Both are wire-compatible — code using lib/pq works with either. services: postgres: image: postgres:16 diff --git a/internal/adapter/templates/templates/skeleton/pkg/database/db.go.tmpl b/internal/adapter/templates/templates/skeleton/pkg/database/db.go.tmpl index 76f76ca..5285ecd 100644 --- a/internal/adapter/templates/templates/skeleton/pkg/database/db.go.tmpl +++ b/internal/adapter/templates/templates/skeleton/pkg/database/db.go.tmpl @@ -1,4 +1,8 @@ -// Package database provides a standardized PostgreSQL/CockroachDB connection pool. +// Package database provides a standardized database connection pool. +// +// Production uses CockroachDB (provisioned by the platform). +// Local development uses PostgreSQL via docker-compose. +// Both are wire-compatible and use the lib/pq driver ("postgres"). // // This package wraps sqlx to provide: // - Connection pool management with sensible defaults @@ -31,7 +35,7 @@ import ( "time" "github.com/jmoiron/sqlx" - _ "github.com/lib/pq" // PostgreSQL/CockroachDB driver + _ "github.com/lib/pq" // PostgreSQL-compatible driver (works with both PostgreSQL and CockroachDB) ) // Pool wraps a sqlx.DB with additional lifecycle management. @@ -64,7 +68,7 @@ type Options struct { } // Connect establishes a connection pool to the database. -// The URL should be a PostgreSQL connection string: +// The URL should be a PostgreSQL-compatible connection string (works with CockroachDB): // // postgres://user:pass@host:port/dbname?sslmode=disable func Connect(ctx context.Context, url string, opts Options) (*Pool, error) { diff --git a/internal/adapter/templates/templates/skeleton/pkg/generation/handlers.go.tmpl b/internal/adapter/templates/templates/skeleton/pkg/generation/handlers.go.tmpl index c1440c5..4442e35 100644 --- a/internal/adapter/templates/templates/skeleton/pkg/generation/handlers.go.tmpl +++ b/internal/adapter/templates/templates/skeleton/pkg/generation/handlers.go.tmpl @@ -179,13 +179,25 @@ func VideoHandler(mg *mediagen.Manager, store storage.Store, pub realtime.EventP for i, vid := range resp.Videos { videoURL := vid.URL - // Persist to storage: download from provider URL, then upload to GCS. - if store != nil && vid.URL != "" { + // Persist to storage if available. + // Prefer vid.Data (already downloaded by provider adapter) over re-downloading from URL. + // Provider URLs (e.g., Gemini API) often require authentication and fail with plain GET. + if store != nil { storagePath := fmt.Sprintf("media/%s/videos/%s_%d.mp4", userID, job.ID, i) - videoData, downloadErr := downloadURL(ctx, vid.URL) - if downloadErr != nil { - logger.Warn("failed to download video from provider", "error", downloadErr, "job_id", job.ID) - } else { + + var videoData []byte + if len(vid.Data) > 0 { + videoData = vid.Data + } else if vid.URL != "" { + downloaded, downloadErr := downloadURL(ctx, vid.URL) + if downloadErr != nil { + logger.Warn("failed to download video from provider", "error", downloadErr, "job_id", job.ID) + } else { + videoData = downloaded + } + } + + if len(videoData) > 0 { persistedURL, uploadErr := store.Upload(ctx, storagePath, videoData, "video/mp4") if uploadErr != nil { logger.Warn("failed to persist video to storage", "error", uploadErr, "job_id", job.ID)