diff --git a/CLAUDE.md b/CLAUDE.md index 67df9bf..fb4c948 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,6 +31,7 @@ Run Claude Code instances in isolated Kubernetes pods with REST API control. Ena | **CockroachDB operations** | [services/cockroachdb.md](.claude/guides/services/cockroachdb.md) | | **Redis operations** | [services/redis.md](.claude/guides/services/redis.md) | | **DNS / Cloudflare** | [services/dns-cloudflare.md](.claude/guides/services/dns-cloudflare.md) | +| **Network policies / internal routing** | [ops/networking.md](.claude/guides/ops/networking.md) | ## Critical Rules @@ -161,9 +162,9 @@ cookbooks/ # End-to-end workflow guides | Database Provisioning | **Done** | CockroachDB adapter with auto-provisioning | | Cache Provisioning | **Done** | Redis ACL-based adapter with auto-provisioning | | Build Orchestration | Planned | Structured build specs via API | -| Composable Monorepo Templates | Planned | Monorepo skeleton + component templates | +| Composable Monorepo Templates | **Done** | Monorepo skeleton + component templates (service, worker, app-astro, app-react, cli) | -**Current Version:** v0.10.12 +**Current Version:** v0.10.24 ## Constraints diff --git a/changelog/v0.10.27.md b/changelog/v0.10.27.md new file mode 100644 index 0000000..a64cc66 --- /dev/null +++ b/changelog/v0.10.27.md @@ -0,0 +1,11 @@ +# v0.10.27 + +**Released:** 2026-02-01 + +## Changes + +fix: woodpecker step YAML multi-line command syntax + +--- + +**Image:** `ghcr.io/orchard9/rdev-api:v0.10.27` diff --git a/cmd/rdev-api/main.go b/cmd/rdev-api/main.go index d4a4415..6557796 100644 --- a/cmd/rdev-api/main.go +++ b/cmd/rdev-api/main.go @@ -351,8 +351,10 @@ func main() { database.DB, templateProvider, bulkFileClient, + deployerAdapter, // Creates initial K8s deployment for new components service.ComponentServiceConfig{ DefaultGitOwner: infraCfg.GiteaDefaultOrg, + RegistryURL: infraCfg.RegistryURL, Logger: logger, }, ) diff --git a/cookbooks/composable-app.md b/cookbooks/composable-app.md index e060b53..3773c03 100644 --- a/cookbooks/composable-app.md +++ b/cookbooks/composable-app.md @@ -7,7 +7,7 @@ This cookbook creates a full-stack application by composing components: ``` -POST /projects +POST /project ↓ Creates: Monorepo skeleton with shared packages ↓ @@ -19,9 +19,9 @@ POST /projects/{id}/components (app) ↓ Adds: React/Astro frontend with CI step ↓ -POST /projects/{id}/deploy +Git push triggers Woodpecker CI ↓ -Deploys all components to K8s +CI builds and deploys all components to K8s ``` **Composable. Template-driven. CI auto-configured.** @@ -46,7 +46,7 @@ export RDEV_API_KEY="" ## Step 1: Create Project (Monorepo Skeleton) ```bash -curl -X POST "$RDEV_API_URL/projects" \ +curl -X POST "$RDEV_API_URL/project" \ -H "X-API-Key: $RDEV_API_KEY" \ -H "Content-Type: application/json" \ -d '{ @@ -306,7 +306,7 @@ Ports auto-increment as you add components of the same type. ## Teardown ```bash -curl -X DELETE "$RDEV_API_URL/projects/taskapp" \ +curl -X DELETE "$RDEV_API_URL/project/taskapp" \ -H "X-API-Key: $RDEV_API_KEY" ``` @@ -339,7 +339,7 @@ Cleanup: ┌─────────────────────────────────────────────────────────────────────┐ │ Composable Full-Stack Deployment │ │ │ -│ POST /projects │ +│ POST /project │ │ │ │ │ └──► Creates monorepo skeleton with: │ │ - Shared pkg/ (8 packages) │ @@ -377,7 +377,7 @@ Cleanup: ### Component addition fails ```bash # Check project exists -curl -s "$RDEV_API_URL/projects/taskapp" \ +curl -s "$RDEV_API_URL/project/taskapp" \ -H "X-API-Key: $RDEV_API_KEY" | jq '.data' # Check available component templates diff --git a/cookbooks/scripts/common.sh b/cookbooks/scripts/common.sh index aa6e38e..13ad05e 100755 --- a/cookbooks/scripts/common.sh +++ b/cookbooks/scripts/common.sh @@ -103,6 +103,7 @@ wait_for_build() { # Wait for CI pipeline to complete # Arguments: project_id [max_attempts] [poll_interval] # Returns: 0 on success, 1 on failure, 2 on timeout +# On failure, automatically runs diagnostics wait_for_pipeline() { local project_id="$1" local max_attempts="${2:-60}" # 5 minutes default @@ -142,6 +143,8 @@ wait_for_pipeline() { ;; failure|error|killed) echo -e "${RED}Pipeline #$pipeline_number failed with status: $status${NC}" + # Automatically diagnose the failure + diagnose_pipeline_failure "$project_id" return 1 ;; running|pending) @@ -161,13 +164,16 @@ wait_for_pipeline() { } # Wait for site to be accessible -# Arguments: domain [max_attempts] [poll_interval] +# Arguments: domain [max_attempts] [poll_interval] [project_id] # Returns: 0 on success, 1 on timeout +# On timeout, automatically runs diagnostics if project_id is provided wait_for_site() { local domain="$1" local max_attempts="${2:-30}" local poll_interval="${3:-5}" + local project_id="${4:-}" local attempt=0 + local last_http_code="" echo -e "${CYAN}Waiting for site to be accessible at https://$domain...${NC}" @@ -180,12 +186,26 @@ wait_for_site() { return 0 fi - echo " HTTP $http_code... (attempt $((attempt + 1))/$max_attempts)" + # Only print status change or every 5th attempt to reduce noise + if [[ "$http_code" != "$last_http_code" ]] || (( attempt % 5 == 0 )); then + echo " HTTP $http_code... (attempt $((attempt + 1))/$max_attempts)" + fi + last_http_code="$http_code" + sleep "$poll_interval" ((attempt++)) done - echo -e "${YELLOW}Timeout waiting for site to respond${NC}" + echo -e "${YELLOW}Timeout waiting for site to respond (last: HTTP $last_http_code)${NC}" + + # Automatically diagnose if we have project_id + if [[ -n "$project_id" ]]; then + diagnose_site_failure "$domain" "$project_id" + else + echo "" + echo " Tip: Pass project_id to wait_for_site for automatic diagnostics" + fi + return 1 } @@ -211,3 +231,250 @@ print_error() { print_warning() { echo -e "${YELLOW}⚠ $1${NC}" } + +# Print diagnostic section header +print_diagnostic_header() { + local title="$1" + echo "" + echo -e "${CYAN}┌─────────────────────────────────────────────────────────────────┐${NC}" + echo -e "${CYAN}│ DIAGNOSTIC: $title${NC}" + echo -e "${CYAN}└─────────────────────────────────────────────────────────────────┘${NC}" +} + +# Print a suggested fix +print_fix() { + echo -e "${YELLOW} → FIX: $1${NC}" +} + +# Print a command the user can run +print_cmd() { + echo -e "${BLUE} \$ $1${NC}" +} + +# Get git owner from environment or default +get_git_owner() { + echo "${GITEA_DEFAULT_ORG:-jordan}" +} + +# Diagnose a failed pipeline - fetches details and prints actionable info +# Arguments: project_id +diagnose_pipeline_failure() { + local project_id="$1" + local git_owner + git_owner=$(get_git_owner) + + print_diagnostic_header "Pipeline Failure Analysis" + + # Get the latest pipeline + local pipelines + pipelines=$(api_call GET "/projects/$project_id/pipelines") + + local pipeline_number + pipeline_number=$(echo "$pipelines" | jq -r '.data[0].number // "?"') + local pipeline_status + pipeline_status=$(echo "$pipelines" | jq -r '.data[0].status // "unknown"') + local pipeline_errors + pipeline_errors=$(echo "$pipelines" | jq -r '.data[0].errors // []') + local commit_msg + commit_msg=$(echo "$pipelines" | jq -r '.data[0].message // ""' | head -1) + + echo "" + echo " Pipeline #$pipeline_number: $pipeline_status" + echo " Commit: $commit_msg" + + # Show any pipeline-level errors (YAML validation, etc.) + local error_count + error_count=$(echo "$pipeline_errors" | jq 'length // 0') + if [[ "$error_count" -gt 0 ]]; then + echo "" + echo -e "${RED} Pipeline Errors:${NC}" + echo "$pipeline_errors" | jq -r '.[] | " - \(.type): \(.message)"' + fi + + # Try to get step details from the steps API (if available) + local steps_response + steps_response=$(api_call GET "/projects/$project_id/pipelines/$pipeline_number/steps" 2>/dev/null || echo '{"error":"not available"}') + + local has_steps + has_steps=$(echo "$steps_response" | jq 'has("data")' 2>/dev/null || echo "false") + + if [[ "$has_steps" == "true" ]]; then + echo "" + echo " Steps:" + # Format steps with status icons, duration, and exit code for failures + echo "$steps_response" | jq -r '.data.steps[] | + (if .duration_seconds > 0 then " (\(.duration_seconds)s)" else "" end) as $dur | + if .status == "failure" or .status == "error" or .status == "killed" then + " \u001b[31m✗\u001b[0m \(.name): FAILED (exit \(.exit_code // "?"))\($dur)" + elif .status == "success" then + " \u001b[32m✓\u001b[0m \(.name): success\($dur)" + elif .status == "running" then + " \u001b[33m◐\u001b[0m \(.name): running..." + elif .status == "pending" then + " ○ \(.name): pending" + elif .status == "skipped" then + " ○ \(.name): skipped" + else + " ? \(.name): \(.status)" + end' + + # Show logs from failed steps + local failed_steps + failed_steps=$(echo "$steps_response" | jq -r '.data.steps[] | select(.status == "failure" or .status == "error" or .status == "killed")') + + if [[ -n "$failed_steps" ]]; then + echo "" + echo -e "${RED} Failed Step Details:${NC}" + + # For each failed step, show error and log + echo "$steps_response" | jq -r '.data.steps[] | select(.status == "failure" or .status == "error" or .status == "killed") | + "\n Step: \(.name)" + + (if .error and .error != "" then "\n Error: \(.error)" else "" end) + + (if .log and .log != "" then "\n Last lines of log:\n\(.log | split("\n") | .[-20:] | join("\n") | gsub("^"; " "))" else "" end)' + fi + else + echo "" + echo -e "${YELLOW} Steps API not available - upgrade rdev-api for detailed step info${NC}" + fi + + # Always provide direct links + echo "" + echo " View full logs:" + print_cmd "open https://ci.threesix.ai/$git_owner/$project_id/$pipeline_number" + + # Pattern match common errors and suggest fixes + echo "" + diagnose_common_pipeline_errors "$project_id" "$pipeline_number" +} + +# Pattern match common pipeline errors and suggest fixes +diagnose_common_pipeline_errors() { + local project_id="$1" + local pipeline_number="$2" + + echo " Common issues to check:" + echo "" + + # Check 1: Missing K8s deployment (most common issue) + echo " 1. Missing Kubernetes Deployment?" + echo " The CI pipeline tries to 'kubectl set image' but deployment may not exist." + print_cmd "kubectl get deployment -n projects -l app=$project_id" + print_fix "Component may need initial deployment created" + echo "" + + # Check 2: Docker build issues + echo " 2. Docker Build Failed?" + echo " Check if Dockerfile exists and workspace files are correct." + print_cmd "Check the build step in Woodpecker UI for specific error" + echo "" + + # Check 3: Registry auth + echo " 3. Registry Push Failed?" + echo " Kaniko may not have credentials to push to registry." + print_cmd "kubectl get secret -n woodpecker-agents | grep registry" +} + +# Diagnose why a site is not accessible +# Arguments: domain project_id +diagnose_site_failure() { + local domain="$1" + local project_id="$2" + + print_diagnostic_header "Site Accessibility Analysis" + + echo "" + echo " Domain: https://$domain" + echo " Project: $project_id" + echo "" + + # Check if kubectl is available and configured + if ! command -v kubectl &> /dev/null; then + echo -e "${YELLOW} kubectl not found - cannot check K8s state${NC}" + echo " Install kubectl and set KUBECONFIG to diagnose further" + return + fi + + if [[ -z "${KUBECONFIG:-}" ]]; then + echo -e "${YELLOW} KUBECONFIG not set - trying default context${NC}" + fi + + # Check pods + echo " Checking pods in 'projects' namespace:" + local pods + pods=$(kubectl get pods -n projects -l "app=$project_id" --no-headers 2>/dev/null || echo "ERROR") + + if [[ "$pods" == "ERROR" ]]; then + echo -e "${RED} Failed to query K8s (check KUBECONFIG)${NC}" + print_cmd "export KUBECONFIG=~/.kube/orchard9-k3sf.yaml" + return + elif [[ -z "$pods" ]]; then + echo -e "${RED} No pods found for app=$project_id${NC}" + print_fix "Deployment doesn't exist - CI may have failed or component needs initial deploy" + print_cmd "kubectl get deployments -n projects" + else + echo "$pods" | sed 's/^/ /' + + # Check for common pod issues + if echo "$pods" | grep -q "ImagePullBackOff\|ErrImagePull"; then + echo "" + echo -e "${RED} Issue: ImagePullBackOff${NC}" + print_fix "Image doesn't exist in registry - check CI build step" + print_cmd "kubectl describe pod -n projects -l app=$project_id | grep -A5 'Events:'" + fi + + if echo "$pods" | grep -q "CrashLoopBackOff"; then + echo "" + echo -e "${RED} Issue: CrashLoopBackOff${NC}" + print_fix "Container is crashing - check application logs" + print_cmd "kubectl logs -n projects -l app=$project_id --tail=50" + fi + + if echo "$pods" | grep -q "Pending"; then + echo "" + echo -e "${RED} Issue: Pod stuck in Pending${NC}" + print_fix "Likely resource constraints or scheduling issues" + print_cmd "kubectl describe pod -n projects -l app=$project_id | grep -A10 'Events:'" + fi + + if echo "$pods" | grep -q "0/1\|0/2"; then + echo "" + echo -e "${YELLOW} Issue: Container not ready${NC}" + print_fix "Container may still be starting or failing health checks" + print_cmd "kubectl logs -n projects -l app=$project_id --tail=20" + fi + fi + + # Check services + echo "" + echo " Checking services:" + local svc + svc=$(kubectl get svc -n projects -l "app=$project_id" --no-headers 2>/dev/null || echo "") + if [[ -z "$svc" ]]; then + echo -e "${RED} No service found for app=$project_id${NC}" + print_fix "Service needs to be created along with deployment" + else + echo "$svc" | sed 's/^/ /' + fi + + # Check ingress + echo "" + echo " Checking ingress:" + local ingress + ingress=$(kubectl get ingress -n projects --no-headers 2>/dev/null | grep "$project_id\|$domain" || echo "") + if [[ -z "$ingress" ]]; then + echo -e "${YELLOW} No ingress found matching $project_id or $domain${NC}" + else + echo "$ingress" | sed 's/^/ /' + fi + + # Recent events + echo "" + echo " Recent events:" + kubectl get events -n projects --sort-by='.lastTimestamp' 2>/dev/null | grep "$project_id" | tail -5 | sed 's/^/ /' || echo " No recent events" + + echo "" + echo " Manual investigation commands:" + print_cmd "kubectl logs -n projects -l app=$project_id -f" + print_cmd "kubectl describe pod -n projects -l app=$project_id" + print_cmd "kubectl get events -n projects --sort-by='.lastTimestamp' | tail -20" +} diff --git a/cookbooks/scripts/composable-test.sh b/cookbooks/scripts/composable-test.sh index a9b519d..55e7305 100755 --- a/cookbooks/scripts/composable-test.sh +++ b/cookbooks/scripts/composable-test.sh @@ -23,6 +23,7 @@ if [[ -z "$COMMAND" || -z "$PROJECT_NAME" ]]; then echo "Commands:" echo " run - Create project with components and deploy" echo " status - Check project and component status" + echo " diagnose - Deep diagnostic of pipeline and site issues" echo " teardown - Delete the project" exit 1 fi @@ -73,7 +74,7 @@ run_flow() { '{name: $name, description: $desc}') local create_result - create_result=$(api_call POST "/projects" "$create_payload") + create_result=$(api_call POST "/project" "$create_payload") echo "$create_result" | jq '.' local domain @@ -120,7 +121,7 @@ run_flow() { # Step 6: Wait for site print_header "Step 6: Verifying site is accessible" - if ! wait_for_site "$domain"; then + if ! wait_for_site "$domain" 30 5 "$PROJECT_NAME"; then print_error "Site not accessible" exit 1 fi @@ -152,7 +153,7 @@ check_status() { # Get project info echo "Project:" - api_call GET "/projects/$PROJECT_NAME" | jq '.data // .' + api_call GET "/project/$PROJECT_NAME" | jq '.data // .' echo "" # Get components @@ -169,13 +170,48 @@ teardown() { print_header "Tearing down: $PROJECT_NAME" local result - result=$(api_call DELETE "/projects/$PROJECT_NAME") + result=$(api_call DELETE "/project/$PROJECT_NAME") echo "$result" | jq '.' echo "" print_success "Project deleted. Gitea repo preserved." } +diagnose() { + print_header "Diagnostic: $PROJECT_NAME" + + # Get project info for domain + local project + project=$(api_call GET "/project/$PROJECT_NAME") + + local domain + domain=$(echo "$project" | jq -r '.data.domain // ""') + + if [[ -z "$domain" ]]; then + print_error "Project not found or no domain assigned" + echo "$project" | jq '.' + return 1 + fi + + echo "Project: $PROJECT_NAME" + echo "Domain: https://$domain" + echo "Git: https://git.threesix.ai/$(get_git_owner)/$PROJECT_NAME" + echo "CI: https://ci.threesix.ai/$(get_git_owner)/$PROJECT_NAME" + + # Run pipeline diagnostics + diagnose_pipeline_failure "$PROJECT_NAME" + + # Run site diagnostics + diagnose_site_failure "$domain" "$PROJECT_NAME" + + print_header "Summary" + echo "To retry the pipeline:" + print_cmd "Push a commit to trigger CI, or manually trigger from Woodpecker UI" + echo "" + echo "To check real-time logs:" + print_cmd "kubectl logs -n projects -l app=$PROJECT_NAME -f" +} + case "$COMMAND" in run) run_flow @@ -183,12 +219,15 @@ case "$COMMAND" in status) check_status ;; + diagnose) + diagnose + ;; teardown) teardown ;; *) echo "Unknown command: $COMMAND" - echo "Valid commands: run, status, teardown" + echo "Valid commands: run, status, diagnose, teardown" exit 1 ;; esac diff --git a/deployments/k8s/base/network-policy.yaml b/deployments/k8s/base/network-policy.yaml index f18ef5e..3d1fb04 100644 --- a/deployments/k8s/base/network-policy.yaml +++ b/deployments/k8s/base/network-policy.yaml @@ -16,11 +16,11 @@ spec: - Ingress - Egress ingress: - # Allow ingress from ingress controller + # Allow ingress from Traefik ingress controller (k3s default) - from: - namespaceSelector: matchLabels: - kubernetes.io/metadata.name: ingress-nginx + kubernetes.io/metadata.name: kube-system ports: - protocol: TCP port: 8080 @@ -41,11 +41,32 @@ spec: ports: - protocol: TCP port: 5432 + # Allow egress to CockroachDB in databases namespace + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: databases + ports: + - protocol: TCP + port: 26257 + # Allow egress to Redis in databases namespace + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: databases + ports: + - protocol: TCP + port: 6379 # Allow egress to claudebox pods within the rdev namespace - to: - podSelector: matchLabels: rdev.orchard9.ai/project: "true" + # Allow egress to threesix namespace (Gitea, Woodpecker CI) + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: threesix # Allow DNS resolution - to: - namespaceSelector: {} @@ -57,3 +78,14 @@ spec: port: 53 - protocol: TCP port: 53 + # Allow egress to external HTTPS services (Gitea, Cloudflare, Woodpecker CI) + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + ports: + - protocol: TCP + port: 443 diff --git a/deployments/k8s/base/rdev-api.yaml b/deployments/k8s/base/rdev-api.yaml index 07a2122..829e6b9 100644 --- a/deployments/k8s/base/rdev-api.yaml +++ b/deployments/k8s/base/rdev-api.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: rdev-api containers: - name: rdev-api - image: ghcr.io/orchard9/rdev-api:v0.10.25 + image: ghcr.io/orchard9/rdev-api:v0.10.27 imagePullPolicy: Always ports: diff --git a/internal/adapter/deployer/resources.go b/internal/adapter/deployer/resources.go index 9269c9a..463f6df 100644 --- a/internal/adapter/deployer/resources.go +++ b/internal/adapter/deployer/resources.go @@ -16,6 +16,13 @@ import ( "github.com/orchard9/rdev/internal/domain" ) +// sanitizeLabelValue converts a component path to a valid K8s label value. +// K8s labels must be alphanumeric with '-', '_', or '.' and must start/end with alphanumeric. +// Example: "services/api" -> "services-api" +func sanitizeLabelValue(path string) string { + return strings.ReplaceAll(path, "/", "-") +} + // ensureNamespace verifies the deployment namespace exists, creating it only if needed. func (d *Deployer) ensureNamespace(ctx context.Context) error { _, err := d.client.CoreV1().Namespaces().Get(ctx, d.config.Namespace, metav1.GetOptions{}) @@ -110,7 +117,7 @@ func (d *Deployer) buildDeployment(spec domain.DeploySpec, ns string, replicas i "project": spec.ProjectName, } if spec.ComponentPath != "" { - labels["component"] = spec.ComponentPath + labels["component"] = sanitizeLabelValue(spec.ComponentPath) } return &appsv1.Deployment{ @@ -172,7 +179,7 @@ func (d *Deployer) createOrUpdateService(ctx context.Context, spec domain.Deploy "project": spec.ProjectName, } if spec.ComponentPath != "" { - labels["component"] = spec.ComponentPath + labels["component"] = sanitizeLabelValue(spec.ComponentPath) } service := &corev1.Service{ @@ -239,7 +246,7 @@ func (d *Deployer) buildIngress(spec domain.DeploySpec, ns string, pathType netw "project": spec.ProjectName, } if spec.ComponentPath != "" { - labels["component"] = spec.ComponentPath + labels["component"] = sanitizeLabelValue(spec.ComponentPath) } return &networkingv1.Ingress{ diff --git a/internal/adapter/templates/templates/components/app-astro/.woodpecker.step.yml.tmpl b/internal/adapter/templates/templates/components/app-astro/.woodpecker.step.yml.tmpl index 63400b0..0eac9ad 100644 --- a/internal/adapter/templates/templates/components/app-astro/.woodpecker.step.yml.tmpl +++ b/internal/adapter/templates/templates/components/app-astro/.woodpecker.step.yml.tmpl @@ -20,10 +20,7 @@ build-{{COMPONENT_NAME}}: deploy-{{COMPONENT_NAME}}: image: bitnami/kubectl:latest commands: - - kubectl set image deployment/{{PROJECT_NAME}}-{{COMPONENT_NAME}} - {{COMPONENT_NAME}}=registry.threesix.ai/{{PROJECT_NAME}}/{{COMPONENT_NAME}}:${CI_COMMIT_SHA:0:8} - -n projects - || echo "No deployment found for {{COMPONENT_NAME}}, skipping" + - kubectl set image deployment/{{PROJECT_NAME}}-{{COMPONENT_NAME}} {{COMPONENT_NAME}}=registry.threesix.ai/{{PROJECT_NAME}}/{{COMPONENT_NAME}}:${CI_COMMIT_SHA:0:8} -n projects || echo "Deployment not found, skipping" when: branch: main event: push diff --git a/internal/adapter/templates/templates/components/app-astro/Dockerfile.tmpl b/internal/adapter/templates/templates/components/app-astro/Dockerfile.tmpl index 4ea4f46..f2656b0 100644 --- a/internal/adapter/templates/templates/components/app-astro/Dockerfile.tmpl +++ b/internal/adapter/templates/templates/components/app-astro/Dockerfile.tmpl @@ -1,25 +1,32 @@ -# Build stage +# Build stage - using pnpm for workspace dependency resolution FROM node:20-alpine AS build -WORKDIR /app +# Install pnpm +RUN npm install -g pnpm -# Copy package files -COPY apps/{{COMPONENT_NAME}}/package*.json ./ +WORKDIR /workspace -# Install dependencies -RUN npm install +# Copy workspace configuration files +COPY pnpm-workspace.yaml package.json pnpm-lock.yaml* ./ -# Copy source -COPY apps/{{COMPONENT_NAME}}/ ./ +# Copy shared packages (required for workspace:* dependencies) +COPY packages/ ./packages/ -# Build -RUN npm run build +# Copy the app component +COPY apps/{{COMPONENT_NAME}}/ ./apps/{{COMPONENT_NAME}}/ + +# Install dependencies using pnpm (resolves workspace:* correctly) +RUN pnpm install --frozen-lockfile || pnpm install + +# Build the app +WORKDIR /workspace/apps/{{COMPONENT_NAME}} +RUN pnpm build # Production stage FROM nginx:alpine # Copy built assets -COPY --from=build /app/dist /usr/share/nginx/html +COPY --from=build /workspace/apps/{{COMPONENT_NAME}}/dist /usr/share/nginx/html COPY apps/{{COMPONENT_NAME}}/nginx.conf /etc/nginx/conf.d/default.conf EXPOSE 80 diff --git a/internal/adapter/templates/templates/components/app-react/.woodpecker.step.yml.tmpl b/internal/adapter/templates/templates/components/app-react/.woodpecker.step.yml.tmpl index bcd3b0b..9c84f13 100644 --- a/internal/adapter/templates/templates/components/app-react/.woodpecker.step.yml.tmpl +++ b/internal/adapter/templates/templates/components/app-react/.woodpecker.step.yml.tmpl @@ -20,10 +20,7 @@ build-{{COMPONENT_NAME}}: deploy-{{COMPONENT_NAME}}: image: bitnami/kubectl:latest commands: - - kubectl set image deployment/{{PROJECT_NAME}}-{{COMPONENT_NAME}} - {{COMPONENT_NAME}}=registry.threesix.ai/{{PROJECT_NAME}}/{{COMPONENT_NAME}}:${CI_COMMIT_SHA:0:8} - -n projects - || echo "No deployment found for {{COMPONENT_NAME}}, skipping" + - kubectl set image deployment/{{PROJECT_NAME}}-{{COMPONENT_NAME}} {{COMPONENT_NAME}}=registry.threesix.ai/{{PROJECT_NAME}}/{{COMPONENT_NAME}}:${CI_COMMIT_SHA:0:8} -n projects || echo "Deployment not found, skipping" when: branch: main event: push diff --git a/internal/adapter/templates/templates/components/app-react/Dockerfile.tmpl b/internal/adapter/templates/templates/components/app-react/Dockerfile.tmpl index 4ea4f46..f2656b0 100644 --- a/internal/adapter/templates/templates/components/app-react/Dockerfile.tmpl +++ b/internal/adapter/templates/templates/components/app-react/Dockerfile.tmpl @@ -1,25 +1,32 @@ -# Build stage +# Build stage - using pnpm for workspace dependency resolution FROM node:20-alpine AS build -WORKDIR /app +# Install pnpm +RUN npm install -g pnpm -# Copy package files -COPY apps/{{COMPONENT_NAME}}/package*.json ./ +WORKDIR /workspace -# Install dependencies -RUN npm install +# Copy workspace configuration files +COPY pnpm-workspace.yaml package.json pnpm-lock.yaml* ./ -# Copy source -COPY apps/{{COMPONENT_NAME}}/ ./ +# Copy shared packages (required for workspace:* dependencies) +COPY packages/ ./packages/ -# Build -RUN npm run build +# Copy the app component +COPY apps/{{COMPONENT_NAME}}/ ./apps/{{COMPONENT_NAME}}/ + +# Install dependencies using pnpm (resolves workspace:* correctly) +RUN pnpm install --frozen-lockfile || pnpm install + +# Build the app +WORKDIR /workspace/apps/{{COMPONENT_NAME}} +RUN pnpm build # Production stage FROM nginx:alpine # Copy built assets -COPY --from=build /app/dist /usr/share/nginx/html +COPY --from=build /workspace/apps/{{COMPONENT_NAME}}/dist /usr/share/nginx/html COPY apps/{{COMPONENT_NAME}}/nginx.conf /etc/nginx/conf.d/default.conf EXPOSE 80 diff --git a/internal/adapter/templates/templates/components/service/.woodpecker.step.yml.tmpl b/internal/adapter/templates/templates/components/service/.woodpecker.step.yml.tmpl index 7e75a7b..afbb04b 100644 --- a/internal/adapter/templates/templates/components/service/.woodpecker.step.yml.tmpl +++ b/internal/adapter/templates/templates/components/service/.woodpecker.step.yml.tmpl @@ -20,10 +20,7 @@ build-{{COMPONENT_NAME}}: deploy-{{COMPONENT_NAME}}: image: bitnami/kubectl:latest commands: - - kubectl set image deployment/{{PROJECT_NAME}}-{{COMPONENT_NAME}} - {{COMPONENT_NAME}}=registry.threesix.ai/{{PROJECT_NAME}}/{{COMPONENT_NAME}}:${CI_COMMIT_SHA:0:8} - -n projects - || echo "No deployment found for {{COMPONENT_NAME}}, skipping" + - kubectl set image deployment/{{PROJECT_NAME}}-{{COMPONENT_NAME}} {{COMPONENT_NAME}}=registry.threesix.ai/{{PROJECT_NAME}}/{{COMPONENT_NAME}}:${CI_COMMIT_SHA:0:8} -n projects || echo "Deployment not found, skipping" when: branch: main event: push diff --git a/internal/adapter/templates/templates/components/worker/.woodpecker.step.yml.tmpl b/internal/adapter/templates/templates/components/worker/.woodpecker.step.yml.tmpl index d1c4f50..5184d23 100644 --- a/internal/adapter/templates/templates/components/worker/.woodpecker.step.yml.tmpl +++ b/internal/adapter/templates/templates/components/worker/.woodpecker.step.yml.tmpl @@ -20,10 +20,7 @@ build-{{COMPONENT_NAME}}: deploy-{{COMPONENT_NAME}}: image: bitnami/kubectl:latest commands: - - kubectl set image deployment/{{PROJECT_NAME}}-{{COMPONENT_NAME}} - {{COMPONENT_NAME}}=registry.threesix.ai/{{PROJECT_NAME}}/{{COMPONENT_NAME}}:${CI_COMMIT_SHA:0:8} - -n projects - || echo "No deployment found for {{COMPONENT_NAME}}, skipping" + - kubectl set image deployment/{{PROJECT_NAME}}-{{COMPONENT_NAME}} {{COMPONENT_NAME}}=registry.threesix.ai/{{PROJECT_NAME}}/{{COMPONENT_NAME}}:${CI_COMMIT_SHA:0:8} -n projects || echo "Deployment not found, skipping" when: branch: main event: push diff --git a/internal/adapter/templates/templates/skeleton/.woodpecker.yml.tmpl b/internal/adapter/templates/templates/skeleton/.woodpecker.yml.tmpl index 77bad2b..1f16d55 100644 --- a/internal/adapter/templates/templates/skeleton/.woodpecker.yml.tmpl +++ b/internal/adapter/templates/templates/skeleton/.woodpecker.yml.tmpl @@ -8,7 +8,8 @@ clone: depth: 1 steps: - # COMPONENT_STEPS_BELOW - Do not remove this marker + # COMPONENT_STEPS_BELOW + # Do not remove the marker above - component steps are inserted here verify: image: bitnami/kubectl:latest diff --git a/internal/adapter/templates/templates/skeleton/package.json.tmpl b/internal/adapter/templates/templates/skeleton/package.json.tmpl new file mode 100644 index 0000000..a5bc907 --- /dev/null +++ b/internal/adapter/templates/templates/skeleton/package.json.tmpl @@ -0,0 +1,12 @@ +{ + "name": "{{PROJECT_NAME}}", + "private": true, + "version": "0.0.1", + "packageManager": "pnpm@9.15.0", + "scripts": { + "dev": "pnpm -r dev", + "build": "pnpm -r build", + "lint": "pnpm -r lint", + "test": "pnpm -r test" + } +} diff --git a/internal/adapter/templates/templates/skeleton/scripts/install.sh.tmpl b/internal/adapter/templates/templates/skeleton/scripts/install.sh.tmpl index 67a3abb..2ad41f3 100644 --- a/internal/adapter/templates/templates/skeleton/scripts/install.sh.tmpl +++ b/internal/adapter/templates/templates/skeleton/scripts/install.sh.tmpl @@ -4,13 +4,17 @@ set -e echo "Installing dependencies for {{PROJECT_NAME}}..." echo "" -# Install shared package dependencies first -for dir in packages/*/; do - if [ -f "${dir}package.json" ]; then - echo "Installing deps for package $(basename $dir)..." - (cd "$dir" && npm install) - fi -done +# Check for pnpm, install if not present +if ! command -v pnpm &> /dev/null; then + echo "Installing pnpm..." + npm install -g pnpm +fi + +# Install all Node dependencies at workspace root (pnpm handles workspace:* refs) +if [ -f "pnpm-lock.yaml" ] || [ -f "package.json" ]; then + echo "Installing Node dependencies with pnpm..." + pnpm install +fi # Install Go dependencies for dir in services/*/ workers/*/ cli/*/; do @@ -20,13 +24,5 @@ for dir in services/*/ workers/*/ cli/*/; do fi done -# Install Node dependencies -for dir in apps/*/; do - if [ -f "${dir}package.json" ]; then - echo "Installing Node deps for $(basename $dir)..." - (cd "$dir" && npm install) - fi -done - echo "" echo "All dependencies installed!" diff --git a/internal/adapter/woodpecker/client.go b/internal/adapter/woodpecker/client.go index e7e02e8..e4bc059 100644 --- a/internal/adapter/woodpecker/client.go +++ b/internal/adapter/woodpecker/client.go @@ -418,6 +418,111 @@ func pipelineFromWoodpecker(p *woodpecker.Pipeline) *domain.CIPipeline { } } +// GetPipelineSteps returns detailed step information for a pipeline. +// For failed steps, includes the last 50 lines of log output. +func (c *Client) GetPipelineSteps(ctx context.Context, owner, repo string, number int64) (*domain.CIPipelineSteps, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + fullName := owner + "/" + repo + + r, err := c.client.RepoLookup(fullName) + if err != nil { + return nil, fmt.Errorf("repo not found: %s", fullName) + } + + p, err := c.client.Pipeline(r.ID, number) + if err != nil { + return nil, fmt.Errorf("pipeline %d not found: %w", number, err) + } + + result := &domain.CIPipelineSteps{ + PipelineNumber: number, + URL: fmt.Sprintf("%s/%s/%d", c.url, fullName, number), + Steps: make([]domain.CIPipelineStep, 0), + } + + // Flatten workflows -> steps + for _, wf := range p.Workflows { + if wf == nil { + continue + } + for _, step := range wf.Children { + if step == nil { + continue + } + + // Calculate duration + var duration int + if step.Stopped > 0 && step.Started > 0 { + duration = int(step.Stopped - step.Started) + } + + // Convert exit code + var exitCode *int + if step.State == "success" || step.State == "failure" || step.State == "killed" { + code := step.ExitCode + exitCode = &code + } + + domainStep := domain.CIPipelineStep{ + ID: step.ID, + Name: step.Name, + Status: step.State, + ExitCode: exitCode, + Duration: duration, + Error: step.Error, + } + + // Fetch logs for failed steps + if step.State == "failure" || step.State == "error" || step.State == "killed" { + logs, err := c.client.StepLogEntries(r.ID, number, step.ID) + if err != nil { + c.logger.Warn("failed to fetch step logs", "step", step.Name, "error", err) + } else { + domainStep.Log = formatLogEntries(logs, 50) + } + } + + result.Steps = append(result.Steps, domainStep) + } + } + + return result, nil +} + +// formatLogEntries combines log entries and returns the last N lines. +func formatLogEntries(entries []*woodpecker.LogEntry, maxLines int) string { + if len(entries) == 0 { + return "" + } + + // Collect all lines + var lines []string + for _, entry := range entries { + if entry != nil && len(entry.Data) > 0 { + // Data is []byte, convert to string and split + data := string(entry.Data) + entryLines := strings.Split(data, "\n") + for _, line := range entryLines { + if line != "" { + lines = append(lines, line) + } + } + } + } + + // Return last maxLines + if len(lines) > maxLines { + lines = lines[len(lines)-maxLines:] + } + + return strings.Join(lines, "\n") +} + // TriggerBuild manually starts a new pipeline build on the specified branch. func (c *Client) TriggerBuild(ctx context.Context, owner, repo, branch string) (int64, error) { select { diff --git a/internal/domain/ci.go b/internal/domain/ci.go index 725adea..1f91e0f 100644 --- a/internal/domain/ci.go +++ b/internal/domain/ci.go @@ -101,3 +101,39 @@ type CIPipeline struct { // Errors contains any pipeline errors (e.g., YAML validation failures) Errors []CIPipelineError } + +// CIPipelineStep represents a step within a CI pipeline. +type CIPipelineStep struct { + // ID is the step's unique ID (used for fetching logs) + ID int64 + + // Name is the step name from the pipeline config + Name string + + // Status: pending, running, success, failure, killed, skipped, error + Status string + + // ExitCode from the step's command (nil if not finished) + ExitCode *int + + // Duration in seconds + Duration int + + // Error message if step failed + Error string + + // Log contains the last N lines of output (only populated for failed steps) + Log string +} + +// CIPipelineSteps contains step details for a pipeline. +type CIPipelineSteps struct { + // PipelineNumber is the pipeline number + PipelineNumber int64 + + // URL is the direct link to view this pipeline in the CI UI + URL string + + // Steps in execution order + Steps []CIPipelineStep +} diff --git a/internal/handlers/infrastructure.go b/internal/handlers/infrastructure.go index 3fd9365..3b5b0ff 100644 --- a/internal/handlers/infrastructure.go +++ b/internal/handlers/infrastructure.go @@ -110,6 +110,7 @@ func (h *InfrastructureHandler) Mount(r api.Router) { // CI pipeline endpoints r.Get("/projects/{id}/pipelines", h.ListPipelines) r.Get("/projects/{id}/pipelines/{number}", h.GetPipeline) + r.Get("/projects/{id}/pipelines/{number}/steps", h.GetPipelineSteps) } // CreateRepoRequest is the request body for POST /projects/{id}/repo. diff --git a/internal/handlers/infrastructure_pipelines.go b/internal/handlers/infrastructure_pipelines.go index 5301b9c..5ddbfd1 100644 --- a/internal/handlers/infrastructure_pipelines.go +++ b/internal/handlers/infrastructure_pipelines.go @@ -145,3 +145,72 @@ func mapPipelineErrors(errors []domain.CIPipelineError) []PipelineErrorResponse } return resp } + +// PipelineStepResponse is the JSON representation of a pipeline step. +type PipelineStepResponse struct { + ID int64 `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + ExitCode *int `json:"exit_code,omitempty"` + Duration int `json:"duration_seconds"` + Error string `json:"error,omitempty"` + Log string `json:"log,omitempty"` +} + +// PipelineStepsResponse is the JSON representation of pipeline steps. +type PipelineStepsResponse struct { + PipelineNumber int64 `json:"pipeline_number"` + URL string `json:"url"` + Steps []PipelineStepResponse `json:"steps"` +} + +// GetPipelineSteps returns detailed step information for a pipeline. +// GET /projects/{id}/pipelines/{number}/steps +func (h *InfrastructureHandler) GetPipelineSteps(w http.ResponseWriter, r *http.Request) { + projectID := chi.URLParam(r, "id") + numberStr := chi.URLParam(r, "number") + ctx, cancel := context.WithTimeout(r.Context(), TimeoutLookup) + defer cancel() + + if err := validateProjectID(projectID); err != nil { + api.WriteBadRequest(w, r, err.Error()) + return + } + + number, err := strconv.ParseInt(numberStr, 10, 64) + if err != nil { + api.WriteBadRequest(w, r, "invalid pipeline number") + return + } + + if h.ciProvider == nil { + api.WriteInternalError(w, r, "CI provider not configured") + return + } + + steps, err := h.ciProvider.GetPipelineSteps(ctx, h.defaultGitOwner, projectID, number) + if err != nil { + api.WriteNotFound(w, r, fmt.Sprintf("pipeline steps not found: %v", err)) + return + } + + // Map to response + respSteps := make([]PipelineStepResponse, len(steps.Steps)) + for i, s := range steps.Steps { + respSteps[i] = PipelineStepResponse{ + ID: s.ID, + Name: s.Name, + Status: s.Status, + ExitCode: s.ExitCode, + Duration: s.Duration, + Error: s.Error, + Log: s.Log, + } + } + + api.WriteSuccess(w, r, PipelineStepsResponse{ + PipelineNumber: steps.PipelineNumber, + URL: steps.URL, + Steps: respSteps, + }) +} diff --git a/internal/handlers/infrastructure_pipelines_test.go b/internal/handlers/infrastructure_pipelines_test.go index 145031f..0809cfb 100644 --- a/internal/handlers/infrastructure_pipelines_test.go +++ b/internal/handlers/infrastructure_pipelines_test.go @@ -74,6 +74,28 @@ func (m *mockCIProvider) TriggerBuild(_ context.Context, owner, repo, branch str return 1, nil } +func (m *mockCIProvider) GetPipelineSteps(_ context.Context, owner, repo string, number int64) (*domain.CIPipelineSteps, error) { + if m.err != nil { + return nil, m.err + } + key := owner + "/" + repo + // Check if pipeline exists + for _, p := range m.pipelines[key] { + if p.Number == number { + return &domain.CIPipelineSteps{ + PipelineNumber: number, + URL: fmt.Sprintf("https://ci.example.com/%s/%d", key, number), + Steps: []domain.CIPipelineStep{ + {ID: 1, Name: "clone", Status: "success", Duration: 5}, + {ID: 2, Name: "build", Status: "success", Duration: 30}, + {ID: 3, Name: "test", Status: "success", Duration: 15}, + }, + }, nil + } + } + return nil, fmt.Errorf("pipeline %d not found", number) +} + func setupInfraHandlerWithCI(ci port.CIProvider) chi.Router { h := NewInfrastructureHandler(nil, nil, nil, nil, ci, nil, InfrastructureConfig{ DefaultGitOwner: "threesix", @@ -238,6 +260,53 @@ func TestInfrastructureHandler_GetPipeline(t *testing.T) { }) } +func TestInfrastructureHandler_GetPipelineSteps(t *testing.T) { + t.Run("success", func(t *testing.T) { + ci := newMockCIProvider() + ci.pipelines["threesix/myapp"] = []*domain.CIPipeline{ + { + ID: 100, + Number: 5, + Status: "failure", + }, + } + + router := setupInfraHandlerWithCI(ci) + req := httptest.NewRequest("GET", "/projects/myapp/pipelines/5/steps", nil) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Errorf("status = %d, want %d, body: %s", rec.Code, http.StatusOK, rec.Body.String()) + } + }) + + t.Run("ci not configured", func(t *testing.T) { + router := setupInfraHandlerWithCI(nil) + req := httptest.NewRequest("GET", "/projects/myapp/pipelines/1/steps", nil) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusInternalServerError { + t.Errorf("status = %d, want %d", rec.Code, http.StatusInternalServerError) + } + }) + + t.Run("pipeline not found", func(t *testing.T) { + ci := newMockCIProvider() + ci.pipelines["threesix/myapp"] = []*domain.CIPipeline{} + router := setupInfraHandlerWithCI(ci) + + req := httptest.NewRequest("GET", "/projects/myapp/pipelines/999/steps", nil) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusNotFound { + t.Errorf("status = %d, want %d", rec.Code, http.StatusNotFound) + } + }) +} + func TestFormatTime(t *testing.T) { t.Run("non-zero time", func(t *testing.T) { ts := time.Date(2025, 1, 15, 10, 30, 0, 0, time.UTC) diff --git a/internal/port/ci_provider.go b/internal/port/ci_provider.go index ac8abd4..a4acd18 100644 --- a/internal/port/ci_provider.go +++ b/internal/port/ci_provider.go @@ -36,6 +36,10 @@ type CIProvider interface { // GetPipeline returns a specific pipeline execution by number. GetPipeline(ctx context.Context, owner, repo string, number int64) (*domain.CIPipeline, error) + // GetPipelineSteps returns detailed step information for a pipeline. + // For failed steps, includes the last N lines of log output. + GetPipelineSteps(ctx context.Context, owner, repo string, number int64) (*domain.CIPipelineSteps, error) + // TriggerBuild manually starts a new pipeline build on the specified branch. // Returns the pipeline number of the triggered build. TriggerBuild(ctx context.Context, owner, repo, branch string) (int64, error) diff --git a/internal/service/component.go b/internal/service/component.go index 3f844bf..0ea02f2 100644 --- a/internal/service/component.go +++ b/internal/service/component.go @@ -37,13 +37,16 @@ type ComponentService struct { db *sql.DB templateProvider port.TemplateProvider bulkClient *giteaadapter.BulkFileClient + deployer port.Deployer defaultGitOwner string + registryURL string logger *slog.Logger } // ComponentServiceConfig configures the component service. type ComponentServiceConfig struct { DefaultGitOwner string // e.g., "threesix" + RegistryURL string // e.g., "registry.threesix.ai" Logger *slog.Logger } @@ -52,6 +55,7 @@ func NewComponentService( db *sql.DB, templateProvider port.TemplateProvider, bulkClient *giteaadapter.BulkFileClient, + deployer port.Deployer, cfg ComponentServiceConfig, ) *ComponentService { logger := cfg.Logger @@ -62,7 +66,9 @@ func NewComponentService( db: db, templateProvider: templateProvider, bulkClient: bulkClient, + deployer: deployer, defaultGitOwner: cfg.DefaultGitOwner, + registryURL: cfg.RegistryURL, logger: logger, } } @@ -190,6 +196,10 @@ func (s *ComponentService) AddComponent(ctx context.Context, projectID string, r Dependencies: []string{}, // Could be parsed from component.yaml } + // 13. Create initial K8s deployment for components that need one. + // This ensures kubectl set image will find the deployment when CI runs. + s.createInitialComponentDeployment(ctx, projectID, projectDomain, component) + return component, nil } @@ -217,6 +227,47 @@ func (s *ComponentService) assignPort(ctx context.Context, projectID string, com return maxPort + 1, nil } +// createInitialComponentDeployment creates a K8s Deployment for a newly added component. +// This ensures the deployment exists before CI runs, so kubectl set image succeeds. +// Failures are logged but don't fail the component creation. +func (s *ComponentService) createInitialComponentDeployment( + ctx context.Context, + projectID, projectDomain string, + component *domain.Component, +) { + // Skip if no deployer or component doesn't need a deployment + if s.deployer == nil || !component.Type.NeedsPort() { + return + } + + // Build the image path - uses "latest" as placeholder until CI builds a real image + image := fmt.Sprintf("%s/%s/%s:latest", s.registryURL, projectID, component.Name) + + spec := domain.DeploySpec{ + ProjectName: projectID, + ComponentPath: component.Path, + Image: image, + Domain: projectDomain, + Port: component.Port, + Replicas: 1, + } + + if err := s.deployer.Deploy(ctx, spec); err != nil { + s.logger.Warn("failed to create initial component deployment", + "project", projectID, + "component", component.Name, + "error", err, + ) + return + } + + s.logger.Info("created initial component deployment", + "project", projectID, + "component", component.Name, + "image", image, + ) +} + // prepareMonorepoUpdates reads existing monorepo files and prepares updates. func (s *ComponentService) prepareMonorepoUpdates( ctx context.Context,