From d32283e36a9e98b5b6ec2cc78e06c570362a4596 Mon Sep 17 00:00:00 2001 From: jx12n Date: Thu, 3 Sep 2026 00:22:13 -0600 Subject: [PATCH] make release: deployment that needs no CI credential MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Woodpecker is not activated for this repo — the WOODPECKER_API_TOKEN in rdev-credentials returns 401 on /api/user, so it is the token and not the request shape, and minting a new one needs a browser login I cannot do. That left `git push` silently not deploying, which is a trap for whoever pushes next. So `make release` does what the pipeline's build and deploy steps do: a Kaniko Job for an amd64 image from the pushed git ref, `kubectl set image`, rollout, then the production smoke. When Woodpecker is activated this becomes redundant, and stays useful as the manual path for a hotfix or a rollback when CI is down. Two refusals in it are the interesting part, both for failures that are otherwise silent: * A dirty or unpushed tree is refused. Kaniko builds from the GIT CONTEXT, not the working tree, so uncommitted work would produce an image that does not contain it while every log line says success. * After the rollout it asserts the live image equals the one just built. `kubectl set image` matching no container is silent, and the rollout then "succeeds" against the old pod. --- Makefile | 5 ++- README.md | 1 + docs/DEPLOY.md | 18 ++++++++- scripts/release.sh | 96 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+), 3 deletions(-) create mode 100755 scripts/release.sh diff --git a/Makefile b/Makefile index c11a449..b0159a3 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL := help .PHONY: help fmt vet test test-redis build run dev dev-stop smoke vendor verify ci \ - mcp mcp-install deploy-manifests deploy-status logs alerts-check + mcp mcp-install release deploy-manifests deploy-status logs alerts-check # Local development Redis. A real server, not a mock: the one-time guarantee # rests on GETDEL being atomic, and a fake cannot prove that. @@ -89,3 +89,6 @@ logs: ## Tail hush's structured logs out of VictoriaLogs alerts-check: ## Confirm vmalert has loaded hush's rules @./scripts/alerts-check.sh + +release: ## Build this commit in-cluster and roll it out, then smoke it. Needs no CI credential. + @./scripts/release.sh diff --git a/README.md b/README.md index 49cf80b..74d960a 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,7 @@ from a browser. See [docs/MCP.md](docs/MCP.md). ```bash make help # every target +make release # build this commit in-cluster and roll it out, then smoke it make test # unit tests, no external dependencies make dev # a local Redis in Docker + hushd on :18500 make smoke # full create → reveal → gone against the local instance diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 4579c5f..e3833ec 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -157,8 +157,22 @@ The repo exists on Gitea and `.woodpecker.yml` is committed, but activation failed: the `WOODPECKER_API_TOKEN` in `rdev/rdev-credentials` returns `401 User not authorized`. -Until a valid token replaces it, **pushes do not deploy** — use the Kaniko Job -above and `kubectl set image`. To finish it: +Until a valid token replaces it, **pushes do not deploy**. Rather than leave +that as a trap, `make release` does exactly what the pipeline's build and deploy +steps do — Kaniko Job, `set image`, rollout, then the production smoke — and +needs no CI credential: + +```bash +make release +``` + +It refuses on a dirty or unpushed tree, because Kaniko builds from the pushed +git ref and would otherwise silently build something other than what you are +looking at. It also asserts the live image equals the one just built, since +`set image` matching nothing is silent and the rollout would "succeed" on the +old pod. + +To finish the CI wiring: ```bash WP=$(curl -s -H "X-API-Key: $RDEV_API_KEY" "$RDEV_API_URL/credentials/WOODPECKER_API_TOKEN" | jq -r '.data.value') diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 0000000..3a2964f --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Build the current commit in-cluster and roll it out. No CI credential needed. +# +# This exists because Woodpecker is not activated for this repo (its API token +# in rdev returns 401, and minting a new one needs a browser login). Rather than +# leave "git push does not deploy" as a trap for whoever pushes next, this does +# exactly what the pipeline's build+deploy steps do: a Kaniko Job for an amd64 +# image, then `kubectl set image`, then a real end-to-end check. +# +# When Woodpecker is activated this becomes redundant, and that is fine — it is +# also the manual path for a rollback or a hotfix when CI is down. +# +# ./scripts/release.sh +set -euo pipefail + +export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/orchard9-k3sf.yaml}" +NS="${NS:-projects}" +HOST="${HOST:-hush.threesix.ai}" +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +# Kaniko builds from the GIT CONTEXT, not from this working tree. So a dirty or +# unpushed tree would silently build something other than what you are looking +# at — the single most confusing failure this script can have. Refuse instead. +if [ -n "$(git status --porcelain)" ]; then + echo "refusing: working tree is dirty. Kaniko builds from the pushed git ref," >&2 + echo "so uncommitted changes would NOT be in the image." >&2 + git status --short >&2 + exit 1 +fi +if [ -n "$(git log --oneline @{upstream}..HEAD 2>/dev/null)" ]; then + echo "refusing: HEAD is not pushed to origin (Gitea). Kaniko clones from there." >&2 + git log --oneline '@{upstream}..HEAD' >&2 + exit 1 +fi + +SHA="$(git rev-parse --short=8 HEAD)" +IMAGE="registry.threesix.ai/hush/api:$SHA" +JOB="hush-build-$SHA" +echo "releasing $SHA" + +# A previous attempt at the same SHA leaves a completed Job that cannot be +# re-created; replacing it is the idempotent thing to do. +kubectl -n "$NS" delete job "$JOB" --ignore-not-found >/dev/null + +kubectl -n "$NS" apply -f - >/dev/null </dev/null 2>&1; then + echo "build FAILED — last lines:" >&2 + kubectl -n "$NS" logs "job/$JOB" --tail=30 >&2 + exit 1 +fi +echo " built $IMAGE" + +kubectl -n "$NS" set image deployment/hush "hushd=$IMAGE" >/dev/null +kubectl -n "$NS" rollout status deployment/hush --timeout=180s | sed 's/^/ /' + +# Prove the rolled pod is the image we just built. `set image` matching nothing +# is silent, and the rollout would "succeed" on the old pod. +LIVE="$(kubectl -n "$NS" get deployment hush -o jsonpath='{.spec.template.spec.containers[0].image}')" +[ "$LIVE" = "$IMAGE" ] || { echo "live image is $LIVE, expected $IMAGE" >&2; exit 1; } +echo " live image: $LIVE" + +echo +echo "verifying end to end against https://$HOST" +BASE="https://$HOST" "$ROOT/scripts/smoke.sh"