diff --git a/Makefile b/Makefile index c11a449..b0159a3 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL := help .PHONY: help fmt vet test test-redis build run dev dev-stop smoke vendor verify ci \ - mcp mcp-install deploy-manifests deploy-status logs alerts-check + mcp mcp-install release deploy-manifests deploy-status logs alerts-check # Local development Redis. A real server, not a mock: the one-time guarantee # rests on GETDEL being atomic, and a fake cannot prove that. @@ -89,3 +89,6 @@ logs: ## Tail hush's structured logs out of VictoriaLogs alerts-check: ## Confirm vmalert has loaded hush's rules @./scripts/alerts-check.sh + +release: ## Build this commit in-cluster and roll it out, then smoke it. Needs no CI credential. + @./scripts/release.sh diff --git a/README.md b/README.md index 49cf80b..74d960a 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,7 @@ from a browser. See [docs/MCP.md](docs/MCP.md). ```bash make help # every target +make release # build this commit in-cluster and roll it out, then smoke it make test # unit tests, no external dependencies make dev # a local Redis in Docker + hushd on :18500 make smoke # full create → reveal → gone against the local instance diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 4579c5f..e3833ec 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -157,8 +157,22 @@ The repo exists on Gitea and `.woodpecker.yml` is committed, but activation failed: the `WOODPECKER_API_TOKEN` in `rdev/rdev-credentials` returns `401 User not authorized`. -Until a valid token replaces it, **pushes do not deploy** — use the Kaniko Job -above and `kubectl set image`. To finish it: +Until a valid token replaces it, **pushes do not deploy**. Rather than leave +that as a trap, `make release` does exactly what the pipeline's build and deploy +steps do — Kaniko Job, `set image`, rollout, then the production smoke — and +needs no CI credential: + +```bash +make release +``` + +It refuses on a dirty or unpushed tree, because Kaniko builds from the pushed +git ref and would otherwise silently build something other than what you are +looking at. It also asserts the live image equals the one just built, since +`set image` matching nothing is silent and the rollout would "succeed" on the +old pod. + +To finish the CI wiring: ```bash WP=$(curl -s -H "X-API-Key: $RDEV_API_KEY" "$RDEV_API_URL/credentials/WOODPECKER_API_TOKEN" | jq -r '.data.value') diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 0000000..3a2964f --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Build the current commit in-cluster and roll it out. No CI credential needed. +# +# This exists because Woodpecker is not activated for this repo (its API token +# in rdev returns 401, and minting a new one needs a browser login). Rather than +# leave "git push does not deploy" as a trap for whoever pushes next, this does +# exactly what the pipeline's build+deploy steps do: a Kaniko Job for an amd64 +# image, then `kubectl set image`, then a real end-to-end check. +# +# When Woodpecker is activated this becomes redundant, and that is fine — it is +# also the manual path for a rollback or a hotfix when CI is down. +# +# ./scripts/release.sh +set -euo pipefail + +export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/orchard9-k3sf.yaml}" +NS="${NS:-projects}" +HOST="${HOST:-hush.threesix.ai}" +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +# Kaniko builds from the GIT CONTEXT, not from this working tree. So a dirty or +# unpushed tree would silently build something other than what you are looking +# at — the single most confusing failure this script can have. Refuse instead. +if [ -n "$(git status --porcelain)" ]; then + echo "refusing: working tree is dirty. Kaniko builds from the pushed git ref," >&2 + echo "so uncommitted changes would NOT be in the image." >&2 + git status --short >&2 + exit 1 +fi +if [ -n "$(git log --oneline @{upstream}..HEAD 2>/dev/null)" ]; then + echo "refusing: HEAD is not pushed to origin (Gitea). Kaniko clones from there." >&2 + git log --oneline '@{upstream}..HEAD' >&2 + exit 1 +fi + +SHA="$(git rev-parse --short=8 HEAD)" +IMAGE="registry.threesix.ai/hush/api:$SHA" +JOB="hush-build-$SHA" +echo "releasing $SHA" + +# A previous attempt at the same SHA leaves a completed Job that cannot be +# re-created; replacing it is the idempotent thing to do. +kubectl -n "$NS" delete job "$JOB" --ignore-not-found >/dev/null + +kubectl -n "$NS" apply -f - >/dev/null </dev/null 2>&1; then + echo "build FAILED — last lines:" >&2 + kubectl -n "$NS" logs "job/$JOB" --tail=30 >&2 + exit 1 +fi +echo " built $IMAGE" + +kubectl -n "$NS" set image deployment/hush "hushd=$IMAGE" >/dev/null +kubectl -n "$NS" rollout status deployment/hush --timeout=180s | sed 's/^/ /' + +# Prove the rolled pod is the image we just built. `set image` matching nothing +# is silent, and the rollout would "succeed" on the old pod. +LIVE="$(kubectl -n "$NS" get deployment hush -o jsonpath='{.spec.template.spec.containers[0].image}')" +[ "$LIVE" = "$IMAGE" ] || { echo "live image is $LIVE, expected $IMAGE" >&2; exit 1; } +echo " live image: $LIVE" + +echo +echo "verifying end to end against https://$HOST" +BASE="https://$HOST" "$ROOT/scripts/smoke.sh"