From bc3374756822951dcb8828acb7e0a8443f3ba6fe Mon Sep 17 00:00:00 2001
From: jx12n <jordan.washburn@gmail.com>
Date: Sat, 5 Sep 2026 14:21:19 -0600
Subject: [PATCH] release: poll the build Job instead of watching it
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Measured on the previous commit's release: the Kaniko Job reached Complete in
156s and pushed registry.threesix.ai/hush/api:d7cd57f3, and
`kubectl wait --for=condition=complete` sat on its watch until the 900s
timeout, then reported "build FAILED" and exited before the rollout. The
image existed; the deploy did not happen; the message said the opposite of
what had happened.

One long watch against a cluster across a WAN link is the wrong instrument. A
fresh short GET every 5s costs one poll when a connection drops, distinguishes
`.status.failed` from "not finished yet", and reports the deadline as "may
still be running" with the two commands to check — never as a failure.

Both branches exercised against the cluster with the loop as committed: a Job
that exits 1 is reported as FAILED with its last lines, and the already
complete d7cd57f3 Job breaks the loop immediately.
---
 scripts/release.sh | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/scripts/release.sh b/scripts/release.sh
index 5f27048..6a6cc0b 100755
--- a/scripts/release.sh
+++ b/scripts/release.sh
@@ -88,12 +88,35 @@ spec:
             limits: { cpu: "2", memory: 3Gi }
 EOF
 
+# Poll rather than `kubectl wait --for=condition=complete`. That is one long
+# WATCH against a cluster on the other side of a WAN link: measured 2026-09-05,
+# the Job reached Complete in 156s and the watch still sat there until its
+# 900s timeout, then reported "build FAILED" for an image that had already been
+# pushed. Each poll below is a fresh short request, so a dropped connection
+# costs one poll — and a deadline here means "still building", never "failed".
 echo "  building (amd64, in-cluster)…"
-if ! kubectl -n "$NS" wait --for=condition=complete "job/$JOB" --timeout=900s >/dev/null 2>&1; then
-  echo "build FAILED — last lines:" >&2
-  kubectl -n "$NS" logs "job/$JOB" --tail=30 >&2
-  exit 1
-fi
+DEADLINE=$((SECONDS + 900))
+while :; do
+  # A missing field prints nothing, so default to 0 and keep the comparison
+  # numeric. `|| true` covers a poll that loses the connection outright.
+  SUCCEEDED=$(kubectl -n "$NS" get "job/$JOB" -o jsonpath='{.status.succeeded}' 2>/dev/null || true)
+  FAILED=$(kubectl -n "$NS" get "job/$JOB" -o jsonpath='{.status.failed}' 2>/dev/null || true)
+  if [ "${SUCCEEDED:-0}" -ge 1 ]; then
+    break
+  fi
+  if [ "${FAILED:-0}" -ge 1 ]; then
+    echo "build FAILED — last lines:" >&2
+    kubectl -n "$NS" logs "job/$JOB" --tail=30 >&2
+    exit 1
+  fi
+  if [ "$SECONDS" -ge "$DEADLINE" ]; then
+    echo "the build has not finished after 900s. It may still be running:" >&2
+    echo "  kubectl -n $NS get job/$JOB" >&2
+    echo "  kubectl -n $NS logs job/$JOB --tail=30" >&2
+    exit 1
+  fi
+  sleep 5
+done
 echo "  built $IMAGE"
 
 kubectl -n "$NS" set image deployment/hush "hushd=$IMAGE" >/dev/null