138ab7306e
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
Four consecutive Kaniko pushes failed with `dial tcp 208.122.204.172:443: connect: connection refused` while zot and Traefik were both healthy and my laptop reached the same URL fine. Measured from a pod in the same namespace, with the same labels the build Job carries: 14 of 24 requests to https://registry.threesix.ai/v2/ succeeded, and 8 of 8 succeeded against Traefik's ClusterIP with the same SNI. So the flaky leg is pod -> the cluster's own public address, and the build was taking it for both the clone and the push. The Job now carries hostAliases pinning git.threesix.ai and registry.threesix.ai to Traefik's ClusterIP, read from the Service at release time. The image is still named registry.threesix.ai/hush/api:SHA — the route changes, the reference does not, and the kubelet pull path is untouched. No retry loop: a retry would have made a 40% failure rate invisible instead of fixed. docs/DEPLOY.md no longer duplicates the Job YAML. That duplicate is why the push guard drifted onto the wrong remote, and a second copy of a spec nobody runs is worth less than a pointer to the one that runs.
154 lines
6.7 KiB
Bash
Executable File
154 lines
6.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Build the current commit in-cluster and roll it out. No CI credential needed.
|
|
#
|
|
# Woodpecker IS activated for this repo, so a push to main builds and deploys.
|
|
# This is the path for when you do not want to wait for CI, when CI is down, or
|
|
# when you are rolling back — and it is how the first deploy happened, before
|
|
# activation. It does exactly what the pipeline's build and deploy steps do: a
|
|
# Kaniko Job for an amd64 image from the pushed git ref, then
|
|
# `kubectl set image`, then a real end-to-end check.
|
|
#
|
|
# Credentials: none. The Gitea repo is public so the Kaniko git context needs no
|
|
# token, and the rollout uses your kubeconfig.
|
|
#
|
|
# ./scripts/release.sh
|
|
set -euo pipefail
|
|
|
|
export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/orchard9-k3sf.yaml}"
|
|
NS="${NS:-projects}"
|
|
HOST="${HOST:-hush.threesix.ai}"
|
|
# The Gitea repo Kaniko clones, and the remote that points at it. Both are
|
|
# named once: the guard below has to check the ref that gets BUILT, and a
|
|
# guard that checks a different remote is worse than no guard.
|
|
GIT_CONTEXT="${GIT_CONTEXT:-git://git.threesix.ai/jordan/hush.git#refs/heads/main}"
|
|
GIT_REMOTE="${GIT_REMOTE:-origin}"
|
|
GIT_BRANCH="${GIT_BRANCH:-main}"
|
|
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
cd "$ROOT"
|
|
|
|
# Kaniko builds from the GIT CONTEXT, not from this working tree. So a dirty or
|
|
# unpushed tree would silently build something other than what you are looking
|
|
# at — the single most confusing failure this script can have. Refuse instead.
|
|
if [ -n "$(git status --porcelain)" ]; then
|
|
echo "refusing: working tree is dirty. Kaniko builds from the pushed git ref," >&2
|
|
echo "so uncommitted changes would NOT be in the image." >&2
|
|
git status --short >&2
|
|
exit 1
|
|
fi
|
|
# `@{upstream}` is NOT the right comparison: this checkout tracks a mirror, so
|
|
# HEAD can be pushed there while Gitea — the repo Kaniko clones — is behind,
|
|
# and the build would silently produce the previous commit. Compare against the
|
|
# branch that actually gets built.
|
|
git fetch --quiet "$GIT_REMOTE" "$GIT_BRANCH"
|
|
if [ "$(git rev-parse HEAD)" != "$(git rev-parse FETCH_HEAD)" ]; then
|
|
echo "refusing: HEAD is not what $GIT_REMOTE/$GIT_BRANCH points at, and Kaniko clones from there." >&2
|
|
echo " HEAD $(git rev-parse --short=8 HEAD) $(git log -1 --format=%s HEAD)" >&2
|
|
echo " $GIT_REMOTE/$GIT_BRANCH $(git rev-parse --short=8 FETCH_HEAD) $(git log -1 --format=%s FETCH_HEAD)" >&2
|
|
echo "Push to $GIT_REMOTE first: git push $GIT_REMOTE $GIT_BRANCH" >&2
|
|
exit 1
|
|
fi
|
|
|
|
SHA="$(git rev-parse --short=8 HEAD)"
|
|
IMAGE="registry.threesix.ai/hush/api:$SHA"
|
|
JOB="hush-build-$SHA"
|
|
echo "releasing $SHA"
|
|
|
|
# Kaniko clones git.threesix.ai and pushes registry.threesix.ai. Both names
|
|
# resolve to the cluster's PUBLIC address, and reaching that from inside a pod
|
|
# takes a hairpin path that drops connections: measured 2026-09-05, 14 of 24
|
|
# requests from a pod succeeded, four consecutive kaniko pushes were refused,
|
|
# and Traefik's own ClusterIP answered 8 of 8. One Traefik serves both names, so
|
|
# the build resolves them to that ClusterIP and never leaves the cluster. The
|
|
# pushed image is still named registry.threesix.ai/hush/api:SHA, which is what
|
|
# the kubelet pulls — this changes the route, not the reference.
|
|
TRAEFIK_IP="$(kubectl -n kube-system get svc traefik -o jsonpath='{.spec.clusterIP}')"
|
|
if [ -z "$TRAEFIK_IP" ]; then
|
|
echo "refusing: kube-system/traefik has no ClusterIP, so the build has no in-cluster route" >&2
|
|
exit 1
|
|
fi
|
|
echo " build resolves git+registry to traefik at $TRAEFIK_IP"
|
|
|
|
# A previous attempt at the same SHA leaves a completed Job that cannot be
|
|
# re-created; replacing it is the idempotent thing to do.
|
|
kubectl -n "$NS" delete job "$JOB" --ignore-not-found >/dev/null
|
|
|
|
kubectl -n "$NS" apply -f - >/dev/null <<EOF
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: $JOB
|
|
labels: { app: hush, component: build }
|
|
spec:
|
|
backoffLimit: 1
|
|
ttlSecondsAfterFinished: 3600
|
|
template:
|
|
metadata:
|
|
labels: { app: hush, component: build }
|
|
spec:
|
|
restartPolicy: Never
|
|
# See the note above: the public address is not reliably reachable from a
|
|
# pod, and the ClusterIP is.
|
|
hostAliases:
|
|
- ip: $TRAEFIK_IP
|
|
hostnames: [git.threesix.ai, registry.threesix.ai]
|
|
containers:
|
|
- name: kaniko
|
|
image: gcr.io/kaniko-project/executor:v1.23.2
|
|
args:
|
|
# The Gitea repo is public, so the git context needs no credential.
|
|
- --context=$GIT_CONTEXT
|
|
- --dockerfile=Dockerfile
|
|
- --destination=$IMAGE
|
|
# The internal Zot registry serves a self-signed cert.
|
|
- --skip-tls-verify
|
|
- --skip-tls-verify-pull
|
|
- --single-snapshot
|
|
resources:
|
|
requests: { cpu: 500m, memory: 1Gi }
|
|
limits: { cpu: "2", memory: 3Gi }
|
|
EOF
|
|
|
|
# Poll rather than `kubectl wait --for=condition=complete`. That is one long
|
|
# WATCH against a cluster on the other side of a WAN link: measured 2026-09-05,
|
|
# the Job reached Complete in 156s and the watch still sat there until its
|
|
# 900s timeout, then reported "build FAILED" for an image that had already been
|
|
# pushed. Each poll below is a fresh short request, so a dropped connection
|
|
# costs one poll — and a deadline here means "still building", never "failed".
|
|
echo " building (amd64, in-cluster)…"
|
|
DEADLINE=$((SECONDS + 900))
|
|
while :; do
|
|
# A missing field prints nothing, so default to 0 and keep the comparison
|
|
# numeric. `|| true` covers a poll that loses the connection outright.
|
|
SUCCEEDED=$(kubectl -n "$NS" get "job/$JOB" -o jsonpath='{.status.succeeded}' 2>/dev/null || true)
|
|
FAILED=$(kubectl -n "$NS" get "job/$JOB" -o jsonpath='{.status.failed}' 2>/dev/null || true)
|
|
if [ "${SUCCEEDED:-0}" -ge 1 ]; then
|
|
break
|
|
fi
|
|
if [ "${FAILED:-0}" -ge 1 ]; then
|
|
echo "build FAILED — last lines:" >&2
|
|
kubectl -n "$NS" logs "job/$JOB" --tail=30 >&2
|
|
exit 1
|
|
fi
|
|
if [ "$SECONDS" -ge "$DEADLINE" ]; then
|
|
echo "the build has not finished after 900s. It may still be running:" >&2
|
|
echo " kubectl -n $NS get job/$JOB" >&2
|
|
echo " kubectl -n $NS logs job/$JOB --tail=30" >&2
|
|
exit 1
|
|
fi
|
|
sleep 5
|
|
done
|
|
echo " built $IMAGE"
|
|
|
|
kubectl -n "$NS" set image deployment/hush "hushd=$IMAGE" >/dev/null
|
|
kubectl -n "$NS" rollout status deployment/hush --timeout=180s | sed 's/^/ /'
|
|
|
|
# Prove the rolled pod is the image we just built. `set image` matching nothing
|
|
# is silent, and the rollout would "succeed" on the old pod.
|
|
LIVE="$(kubectl -n "$NS" get deployment hush -o jsonpath='{.spec.template.spec.containers[0].image}')"
|
|
[ "$LIVE" = "$IMAGE" ] || { echo "live image is $LIVE, expected $IMAGE" >&2; exit 1; }
|
|
echo " live image: $LIVE"
|
|
|
|
echo
|
|
echo "verifying end to end against https://$HOST"
|
|
BASE="https://$HOST" "$ROOT/scripts/smoke.sh"
|