From d923b036af9729c81a14715732161ed5dc9cdbb7 Mon Sep 17 00:00:00 2001 From: jordan Date: Mon, 17 Aug 2026 20:52:32 -0600 Subject: [PATCH] k8s(cluster): three voters is the desired state, on the read-path fix image Two outages in one session came from this file claiming `replicas: 0` while the cluster served traffic: a plain `kubectl apply` scaled it to zero, twice. Source now states production intent. Parking stays an explicit `kubectl scale` divergence recorded in k3s-fleet/cluster-state.yaml, and scripts/restore-fleet.sh remains the guarded path back - its storage, image, and per-node capacity preflights are exactly what a bare apply does not do. Pins server:m12-consumer-readpath-20260818@sha256:9191233d..., the build with the transport-aware read budget and the scatter-degraded counters. Verified after the rolling update: 3/3 Ready, all regions lag 0, every replica answering 3/3 shards with degraded=false on the default budget. --- k8s/cluster/statefulset.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/k8s/cluster/statefulset.yaml b/k8s/cluster/statefulset.yaml index 1ca78b8..5d56a30 100644 --- a/k8s/cluster/statefulset.yaml +++ b/k8s/cluster/statefulset.yaml @@ -21,7 +21,15 @@ metadata: app.kubernetes.io/component: cluster-node spec: serviceName: tidaldb-peers # the headless peer Service — stable per-pod DNS - replicas: 0 # parked; scripts/restore-fleet.sh restores three voters explicitly + # PRODUCTION as of 2026-08-18: three voters is the desired state, not a value + # only the restore script may set. `replicas: 0` lived here while the cluster + # was parked and it cost two outages in one session: a plain `kubectl apply` of + # this file scaled a LIVE cluster to zero, twice, because the manifest still + # claimed parked. Source states intent; parking is an explicit `kubectl scale` + # divergence recorded in k3s-fleet/cluster-state.yaml, and + # scripts/restore-fleet.sh remains the guarded path back (its storage, image, + # and per-node capacity preflights are what a bare apply does not do). + replicas: 3 # Parallel: bring all pods up at once. There is no ordered-bootstrap # dependency — siblings boot in any order (an unreachable-at-startup peer is # normal; the election + catch-up timer converge them). Ordered start would @@ -83,7 +91,7 @@ spec: mountPath: /data containers: - name: tidaldb - image: registry.threesix.ai/tidal/server:m12-fleet-remediation-20260813@sha256:2e4baaf974ad2cf650609b1843689122bf4fd2751bcb8749906df8089223f4a3 + image: registry.threesix.ai/tidal/server:m12-consumer-readpath-20260818@sha256:9191233d012e41fd92bb6d0efea83b60010995a6fc6e3ebc894f3d7c5f72872a imagePullPolicy: IfNotPresent # The image ENTRYPOINT is the bare binary. We override the command with # a tiny /bin/sh wrapper (the bookworm-slim runtime HAS a shell) so we