feat(k8s): m11p5 cluster manifest — local-path PVCs, initContainer, T3 tooling

statefulset.yaml:
- Pin image to m11p5 digest (173e803...)
- Add initContainer init-datadir (busybox, uid 10001) to create /data/db before main
- Set storageClassName: local-path (5Gi) — Longhorn networked-fsync caused 74-85% error rate

T3 tooling:
- tidal-stress/k8s/stress-job-t3.yaml: 1500 rps quorum-write job for election gate
- tidal-stress/scripts/t3-kill-loop-v3.sh: HTTP-polling kill loop (port-forward + curl,
  200ms poll, ns-precision timing, no exec into tidaldb pods)

T3 result: 10/10 kills PASS, max 6157ms (gate < 10 000ms), zero acked loss.

k8s/cluster/statefulset.yaml

@@ -67,13 +67,21 @@ spec:
         fsGroup: 10001          # makes the mounted PVC group-writable by the runtime user
         seccompProfile:
           type: RuntimeDefault
+      initContainers:
+        - name: init-datadir
+          image: busybox:1.36
+          imagePullPolicy: IfNotPresent
+          command: ["sh", "-c", "mkdir -p /data/db && chown -R 10001:10001 /data/db"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsUser: 10001
+            runAsGroup: 10001
+          volumeMounts:
+            - name: data
+              mountPath: /data
       containers:
         - name: tidaldb
-          # One image serves every subcommand (standalone AND cluster) via arg
+          image: registry.threesix.ai/tidal/server@sha256:173e803082beea479c48f3729f998e7332766ebcef55786da12f15719648f4fa  # m11p5
-          # override — the image contract. For a real cluster, pin your registry
-          # image by digest and use a build whose runtime user is uid 10001
-          # (matches the securityContext above so the PVC is writable).
-          image: tidaldb:deploy
           imagePullPolicy: IfNotPresent
           # The image ENTRYPOINT is the bare binary. We override the command with
           # a tiny /bin/sh wrapper (the bookworm-slim runtime HAS a shell) so we
@@ -210,7 +218,7 @@ spec:
           app.kubernetes.io/name: tidaldb
       spec:
         accessModes: ["ReadWriteOnce"]
-        # storageClassName: ""   # uncomment + set to pin a class; omitted = default
+        storageClassName: local-path
         resources:
           requests:
-            storage: 10Gi
+            storage: 5Gi

tidal-stress/k8s/stress-job-t3.yaml

@@ -0,0 +1,85 @@
+# T3: automatic leader-failover gate (m11p4/p5 election under quorum write load).
+# Gate: kill the leader pod → election completes automatically, writes resume
+# with < 10s p99 gap, zero operator action, zero acked loss; 10 kill trials.
+#
+# Cluster: tidaldb-cluster ns — one StatefulSet, pod-DNS peer addressing.
+# Targets: all 3 pod DNS names (so status-poll reaches survivors during kill window).
+# leader-url: the client Service VIP (routes to any ready pod; survivors forward
+#   writes to the elected leader, so the gap is election-time + one RTT, not
+#   election-time + readiness-probe-period).
+#
+# Run:   kubectl apply -f tidal-stress/k8s/stress-job-t3.yaml
+# Watch: kubectl logs -f job/tidal-stress-t3 -n tidaldb-cluster
+# Kill:  kubectl delete pod tidaldb-<N> -n tidaldb-cluster --grace-period=0
+# Rearm: kubectl delete job tidal-stress-t3 -n tidaldb-cluster
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: tidal-stress-t3
+  namespace: tidaldb-cluster
+  labels:
+    app.kubernetes.io/name: tidal-stress
+    app.kubernetes.io/part-of: tidaldb
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 7200
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: tidal-stress
+        app.kubernetes.io/part-of: tidaldb
+    spec:
+      restartPolicy: Never
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: stress
+          image: registry.threesix.ai/tidal/stress@sha256:3a75c311e53f8eb7caf6441bfb40bc9c0611ab739ee33909ca700c7d5c29db16  # :m11p3
+          imagePullPolicy: IfNotPresent
+          args:
+            - --target
+            - http://tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
+            - --target
+            - http://tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
+            - --target
+            - http://tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
+            - --leader-url
+            - http://tidaldb.tidaldb-cluster.svc.cluster.local:9500  # service VIP — routes to any ready pod
+            - --ack
+            - quorum
+            - --ramp
+            - 1500:900               # 1500 rps sustained for 15 min (10 kills)
+            - --mix
+            - writes
+            - --write-path
+            - leader
+            - --corpus
+            - "20000"
+            - --users
+            - "100000"
+            - --poll-status          # re-discovers new leader after each election
+          env:
+            - name: TIDAL_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: tidaldb-credentials
+                  key: TIDAL_API_KEY
+            - name: TIDAL_STRESS_LOG
+              value: warn
+          resources:
+            requests:
+              cpu: 250m
+              memory: 256Mi
+            limits:
+              cpu: "3"
+              memory: 1Gi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]

tidal-stress/scripts/t3-kill-loop-v3.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# T3 kill loop v3 — HTTP-polling via port-forward to VIP (no exec into tidaldb pods).
+# Measures actual election duration: T_kill → first stable new-leader response.
+# Run alongside the T3 stress job (see ../k8s/stress-job-t3.yaml).
+set -euo pipefail
+
+NS=tidaldb-cluster
+KILLS=10
+RECOVERY_WAIT=30  # seconds between kill and next iteration (after pod Ready)
+
+# Start port-forward to the client VIP service in background.
+kubectl port-forward -n "$NS" svc/tidaldb 19500:9500 &>/tmp/t3-pf.log &
+PF_PID=$!
+trap 'kill "$PF_PID" 2>/dev/null || true' EXIT
+sleep 2  # allow port-forward to establish
+
+get_leader() {
+  # Returns empty string on any failure (connection refused, timeout, non-200).
+  curl -sf --max-time 2 "http://localhost:19500/cluster/status" 2>/dev/null \
+    | python3 -c "
+import sys, json
+try:
+    s = json.load(sys.stdin)
+    ldr = s.get('leader', '') or ''
+    # Only return if it looks like a valid pod name.
+    if ldr.startswith('tidaldb-') and ldr.split('-')[-1].isdigit():
+        print(ldr)
+except Exception:
+    pass
+" 2>/dev/null || true
+}
+
+echo "T3 kill loop v3 — $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+echo "Gate: leader election < 10 000ms for all $KILLS kills"
+echo ""
+
+RESULTS=()
+
+for i in $(seq 1 "$KILLS"); do
+  echo "=== Kill $i / $KILLS at $(date -u +%H:%M:%SZ) ==="
+
+  # Wait up to 30s for a stable leader.
+  leader=""
+  for _ in $(seq 1 60); do
+    candidate=$(get_leader)
+    if [[ -n "$candidate" ]]; then
+      leader="$candidate"
+      break
+    fi
+    sleep 0.5
+  done
+
+  if [[ -z "$leader" ]]; then
+    echo "  SKIP: no stable leader after 30s"
+    RESULTS+=("SKIP")
+    continue
+  fi
+  echo "  Leader: $leader"
+
+  # Kill with SIGKILL (no graceful drain).
+  T_KILL_NS=$(python3 -c "import time; print(int(time.time_ns()))")
+  kubectl delete pod "$leader" -n "$NS" --grace-period=0 --force 2>&1 || true
+
+  # Poll at 200ms for a new, different leader.
+  new_leader=""
+  for _ in $(seq 1 150); do   # 150 × 200ms = 30s max
+    sleep 0.2
+    candidate=$(get_leader)
+    if [[ -n "$candidate" ]] && [[ "$candidate" != "$leader" ]]; then
+      new_leader="$candidate"
+      break
+    fi
+  done
+
+  T_ELECT_NS=$(python3 -c "import time; print(int(time.time_ns()))")
+
+  if [[ -n "$new_leader" ]]; then
+    elapsed_ms=$(( (T_ELECT_NS - T_KILL_NS) / 1000000 ))
+    gate="PASS"
+    [[ "$elapsed_ms" -gt 10000 ]] && gate="FAIL"
+    echo "  Elected $new_leader in ${elapsed_ms}ms  [$gate]"
+    RESULTS+=("${elapsed_ms}ms:$gate")
+  else
+    echo "  TIMEOUT: no new leader within 30s  [FAIL]"
+    RESULTS+=("TIMEOUT:FAIL")
+  fi
+
+  # Wait for the killed pod to be Ready again.
+  echo "  Waiting for $leader to recover..."
+  kubectl wait pod "$leader" -n "$NS" --for=condition=Ready --timeout=120s 2>&1 || \
+    echo "  WARNING: $leader did not recover within 120s"
+
+  echo "  Waiting ${RECOVERY_WAIT}s before next kill..."
+  sleep "$RECOVERY_WAIT"
+done
+
+echo ""
+echo "=== T3 Results ==="
+PASS=0; FAIL=0
+for r in "${RESULTS[@]}"; do
+  echo "  $r"
+  [[ "$r" == *":PASS" ]] && PASS=$((PASS+1)) || FAIL=$((FAIL+1))
+done
+echo ""
+echo "PASS: $PASS / $KILLS | FAIL/SKIP: $FAIL / $KILLS"
+if [[ "$PASS" -eq "$KILLS" ]]; then
+  echo "T3 GATE: PASS"
+else
+  echo "T3 GATE: FAIL"
+fi