diff --git a/k8s/cluster/statefulset.yaml b/k8s/cluster/statefulset.yaml index 56d4024..c557f51 100644 --- a/k8s/cluster/statefulset.yaml +++ b/k8s/cluster/statefulset.yaml @@ -67,13 +67,21 @@ spec: fsGroup: 10001 # makes the mounted PVC group-writable by the runtime user seccompProfile: type: RuntimeDefault + initContainers: + - name: init-datadir + image: busybox:1.36 + imagePullPolicy: IfNotPresent + command: ["sh", "-c", "mkdir -p /data/db && chown -R 10001:10001 /data/db"] + securityContext: + allowPrivilegeEscalation: false + runAsUser: 10001 + runAsGroup: 10001 + volumeMounts: + - name: data + mountPath: /data containers: - name: tidaldb - # One image serves every subcommand (standalone AND cluster) via arg - # override — the image contract. For a real cluster, pin your registry - # image by digest and use a build whose runtime user is uid 10001 - # (matches the securityContext above so the PVC is writable). - image: tidaldb:deploy + image: registry.threesix.ai/tidal/server@sha256:173e803082beea479c48f3729f998e7332766ebcef55786da12f15719648f4fa # m11p5 imagePullPolicy: IfNotPresent # The image ENTRYPOINT is the bare binary. We override the command with # a tiny /bin/sh wrapper (the bookworm-slim runtime HAS a shell) so we @@ -210,7 +218,7 @@ spec: app.kubernetes.io/name: tidaldb spec: accessModes: ["ReadWriteOnce"] - # storageClassName: "" # uncomment + set to pin a class; omitted = default + storageClassName: local-path resources: requests: - storage: 10Gi + storage: 5Gi diff --git a/tidal-stress/k8s/stress-job-t3.yaml b/tidal-stress/k8s/stress-job-t3.yaml new file mode 100644 index 0000000..ac9caed --- /dev/null +++ b/tidal-stress/k8s/stress-job-t3.yaml @@ -0,0 +1,85 @@ +# T3: automatic leader-failover gate (m11p4/p5 election under quorum write load). +# Gate: kill the leader pod → election completes automatically, writes resume +# with < 10s p99 gap, zero operator action, zero acked loss; 10 kill trials. +# +# Cluster: tidaldb-cluster ns — one StatefulSet, pod-DNS peer addressing. +# Targets: all 3 pod DNS names (so status-poll reaches survivors during kill window). +# leader-url: the client Service VIP (routes to any ready pod; survivors forward +# writes to the elected leader, so the gap is election-time + one RTT, not +# election-time + readiness-probe-period). +# +# Run: kubectl apply -f tidal-stress/k8s/stress-job-t3.yaml +# Watch: kubectl logs -f job/tidal-stress-t3 -n tidaldb-cluster +# Kill: kubectl delete pod tidaldb- -n tidaldb-cluster --grace-period=0 +# Rearm: kubectl delete job tidal-stress-t3 -n tidaldb-cluster +apiVersion: batch/v1 +kind: Job +metadata: + name: tidal-stress-t3 + namespace: tidaldb-cluster + labels: + app.kubernetes.io/name: tidal-stress + app.kubernetes.io/part-of: tidaldb +spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 7200 + template: + metadata: + labels: + app.kubernetes.io/name: tidal-stress + app.kubernetes.io/part-of: tidaldb + spec: + restartPolicy: Never + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: stress + image: registry.threesix.ai/tidal/stress@sha256:3a75c311e53f8eb7caf6441bfb40bc9c0611ab739ee33909ca700c7d5c29db16 # :m11p3 + imagePullPolicy: IfNotPresent + args: + - --target + - http://tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + - --target + - http://tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + - --target + - http://tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + - --leader-url + - http://tidaldb.tidaldb-cluster.svc.cluster.local:9500 # service VIP — routes to any ready pod + - --ack + - quorum + - --ramp + - 1500:900 # 1500 rps sustained for 15 min (10 kills) + - --mix + - writes + - --write-path + - leader + - --corpus + - "20000" + - --users + - "100000" + - --poll-status # re-discovers new leader after each election + env: + - name: TIDAL_API_KEY + valueFrom: + secretKeyRef: + name: tidaldb-credentials + key: TIDAL_API_KEY + - name: TIDAL_STRESS_LOG + value: warn + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: "3" + memory: 1Gi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] diff --git a/tidal-stress/scripts/t3-kill-loop-v3.sh b/tidal-stress/scripts/t3-kill-loop-v3.sh new file mode 100755 index 0000000..1e19ed4 --- /dev/null +++ b/tidal-stress/scripts/t3-kill-loop-v3.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# T3 kill loop v3 — HTTP-polling via port-forward to VIP (no exec into tidaldb pods). +# Measures actual election duration: T_kill → first stable new-leader response. +# Run alongside the T3 stress job (see ../k8s/stress-job-t3.yaml). +set -euo pipefail + +NS=tidaldb-cluster +KILLS=10 +RECOVERY_WAIT=30 # seconds between kill and next iteration (after pod Ready) + +# Start port-forward to the client VIP service in background. +kubectl port-forward -n "$NS" svc/tidaldb 19500:9500 &>/tmp/t3-pf.log & +PF_PID=$! +trap 'kill "$PF_PID" 2>/dev/null || true' EXIT +sleep 2 # allow port-forward to establish + +get_leader() { + # Returns empty string on any failure (connection refused, timeout, non-200). + curl -sf --max-time 2 "http://localhost:19500/cluster/status" 2>/dev/null \ + | python3 -c " +import sys, json +try: + s = json.load(sys.stdin) + ldr = s.get('leader', '') or '' + # Only return if it looks like a valid pod name. + if ldr.startswith('tidaldb-') and ldr.split('-')[-1].isdigit(): + print(ldr) +except Exception: + pass +" 2>/dev/null || true +} + +echo "T3 kill loop v3 — $(date -u +%Y-%m-%dT%H:%M:%SZ)" +echo "Gate: leader election < 10 000ms for all $KILLS kills" +echo "" + +RESULTS=() + +for i in $(seq 1 "$KILLS"); do + echo "=== Kill $i / $KILLS at $(date -u +%H:%M:%SZ) ===" + + # Wait up to 30s for a stable leader. + leader="" + for _ in $(seq 1 60); do + candidate=$(get_leader) + if [[ -n "$candidate" ]]; then + leader="$candidate" + break + fi + sleep 0.5 + done + + if [[ -z "$leader" ]]; then + echo " SKIP: no stable leader after 30s" + RESULTS+=("SKIP") + continue + fi + echo " Leader: $leader" + + # Kill with SIGKILL (no graceful drain). + T_KILL_NS=$(python3 -c "import time; print(int(time.time_ns()))") + kubectl delete pod "$leader" -n "$NS" --grace-period=0 --force 2>&1 || true + + # Poll at 200ms for a new, different leader. + new_leader="" + for _ in $(seq 1 150); do # 150 × 200ms = 30s max + sleep 0.2 + candidate=$(get_leader) + if [[ -n "$candidate" ]] && [[ "$candidate" != "$leader" ]]; then + new_leader="$candidate" + break + fi + done + + T_ELECT_NS=$(python3 -c "import time; print(int(time.time_ns()))") + + if [[ -n "$new_leader" ]]; then + elapsed_ms=$(( (T_ELECT_NS - T_KILL_NS) / 1000000 )) + gate="PASS" + [[ "$elapsed_ms" -gt 10000 ]] && gate="FAIL" + echo " Elected $new_leader in ${elapsed_ms}ms [$gate]" + RESULTS+=("${elapsed_ms}ms:$gate") + else + echo " TIMEOUT: no new leader within 30s [FAIL]" + RESULTS+=("TIMEOUT:FAIL") + fi + + # Wait for the killed pod to be Ready again. + echo " Waiting for $leader to recover..." + kubectl wait pod "$leader" -n "$NS" --for=condition=Ready --timeout=120s 2>&1 || \ + echo " WARNING: $leader did not recover within 120s" + + echo " Waiting ${RECOVERY_WAIT}s before next kill..." + sleep "$RECOVERY_WAIT" +done + +echo "" +echo "=== T3 Results ===" +PASS=0; FAIL=0 +for r in "${RESULTS[@]}"; do + echo " $r" + [[ "$r" == *":PASS" ]] && PASS=$((PASS+1)) || FAIL=$((FAIL+1)) +done +echo "" +echo "PASS: $PASS / $KILLS | FAIL/SKIP: $FAIL / $KILLS" +if [[ "$PASS" -eq "$KILLS" ]]; then + echo "T3 GATE: PASS" +else + echo "T3 GATE: FAIL" +fi