25ec7630a1
statefulset.yaml: - Pin image to m11p5 digest (173e803...) - Add initContainer init-datadir (busybox, uid 10001) to create /data/db before main - Set storageClassName: local-path (5Gi) — Longhorn networked-fsync caused 74-85% error rate T3 tooling: - tidal-stress/k8s/stress-job-t3.yaml: 1500 rps quorum-write job for election gate - tidal-stress/scripts/t3-kill-loop-v3.sh: HTTP-polling kill loop (port-forward + curl, 200ms poll, ns-precision timing, no exec into tidaldb pods) T3 result: 10/10 kills PASS, max 6157ms (gate < 10 000ms), zero acked loss.
111 lines
3.1 KiB
Bash
Executable File
111 lines
3.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# T3 kill loop v3 — HTTP-polling via port-forward to VIP (no exec into tidaldb pods).
|
||
# Measures actual election duration: T_kill → first stable new-leader response.
|
||
# Run alongside the T3 stress job (see ../k8s/stress-job-t3.yaml).
|
||
set -euo pipefail
|
||
|
||
NS=tidaldb-cluster
|
||
KILLS=10
|
||
RECOVERY_WAIT=30 # seconds between kill and next iteration (after pod Ready)
|
||
|
||
# Start port-forward to the client VIP service in background.
|
||
kubectl port-forward -n "$NS" svc/tidaldb 19500:9500 &>/tmp/t3-pf.log &
|
||
PF_PID=$!
|
||
trap 'kill "$PF_PID" 2>/dev/null || true' EXIT
|
||
sleep 2 # allow port-forward to establish
|
||
|
||
get_leader() {
|
||
# Returns empty string on any failure (connection refused, timeout, non-200).
|
||
curl -sf --max-time 2 "http://localhost:19500/cluster/status" 2>/dev/null \
|
||
| python3 -c "
|
||
import sys, json
|
||
try:
|
||
s = json.load(sys.stdin)
|
||
ldr = s.get('leader', '') or ''
|
||
# Only return if it looks like a valid pod name.
|
||
if ldr.startswith('tidaldb-') and ldr.split('-')[-1].isdigit():
|
||
print(ldr)
|
||
except Exception:
|
||
pass
|
||
" 2>/dev/null || true
|
||
}
|
||
|
||
echo "T3 kill loop v3 — $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||
echo "Gate: leader election < 10 000ms for all $KILLS kills"
|
||
echo ""
|
||
|
||
RESULTS=()
|
||
|
||
for i in $(seq 1 "$KILLS"); do
|
||
echo "=== Kill $i / $KILLS at $(date -u +%H:%M:%SZ) ==="
|
||
|
||
# Wait up to 30s for a stable leader.
|
||
leader=""
|
||
for _ in $(seq 1 60); do
|
||
candidate=$(get_leader)
|
||
if [[ -n "$candidate" ]]; then
|
||
leader="$candidate"
|
||
break
|
||
fi
|
||
sleep 0.5
|
||
done
|
||
|
||
if [[ -z "$leader" ]]; then
|
||
echo " SKIP: no stable leader after 30s"
|
||
RESULTS+=("SKIP")
|
||
continue
|
||
fi
|
||
echo " Leader: $leader"
|
||
|
||
# Kill with SIGKILL (no graceful drain).
|
||
T_KILL_NS=$(python3 -c "import time; print(int(time.time_ns()))")
|
||
kubectl delete pod "$leader" -n "$NS" --grace-period=0 --force 2>&1 || true
|
||
|
||
# Poll at 200ms for a new, different leader.
|
||
new_leader=""
|
||
for _ in $(seq 1 150); do # 150 × 200ms = 30s max
|
||
sleep 0.2
|
||
candidate=$(get_leader)
|
||
if [[ -n "$candidate" ]] && [[ "$candidate" != "$leader" ]]; then
|
||
new_leader="$candidate"
|
||
break
|
||
fi
|
||
done
|
||
|
||
T_ELECT_NS=$(python3 -c "import time; print(int(time.time_ns()))")
|
||
|
||
if [[ -n "$new_leader" ]]; then
|
||
elapsed_ms=$(( (T_ELECT_NS - T_KILL_NS) / 1000000 ))
|
||
gate="PASS"
|
||
[[ "$elapsed_ms" -gt 10000 ]] && gate="FAIL"
|
||
echo " Elected $new_leader in ${elapsed_ms}ms [$gate]"
|
||
RESULTS+=("${elapsed_ms}ms:$gate")
|
||
else
|
||
echo " TIMEOUT: no new leader within 30s [FAIL]"
|
||
RESULTS+=("TIMEOUT:FAIL")
|
||
fi
|
||
|
||
# Wait for the killed pod to be Ready again.
|
||
echo " Waiting for $leader to recover..."
|
||
kubectl wait pod "$leader" -n "$NS" --for=condition=Ready --timeout=120s 2>&1 || \
|
||
echo " WARNING: $leader did not recover within 120s"
|
||
|
||
echo " Waiting ${RECOVERY_WAIT}s before next kill..."
|
||
sleep "$RECOVERY_WAIT"
|
||
done
|
||
|
||
echo ""
|
||
echo "=== T3 Results ==="
|
||
PASS=0; FAIL=0
|
||
for r in "${RESULTS[@]}"; do
|
||
echo " $r"
|
||
[[ "$r" == *":PASS" ]] && PASS=$((PASS+1)) || FAIL=$((FAIL+1))
|
||
done
|
||
echo ""
|
||
echo "PASS: $PASS / $KILLS | FAIL/SKIP: $FAIL / $KILLS"
|
||
if [[ "$PASS" -eq "$KILLS" ]]; then
|
||
echo "T3 GATE: PASS"
|
||
else
|
||
echo "T3 GATE: FAIL"
|
||
fi
|