feat(k8s): m11p5 cluster manifest — local-path PVCs, initContainer, T3 tooling
statefulset.yaml: - Pin image to m11p5 digest (173e803...) - Add initContainer init-datadir (busybox, uid 10001) to create /data/db before main - Set storageClassName: local-path (5Gi) — Longhorn networked-fsync caused 74-85% error rate T3 tooling: - tidal-stress/k8s/stress-job-t3.yaml: 1500 rps quorum-write job for election gate - tidal-stress/scripts/t3-kill-loop-v3.sh: HTTP-polling kill loop (port-forward + curl, 200ms poll, ns-precision timing, no exec into tidaldb pods) T3 result: 10/10 kills PASS, max 6157ms (gate < 10 000ms), zero acked loss.
This commit is contained in:
jx12n
parent
bf57be18e1
commit
25ec7630a1
@ -67,13 +67,21 @@ spec:
|
||||
fsGroup: 10001 # makes the mounted PVC group-writable by the runtime user
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
initContainers:
|
||||
- name: init-datadir
|
||||
image: busybox:1.36
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: ["sh", "-c", "mkdir -p /data/db && chown -R 10001:10001 /data/db"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
containers:
|
||||
- name: tidaldb
|
||||
# One image serves every subcommand (standalone AND cluster) via arg
|
||||
# override — the image contract. For a real cluster, pin your registry
|
||||
# image by digest and use a build whose runtime user is uid 10001
|
||||
# (matches the securityContext above so the PVC is writable).
|
||||
image: tidaldb:deploy
|
||||
image: registry.threesix.ai/tidal/server@sha256:173e803082beea479c48f3729f998e7332766ebcef55786da12f15719648f4fa # m11p5
|
||||
imagePullPolicy: IfNotPresent
|
||||
# The image ENTRYPOINT is the bare binary. We override the command with
|
||||
# a tiny /bin/sh wrapper (the bookworm-slim runtime HAS a shell) so we
|
||||
@ -210,7 +218,7 @@ spec:
|
||||
app.kubernetes.io/name: tidaldb
|
||||
spec:
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
# storageClassName: "" # uncomment + set to pin a class; omitted = default
|
||||
storageClassName: local-path
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
storage: 5Gi
|
||||
|
||||
85
tidal-stress/k8s/stress-job-t3.yaml
Normal file
85
tidal-stress/k8s/stress-job-t3.yaml
Normal file
@ -0,0 +1,85 @@
|
||||
# T3: automatic leader-failover gate (m11p4/p5 election under quorum write load).
|
||||
# Gate: kill the leader pod → election completes automatically, writes resume
|
||||
# with < 10s p99 gap, zero operator action, zero acked loss; 10 kill trials.
|
||||
#
|
||||
# Cluster: tidaldb-cluster ns — one StatefulSet, pod-DNS peer addressing.
|
||||
# Targets: all 3 pod DNS names (so status-poll reaches survivors during kill window).
|
||||
# leader-url: the client Service VIP (routes to any ready pod; survivors forward
|
||||
# writes to the elected leader, so the gap is election-time + one RTT, not
|
||||
# election-time + readiness-probe-period).
|
||||
#
|
||||
# Run: kubectl apply -f tidal-stress/k8s/stress-job-t3.yaml
|
||||
# Watch: kubectl logs -f job/tidal-stress-t3 -n tidaldb-cluster
|
||||
# Kill: kubectl delete pod tidaldb-<N> -n tidaldb-cluster --grace-period=0
|
||||
# Rearm: kubectl delete job tidal-stress-t3 -n tidaldb-cluster
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: tidal-stress-t3
|
||||
namespace: tidaldb-cluster
|
||||
labels:
|
||||
app.kubernetes.io/name: tidal-stress
|
||||
app.kubernetes.io/part-of: tidaldb
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
ttlSecondsAfterFinished: 7200
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: tidal-stress
|
||||
app.kubernetes.io/part-of: tidaldb
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: stress
|
||||
image: registry.threesix.ai/tidal/stress@sha256:3a75c311e53f8eb7caf6441bfb40bc9c0611ab739ee33909ca700c7d5c29db16 # :m11p3
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- --target
|
||||
- http://tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
|
||||
- --target
|
||||
- http://tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
|
||||
- --target
|
||||
- http://tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
|
||||
- --leader-url
|
||||
- http://tidaldb.tidaldb-cluster.svc.cluster.local:9500 # service VIP — routes to any ready pod
|
||||
- --ack
|
||||
- quorum
|
||||
- --ramp
|
||||
- 1500:900 # 1500 rps sustained for 15 min (10 kills)
|
||||
- --mix
|
||||
- writes
|
||||
- --write-path
|
||||
- leader
|
||||
- --corpus
|
||||
- "20000"
|
||||
- --users
|
||||
- "100000"
|
||||
- --poll-status # re-discovers new leader after each election
|
||||
env:
|
||||
- name: TIDAL_API_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: tidaldb-credentials
|
||||
key: TIDAL_API_KEY
|
||||
- name: TIDAL_STRESS_LOG
|
||||
value: warn
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "3"
|
||||
memory: 1Gi
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
110
tidal-stress/scripts/t3-kill-loop-v3.sh
Executable file
110
tidal-stress/scripts/t3-kill-loop-v3.sh
Executable file
@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env bash
|
||||
# T3 kill loop v3 — HTTP-polling via port-forward to VIP (no exec into tidaldb pods).
|
||||
# Measures actual election duration: T_kill → first stable new-leader response.
|
||||
# Run alongside the T3 stress job (see ../k8s/stress-job-t3.yaml).
|
||||
set -euo pipefail
|
||||
|
||||
NS=tidaldb-cluster
|
||||
KILLS=10
|
||||
RECOVERY_WAIT=30 # seconds between kill and next iteration (after pod Ready)
|
||||
|
||||
# Start port-forward to the client VIP service in background.
|
||||
kubectl port-forward -n "$NS" svc/tidaldb 19500:9500 &>/tmp/t3-pf.log &
|
||||
PF_PID=$!
|
||||
trap 'kill "$PF_PID" 2>/dev/null || true' EXIT
|
||||
sleep 2 # allow port-forward to establish
|
||||
|
||||
get_leader() {
|
||||
# Returns empty string on any failure (connection refused, timeout, non-200).
|
||||
curl -sf --max-time 2 "http://localhost:19500/cluster/status" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import sys, json
|
||||
try:
|
||||
s = json.load(sys.stdin)
|
||||
ldr = s.get('leader', '') or ''
|
||||
# Only return if it looks like a valid pod name.
|
||||
if ldr.startswith('tidaldb-') and ldr.split('-')[-1].isdigit():
|
||||
print(ldr)
|
||||
except Exception:
|
||||
pass
|
||||
" 2>/dev/null || true
|
||||
}
|
||||
|
||||
echo "T3 kill loop v3 — $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "Gate: leader election < 10 000ms for all $KILLS kills"
|
||||
echo ""
|
||||
|
||||
RESULTS=()
|
||||
|
||||
for i in $(seq 1 "$KILLS"); do
|
||||
echo "=== Kill $i / $KILLS at $(date -u +%H:%M:%SZ) ==="
|
||||
|
||||
# Wait up to 30s for a stable leader.
|
||||
leader=""
|
||||
for _ in $(seq 1 60); do
|
||||
candidate=$(get_leader)
|
||||
if [[ -n "$candidate" ]]; then
|
||||
leader="$candidate"
|
||||
break
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
if [[ -z "$leader" ]]; then
|
||||
echo " SKIP: no stable leader after 30s"
|
||||
RESULTS+=("SKIP")
|
||||
continue
|
||||
fi
|
||||
echo " Leader: $leader"
|
||||
|
||||
# Kill with SIGKILL (no graceful drain).
|
||||
T_KILL_NS=$(python3 -c "import time; print(int(time.time_ns()))")
|
||||
kubectl delete pod "$leader" -n "$NS" --grace-period=0 --force 2>&1 || true
|
||||
|
||||
# Poll at 200ms for a new, different leader.
|
||||
new_leader=""
|
||||
for _ in $(seq 1 150); do # 150 × 200ms = 30s max
|
||||
sleep 0.2
|
||||
candidate=$(get_leader)
|
||||
if [[ -n "$candidate" ]] && [[ "$candidate" != "$leader" ]]; then
|
||||
new_leader="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
T_ELECT_NS=$(python3 -c "import time; print(int(time.time_ns()))")
|
||||
|
||||
if [[ -n "$new_leader" ]]; then
|
||||
elapsed_ms=$(( (T_ELECT_NS - T_KILL_NS) / 1000000 ))
|
||||
gate="PASS"
|
||||
[[ "$elapsed_ms" -gt 10000 ]] && gate="FAIL"
|
||||
echo " Elected $new_leader in ${elapsed_ms}ms [$gate]"
|
||||
RESULTS+=("${elapsed_ms}ms:$gate")
|
||||
else
|
||||
echo " TIMEOUT: no new leader within 30s [FAIL]"
|
||||
RESULTS+=("TIMEOUT:FAIL")
|
||||
fi
|
||||
|
||||
# Wait for the killed pod to be Ready again.
|
||||
echo " Waiting for $leader to recover..."
|
||||
kubectl wait pod "$leader" -n "$NS" --for=condition=Ready --timeout=120s 2>&1 || \
|
||||
echo " WARNING: $leader did not recover within 120s"
|
||||
|
||||
echo " Waiting ${RECOVERY_WAIT}s before next kill..."
|
||||
sleep "$RECOVERY_WAIT"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== T3 Results ==="
|
||||
PASS=0; FAIL=0
|
||||
for r in "${RESULTS[@]}"; do
|
||||
echo " $r"
|
||||
[[ "$r" == *":PASS" ]] && PASS=$((PASS+1)) || FAIL=$((FAIL+1))
|
||||
done
|
||||
echo ""
|
||||
echo "PASS: $PASS / $KILLS | FAIL/SKIP: $FAIL / $KILLS"
|
||||
if [[ "$PASS" -eq "$KILLS" ]]; then
|
||||
echo "T3 GATE: PASS"
|
||||
else
|
||||
echo "T3 GATE: FAIL"
|
||||
fi
|
||||
Loading…
Reference in New Issue
Block a user