25ec7630a1
statefulset.yaml: - Pin image to m11p5 digest (173e803...) - Add initContainer init-datadir (busybox, uid 10001) to create /data/db before main - Set storageClassName: local-path (5Gi) — Longhorn networked-fsync caused 74-85% error rate T3 tooling: - tidal-stress/k8s/stress-job-t3.yaml: 1500 rps quorum-write job for election gate - tidal-stress/scripts/t3-kill-loop-v3.sh: HTTP-polling kill loop (port-forward + curl, 200ms poll, ns-precision timing, no exec into tidaldb pods) T3 result: 10/10 kills PASS, max 6157ms (gate < 10 000ms), zero acked loss.
86 lines
3.1 KiB
YAML
86 lines
3.1 KiB
YAML
# T3: automatic leader-failover gate (m11p4/p5 election under quorum write load).
|
|
# Gate: kill the leader pod → election completes automatically, writes resume
|
|
# with < 10s p99 gap, zero operator action, zero acked loss; 10 kill trials.
|
|
#
|
|
# Cluster: tidaldb-cluster ns — one StatefulSet, pod-DNS peer addressing.
|
|
# Targets: all 3 pod DNS names (so status-poll reaches survivors during kill window).
|
|
# leader-url: the client Service VIP (routes to any ready pod; survivors forward
|
|
# writes to the elected leader, so the gap is election-time + one RTT, not
|
|
# election-time + readiness-probe-period).
|
|
#
|
|
# Run: kubectl apply -f tidal-stress/k8s/stress-job-t3.yaml
|
|
# Watch: kubectl logs -f job/tidal-stress-t3 -n tidaldb-cluster
|
|
# Kill: kubectl delete pod tidaldb-<N> -n tidaldb-cluster --grace-period=0
|
|
# Rearm: kubectl delete job tidal-stress-t3 -n tidaldb-cluster
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: tidal-stress-t3
|
|
namespace: tidaldb-cluster
|
|
labels:
|
|
app.kubernetes.io/name: tidal-stress
|
|
app.kubernetes.io/part-of: tidaldb
|
|
spec:
|
|
backoffLimit: 0
|
|
ttlSecondsAfterFinished: 7200
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: tidal-stress
|
|
app.kubernetes.io/part-of: tidaldb
|
|
spec:
|
|
restartPolicy: Never
|
|
automountServiceAccountToken: false
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
runAsGroup: 1000
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: stress
|
|
image: registry.threesix.ai/tidal/stress@sha256:3a75c311e53f8eb7caf6441bfb40bc9c0611ab739ee33909ca700c7d5c29db16 # :m11p3
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- --target
|
|
- http://tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
|
|
- --target
|
|
- http://tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
|
|
- --target
|
|
- http://tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
|
|
- --leader-url
|
|
- http://tidaldb.tidaldb-cluster.svc.cluster.local:9500 # service VIP — routes to any ready pod
|
|
- --ack
|
|
- quorum
|
|
- --ramp
|
|
- 1500:900 # 1500 rps sustained for 15 min (10 kills)
|
|
- --mix
|
|
- writes
|
|
- --write-path
|
|
- leader
|
|
- --corpus
|
|
- "20000"
|
|
- --users
|
|
- "100000"
|
|
- --poll-status # re-discovers new leader after each election
|
|
env:
|
|
- name: TIDAL_API_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: tidaldb-credentials
|
|
key: TIDAL_API_KEY
|
|
- name: TIDAL_STRESS_LOG
|
|
value: warn
|
|
resources:
|
|
requests:
|
|
cpu: 250m
|
|
memory: 256Mi
|
|
limits:
|
|
cpu: "3"
|
|
memory: 1Gi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities:
|
|
drop: ["ALL"]
|