d0a52e4530
WAL segment format: 8-byte TSEG header (magic + version byte + 3 reserved)
prepended to every new segment. Legacy headerless segments (m0-m11p3) read
as implicit v0 — no migration. Unknown magic/version surfaces as
WalError::SegmentFormatUnknown at open time; foreign files are never
repaired or truncated (fixes the silent data-loss path from the p3 rollout
incident where torn-tail repair zeroed a follower's unreadable segments).
Catch-up transport: FAILED_PRECONDITION ("snapshot required") and stream
errors that skip the shard now arm a timer retry (re-arm-on-skip is the
load-bearing liveness fix — without it a skipped pull never re-fires and
the follower stays permanently behind). Single retry pending per shard;
CatchupRunner owns the Arc'd state shared between the retry tasks and the
transport. Test: tidal-net/tests/catchup_retry.rs covers the retry path.
Stress: k8s stress-job-t2a/t2b yaml + ops/stress-test-p3-t2 runbook.
87 lines
2.7 KiB
YAML
87 lines
2.7 KiB
YAML
# T2-B: leader-kill chaos under quorum write load.
|
|
# Sustained 1500 rps (solidly within SLO per T2-A) for 10 min.
|
|
# Kill the leader pod mid-run, promote a survivor, observe recovery.
|
|
#
|
|
# Apply: kubectl apply -f tidal-stress/k8s/stress-job-t2b.yaml
|
|
# Watch: kubectl logs -f job/tidal-stress-t2b -n tidaldb
|
|
# Rearm: kubectl delete job tidal-stress-t2b -n tidaldb
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: tidal-stress-t2b
|
|
namespace: tidaldb
|
|
labels:
|
|
app.kubernetes.io/name: tidal-stress
|
|
app.kubernetes.io/part-of: tidaldb
|
|
spec:
|
|
backoffLimit: 0
|
|
ttlSecondsAfterFinished: 7200
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: tidal-stress
|
|
app.kubernetes.io/part-of: tidaldb
|
|
spec:
|
|
restartPolicy: Never
|
|
automountServiceAccountToken: false
|
|
affinity:
|
|
podAntiAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
- labelSelector:
|
|
matchLabels:
|
|
tidaldb.region: us-east
|
|
topologyKey: kubernetes.io/hostname
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
runAsGroup: 1000
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: stress
|
|
image: registry.threesix.ai/tidal/stress@sha256:3a75c311e53f8eb7caf6441bfb40bc9c0611ab739ee33909ca700c7d5c29db16 # :m11p3
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- --target
|
|
- http://10.43.99.11:9500 # us-east (initial leader)
|
|
- --target
|
|
- http://10.43.99.12:9500 # eu-west
|
|
- --target
|
|
- http://10.43.99.13:9500 # ap-south
|
|
- --leader-url
|
|
- http://10.43.99.11:9500
|
|
- --ack
|
|
- quorum
|
|
- --ramp
|
|
- 1500:600 # 1500 rps sustained for 10 min
|
|
- --mix
|
|
- writes
|
|
- --write-path
|
|
- leader
|
|
- --corpus
|
|
- "20000"
|
|
- --users
|
|
- "100000"
|
|
- --skip-seed # corpus seeded during T2-A
|
|
- --poll-status
|
|
env:
|
|
- name: TIDAL_API_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: tidaldb-credentials
|
|
key: TIDAL_API_KEY
|
|
- name: TIDAL_STRESS_LOG
|
|
value: warn
|
|
resources:
|
|
requests:
|
|
cpu: 250m
|
|
memory: 256Mi
|
|
limits:
|
|
cpu: "3"
|
|
memory: 1Gi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities:
|
|
drop: ["ALL"]
|