580142df49
Durable `leader_acked` frontier in `ShardReplica` tracks the highest seqno acked under `ack=leader` (journal-only, un-replicated); `decide_join` now quarantines on THIS node's own frontier rather than comparing stream numbers across stream boundaries — eliminates false-quarantine churn on rolling restarts. `SHUTDOWN_HANDOFF_WAIT` (3s) drains the leader's tail to quorum before step-down so the next leader inherits a clean prefix. New `load_leader_acked`/`persist_leader_acked` helpers; `cluster_reseed.rs` gains the divergence-fix regression suite; `replication_ops.rs` threads the signal. Soak-eval: `tidal_stress::soak_eval` + `soak-eval` binary implement the 30-night streak (ledger.tsv × restarts.tsv → streak.tsv); monitor and nightly CronJob k8s YAMLs updated; phase-9 doc clarifies the dual-stream streak definition (ledger PASS AND zero pod restarts in window). `run-reliability.sh` gates the election-divergence suite before any k8s push. Release tooling: `docker/release/` multi-stage Dockerfile + DR image; `scripts/build-release.sh` single repeatable cross-compile+buildx path.
226 lines
9.6 KiB
YAML
226 lines
9.6 KiB
YAML
# Soak monitor — the always-on in-cluster observer for the 30-night soak.
|
|
#
|
|
# Two jobs, both writing to the SAME durable RWX result PVC the nightly CronJob
|
|
# writes its JSON summaries to, so EVERYTHING the operator needs to judge the
|
|
# 30-night streak lives in one place and survives the laptop session ending:
|
|
#
|
|
# 1. Restart watch: every 5 min, snapshot the tidaldb-{0,1,2} pod restart
|
|
# counts + phase into /results/restarts.tsv. The GA bar is not just
|
|
# "30 green soak verdicts" — it is ZERO unrecovered failures over the
|
|
# window. An under-load pod restart during a soak night must be visible
|
|
# even if the soak Job itself still passed, so we record it independently.
|
|
#
|
|
# 2. HTTP read surface: serve /results over HTTP on :8080 so the operator can
|
|
# `kubectl port-forward deploy/tidal-soak-monitor 8080:8080 -n tidaldb-cluster`
|
|
# and read the ledger / nightly summaries / restart log from a browser at
|
|
# any time, from any machine, without exec'ing into a pod.
|
|
#
|
|
# The monitor does NOT generate load and does NOT gate anything — it is a passive
|
|
# recorder. The pass/fail signal is the CronJob's Job exit codes; this just makes
|
|
# the 30-night picture observable and durable.
|
|
#
|
|
# Apply: kubectl apply -f tidal-stress/k8s/soak-monitor.yaml
|
|
# Read: kubectl port-forward deploy/tidal-soak-monitor 8080:8080 -n tidaldb-cluster
|
|
# then open http://localhost:8080/ledger.tsv (and /restarts.tsv, /)
|
|
---
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: tidal-soak-monitor
|
|
namespace: tidaldb-cluster
|
|
labels:
|
|
app.kubernetes.io/name: tidal-soak
|
|
app.kubernetes.io/part-of: tidaldb
|
|
automountServiceAccountToken: true
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: Role
|
|
metadata:
|
|
name: tidal-soak-monitor
|
|
namespace: tidaldb-cluster
|
|
labels:
|
|
app.kubernetes.io/name: tidal-soak
|
|
app.kubernetes.io/part-of: tidaldb
|
|
rules:
|
|
# Read-only: pod restart counts + phase, and the nightly soak Job verdicts.
|
|
- apiGroups: [""]
|
|
resources: ["pods"]
|
|
verbs: ["get", "list", "watch"]
|
|
- apiGroups: ["batch"]
|
|
resources: ["jobs"]
|
|
verbs: ["get", "list", "watch"]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: RoleBinding
|
|
metadata:
|
|
name: tidal-soak-monitor
|
|
namespace: tidaldb-cluster
|
|
labels:
|
|
app.kubernetes.io/name: tidal-soak
|
|
app.kubernetes.io/part-of: tidaldb
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: tidal-soak-monitor
|
|
namespace: tidaldb-cluster
|
|
roleRef:
|
|
kind: Role
|
|
name: tidal-soak-monitor
|
|
apiGroup: rbac.authorization.k8s.io
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: tidal-soak-monitor
|
|
namespace: tidaldb-cluster
|
|
labels:
|
|
app.kubernetes.io/name: tidal-soak
|
|
app.kubernetes.io/part-of: tidaldb
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: tidal-soak-monitor
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: tidal-soak-monitor
|
|
app.kubernetes.io/part-of: tidaldb
|
|
spec:
|
|
serviceAccountName: tidal-soak-monitor
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 1001
|
|
runAsGroup: 1001
|
|
fsGroup: 1001
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
# ── Restart-watch sidecar: kubectl snapshot loop ──────────────────────
|
|
- name: restart-watch
|
|
image: bitnami/kubectl:latest
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
echo "restart-watch up $(date -u +%FT%TZ)"
|
|
# Header once (only if the file is new/empty).
|
|
if [ ! -s /results/restarts.tsv ]; then
|
|
printf 'ts_utc\tpod\trestarts\tphase\tready\n' >> /results/restarts.tsv
|
|
fi
|
|
while true; do
|
|
TS="$(date -u +%FT%TZ)"
|
|
kubectl get pods -n tidaldb-cluster \
|
|
-l app.kubernetes.io/name=tidaldb \
|
|
-o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.containerStatuses[0].restartCount}{"\t"}{.status.phase}{"\t"}{.status.containerStatuses[0].ready}{"\n"}{end}' 2>/dev/null \
|
|
| while IFS="$(printf '\t')" read -r POD RC PH RD; do
|
|
[ -n "$POD" ] && printf '%s\t%s\t%s\t%s\t%s\n' "$TS" "$POD" "$RC" "$PH" "$RD" >> /results/restarts.tsv
|
|
done
|
|
sleep 300
|
|
done
|
|
resources:
|
|
requests: { cpu: 10m, memory: 32Mi }
|
|
limits: { cpu: 100m, memory: 128Mi }
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities: { drop: ["ALL"] }
|
|
volumeMounts:
|
|
- name: results
|
|
mountPath: /results
|
|
# ── Streak evaluator: the GA-bar verdict + alerting ───────────────────
|
|
# Joins ledger.tsv (the nightly Job verdicts) with restarts.tsv (the
|
|
# cumulative per-pod restart counts the sidecar above records) into the
|
|
# HONEST streak: a night is green iff the soak passed AND no pod restarted
|
|
# under load. Runs the proven `soak-eval` core (table-driven-tested in
|
|
# tidal_stress::soak_eval), writes streak.tsv, and emits EXACTLY ONE alert
|
|
# per non-green night transition (a durable ALERT-<date>.txt the http
|
|
# surface serves, plus an optional webhook POST when NOTIFY_WEBHOOK is set
|
|
# — reuse the cluster's notify service by pointing that at it). It does NOT
|
|
# gate load; it makes the 30-night picture trustworthy and observable.
|
|
- name: evaluator
|
|
image: registry.threesix.ai/tidal/stress:m12-soak-eval
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
echo "streak-evaluator up $(date -u +%FT%TZ); target=30 nights"
|
|
while true; do
|
|
# soak-eval reads /results/{ledger,restarts}.tsv, writes streak.tsv,
|
|
# and exits non-zero IFF the most recent night is non-green.
|
|
if soak-eval --results-dir /results --target 30; then
|
|
: # last night green (or no nights yet) — nothing to alert
|
|
else
|
|
# Non-green last night. Alert ONCE: key on the last streak.tsv
|
|
# date row so a persistent break does not re-fire every loop.
|
|
LAST="$(grep -v '^date' /results/streak.tsv 2>/dev/null | grep -v '^#' | tail -1 | cut -f1)"
|
|
SENT="$(cat /results/.last-alert 2>/dev/null || true)"
|
|
if [ -n "$LAST" ] && [ "$LAST" != "$SENT" ]; then
|
|
REASON="$(grep -v '^date' /results/streak.tsv | grep -v '^#' | tail -1 | cut -f4)"
|
|
MSG="tidalDB soak: night $LAST NON-GREEN — $REASON (streak reset)"
|
|
echo "ALERT: $MSG"
|
|
printf '%s\t%s\n' "$LAST" "$REASON" > "/results/ALERT-$LAST.txt"
|
|
[ -n "${NOTIFY_WEBHOOK:-}" ] && \
|
|
wget -q -O- --post-data="{\"text\":\"$MSG\"}" --header='Content-Type: application/json' "$NOTIFY_WEBHOOK" >/dev/null 2>&1 || true
|
|
echo "$LAST" > /results/.last-alert
|
|
fi
|
|
fi
|
|
# Bound the Retain volume: drop nightly summaries older than 31
|
|
# days (the full window + 1) so successive 30-night runs never grow
|
|
# the PVC without limit. ledger/restarts/streak are append/rewrite
|
|
# and stay small.
|
|
find /results -maxdepth 1 -name 'soak-*.json' -mtime +31 -delete 2>/dev/null || true
|
|
sleep 300
|
|
done
|
|
env:
|
|
# Point at the cluster's notify service to deliver alerts off-cluster;
|
|
# unset = log + durable ALERT-<date>.txt only (still status-surfaced).
|
|
- name: NOTIFY_WEBHOOK
|
|
value: ""
|
|
resources:
|
|
requests: { cpu: 10m, memory: 32Mi }
|
|
limits: { cpu: 200m, memory: 128Mi }
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities: { drop: ["ALL"] }
|
|
volumeMounts:
|
|
- name: results
|
|
mountPath: /results
|
|
# ── HTTP read surface: serve the durable result dir ───────────────────
|
|
- name: http
|
|
image: busybox:1.36
|
|
imagePullPolicy: IfNotPresent
|
|
# busybox httpd: one-shot static file server rooted at /results.
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
echo "http surface up $(date -u +%FT%TZ) on :8080 serving /results"
|
|
exec httpd -f -p 8080 -h /results
|
|
ports:
|
|
- name: http
|
|
containerPort: 8080
|
|
# Readiness gates the Service endpoint on the http surface actually
|
|
# serving — a monitor whose node is rebooting must drop out of endpoints
|
|
# rather than silently serve nothing while a restart it should be
|
|
# recording slips by unobserved.
|
|
readinessProbe:
|
|
httpGet: { path: /, port: 8080 }
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 10
|
|
failureThreshold: 3
|
|
resources:
|
|
requests: { cpu: 10m, memory: 16Mi }
|
|
limits: { cpu: 100m, memory: 64Mi }
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities: { drop: ["ALL"] }
|
|
volumeMounts:
|
|
- name: results
|
|
mountPath: /results
|
|
readOnly: true
|
|
volumes:
|
|
- name: results
|
|
persistentVolumeClaim:
|
|
claimName: tidal-soak-results
|