# Soak monitor — the always-on in-cluster observer for the 30-night soak. # # Two jobs, both writing to the SAME durable RWX result PVC the nightly CronJob # writes its JSON summaries to, so EVERYTHING the operator needs to judge the # 30-night streak lives in one place and survives the laptop session ending: # # 1. Restart watch: every 5 min, snapshot the tidaldb-{0,1,2} pod restart # counts + phase into /results/restarts.tsv. The GA bar is not just # "30 green soak verdicts" — it is ZERO unrecovered failures over the # window. An under-load pod restart during a soak night must be visible # even if the soak Job itself still passed, so we record it independently. # # 2. HTTP read surface: serve /results over HTTP on :8080 so the operator can # `kubectl port-forward deploy/tidal-soak-monitor 8080:8080 -n tidaldb-cluster` # and read the ledger / nightly summaries / restart log from a browser at # any time, from any machine, without exec'ing into a pod. # # The monitor does NOT generate load and does NOT gate anything — it is a passive # recorder. The pass/fail signal is the CronJob's Job exit codes; this just makes # the 30-night picture observable and durable. # # Apply: kubectl apply -f tidal-stress/k8s/soak-monitor.yaml # Read: kubectl port-forward deploy/tidal-soak-monitor 8080:8080 -n tidaldb-cluster # then open http://localhost:8080/ledger.tsv (and /restarts.tsv, /) --- apiVersion: v1 kind: ServiceAccount metadata: name: tidal-soak-monitor namespace: tidaldb-cluster labels: app.kubernetes.io/name: tidal-soak app.kubernetes.io/part-of: tidaldb automountServiceAccountToken: true --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: tidal-soak-monitor namespace: tidaldb-cluster labels: app.kubernetes.io/name: tidal-soak app.kubernetes.io/part-of: tidaldb rules: # Read-only: pod restart counts + phase, and the nightly soak Job verdicts. - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "watch"] - apiGroups: ["batch"] resources: ["jobs"] verbs: ["get", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: tidal-soak-monitor namespace: tidaldb-cluster labels: app.kubernetes.io/name: tidal-soak app.kubernetes.io/part-of: tidaldb subjects: - kind: ServiceAccount name: tidal-soak-monitor namespace: tidaldb-cluster roleRef: kind: Role name: tidal-soak-monitor apiGroup: rbac.authorization.k8s.io --- apiVersion: apps/v1 kind: Deployment metadata: name: tidal-soak-monitor namespace: tidaldb-cluster labels: app.kubernetes.io/name: tidal-soak app.kubernetes.io/part-of: tidaldb spec: replicas: 1 selector: matchLabels: app.kubernetes.io/name: tidal-soak-monitor template: metadata: labels: app.kubernetes.io/name: tidal-soak-monitor app.kubernetes.io/part-of: tidaldb spec: serviceAccountName: tidal-soak-monitor securityContext: runAsNonRoot: true runAsUser: 1001 runAsGroup: 1001 fsGroup: 1001 seccompProfile: type: RuntimeDefault containers: # ── Restart-watch sidecar: kubectl snapshot loop ────────────────────── - name: restart-watch image: bitnami/kubectl:latest imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: - | echo "restart-watch up $(date -u +%FT%TZ)" # Header once (only if the file is new/empty). if [ ! -s /results/restarts.tsv ]; then printf 'ts_utc\tpod\trestarts\tphase\tready\n' >> /results/restarts.tsv fi while true; do TS="$(date -u +%FT%TZ)" kubectl get pods -n tidaldb-cluster \ -l app.kubernetes.io/name=tidaldb \ -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.containerStatuses[0].restartCount}{"\t"}{.status.phase}{"\t"}{.status.containerStatuses[0].ready}{"\n"}{end}' 2>/dev/null \ | while IFS="$(printf '\t')" read -r POD RC PH RD; do [ -n "$POD" ] && printf '%s\t%s\t%s\t%s\t%s\n' "$TS" "$POD" "$RC" "$PH" "$RD" >> /results/restarts.tsv done sleep 300 done resources: requests: { cpu: 10m, memory: 32Mi } limits: { cpu: 100m, memory: 128Mi } securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true capabilities: { drop: ["ALL"] } volumeMounts: - name: results mountPath: /results # ── Streak evaluator: the GA-bar verdict + alerting ─────────────────── # Joins ledger.tsv (the nightly Job verdicts) with restarts.tsv (the # cumulative per-pod restart counts the sidecar above records) into the # HONEST streak: a night is green iff the soak passed AND no pod restarted # under load. Runs the proven `soak-eval` core (table-driven-tested in # tidal_stress::soak_eval), writes streak.tsv, and emits EXACTLY ONE alert # per non-green night transition (a durable ALERT-.txt the http # surface serves, plus an optional webhook POST when NOTIFY_WEBHOOK is set # — reuse the cluster's notify service by pointing that at it). It does NOT # gate load; it makes the 30-night picture trustworthy and observable. - name: evaluator image: registry.threesix.ai/tidal/stress:m12-soak-eval imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: - | echo "streak-evaluator up $(date -u +%FT%TZ); target=30 nights" while true; do # soak-eval reads /results/{ledger,restarts}.tsv, writes streak.tsv, # and exits non-zero IFF the most recent night is non-green. if soak-eval --results-dir /results --target 30; then : # last night green (or no nights yet) — nothing to alert else # Non-green last night. Alert ONCE: key on the last streak.tsv # date row so a persistent break does not re-fire every loop. LAST="$(grep -v '^date' /results/streak.tsv 2>/dev/null | grep -v '^#' | tail -1 | cut -f1)" SENT="$(cat /results/.last-alert 2>/dev/null || true)" if [ -n "$LAST" ] && [ "$LAST" != "$SENT" ]; then REASON="$(grep -v '^date' /results/streak.tsv | grep -v '^#' | tail -1 | cut -f4)" MSG="tidalDB soak: night $LAST NON-GREEN — $REASON (streak reset)" echo "ALERT: $MSG" printf '%s\t%s\n' "$LAST" "$REASON" > "/results/ALERT-$LAST.txt" [ -n "${NOTIFY_WEBHOOK:-}" ] && \ wget -q -O- --post-data="{\"text\":\"$MSG\"}" --header='Content-Type: application/json' "$NOTIFY_WEBHOOK" >/dev/null 2>&1 || true echo "$LAST" > /results/.last-alert fi fi # Bound the Retain volume: drop nightly summaries older than 31 # days (the full window + 1) so successive 30-night runs never grow # the PVC without limit. ledger/restarts/streak are append/rewrite # and stay small. find /results -maxdepth 1 -name 'soak-*.json' -mtime +31 -delete 2>/dev/null || true sleep 300 done env: # Point at the cluster's notify service to deliver alerts off-cluster; # unset = log + durable ALERT-.txt only (still status-surfaced). - name: NOTIFY_WEBHOOK value: "" resources: requests: { cpu: 10m, memory: 32Mi } limits: { cpu: 200m, memory: 128Mi } securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true capabilities: { drop: ["ALL"] } volumeMounts: - name: results mountPath: /results # ── HTTP read surface: serve the durable result dir ─────────────────── - name: http image: busybox:1.36 imagePullPolicy: IfNotPresent # busybox httpd: one-shot static file server rooted at /results. command: ["/bin/sh", "-c"] args: - | echo "http surface up $(date -u +%FT%TZ) on :8080 serving /results" exec httpd -f -p 8080 -h /results ports: - name: http containerPort: 8080 # Readiness gates the Service endpoint on the http surface actually # serving — a monitor whose node is rebooting must drop out of endpoints # rather than silently serve nothing while a restart it should be # recording slips by unobserved. readinessProbe: httpGet: { path: /, port: 8080 } initialDelaySeconds: 5 periodSeconds: 10 failureThreshold: 3 resources: requests: { cpu: 10m, memory: 16Mi } limits: { cpu: 100m, memory: 64Mi } securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true capabilities: { drop: ["ALL"] } volumeMounts: - name: results mountPath: /results readOnly: true volumes: - name: results persistentVolumeClaim: claimName: tidal-soak-results