tidaldb/tidal-stress/k8s/soak-monitor.yaml

# Soak monitor — the always-on in-cluster observer for the 30-night soak.
#
# Two jobs, both writing to the SAME durable RWX result PVC the nightly CronJob
# writes its JSON summaries to, so EVERYTHING the operator needs to judge the
# 30-night streak lives in one place and survives the laptop session ending:
#
#   1. Restart watch: every minute, snapshot the tidaldb-{0,1,2} pod identity,
#      counts + phase into /results/restarts.tsv. The GA bar is not just
#      "30 green soak verdicts" — it is ZERO unrecovered failures over the
#      window. An under-load pod restart during a soak night must be visible
#      even if the soak Job itself still passed, so we record it independently.
#
#   2. HTTP read surface: serve /results over HTTP on :8080 so the operator can
#      `kubectl port-forward deploy/tidal-soak-monitor 8080:8080 -n tidaldb-cluster`
#      and read the ledger / nightly summaries / restart log from a browser at
#      any time, from any machine, without exec'ing into a pod.
#
# The monitor does not generate load. The nightly Job is the authoritative gate;
# this deployment maintains the evidence stream and continuously recomputes the
# durable streak/status surface.
#
# Apply: kubectl apply -f tidal-stress/k8s/soak-monitor.yaml
# Read:  kubectl port-forward deploy/tidal-soak-monitor 8080:8080 -n tidaldb-cluster
#        then open http://localhost:8080/ledger.tsv  (and /restarts.tsv, /)
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: tidal-soak-monitor
  namespace: tidaldb-cluster
  labels:
    app.kubernetes.io/name: tidal-soak
    app.kubernetes.io/part-of: tidaldb
automountServiceAccountToken: true
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: tidal-soak-monitor
  namespace: tidaldb-cluster
  labels:
    app.kubernetes.io/name: tidal-soak
    app.kubernetes.io/part-of: tidaldb
rules:
  # Read-only: pod restart counts + phase, and the nightly soak Job verdicts.
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["batch"]
    resources: ["jobs"]
    verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: tidal-soak-monitor
  namespace: tidaldb-cluster
  labels:
    app.kubernetes.io/name: tidal-soak
    app.kubernetes.io/part-of: tidaldb
subjects:
  - kind: ServiceAccount
    name: tidal-soak-monitor
    namespace: tidaldb-cluster
roleRef:
  kind: Role
  name: tidal-soak-monitor
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tidal-soak-monitor
  namespace: tidaldb-cluster
  labels:
    app.kubernetes.io/name: tidal-soak
    app.kubernetes.io/part-of: tidaldb
spec:
  replicas: 0  # parked; scripts/restore-fleet.sh starts it only after capacity checks
  selector:
    matchLabels:
      app.kubernetes.io/name: tidal-soak-monitor
  template:
    metadata:
      labels:
        app.kubernetes.io/name: tidal-soak-monitor
        app.kubernetes.io/part-of: tidaldb
    spec:
      serviceAccountName: tidal-soak-monitor
      securityContext:
        runAsNonRoot: true
        runAsUser: 1001
        runAsGroup: 1001
        fsGroup: 1001
        seccompProfile:
          type: RuntimeDefault
      containers:
        # ── Restart watcher: direct Kubernetes API sampler ───────────────────
        - name: restart-watch
          image: registry.threesix.ai/tidal/stress:m12-fleet-remediation-20260815-r2@sha256:86bfa95f63ba01ed4afa120e41684cd8694743ec39aca72ef0bd68e5d9ef7f8b
          imagePullPolicy: IfNotPresent
          command: ["soak-watch"]
          args:
            - --namespace
            - tidaldb-cluster
            - --label-selector
            - app.kubernetes.io/name=tidaldb
            - --container
            - tidaldb
            - --results-file
            - /results/restarts.tsv
            - --interval-seconds
            - "60"
          resources:
            requests: { cpu: 10m, memory: 32Mi }
            limits:   { cpu: 100m, memory: 128Mi }
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities: { drop: ["ALL"] }
          volumeMounts:
            - name: results
              mountPath: /results
        # ── Streak evaluator: durable GA-bar status ────────────────────────────
        # Joins the nightly verdict ledger with the restart evidence above.
        # The nightly Job fails closed on this same evaluator; this sidecar keeps
        # streak.tsv and one durable ALERT-<date>.txt current for operators.
        - name: evaluator
          image: registry.threesix.ai/tidal/stress:m12-fleet-remediation-20260815-r2@sha256:86bfa95f63ba01ed4afa120e41684cd8694743ec39aca72ef0bd68e5d9ef7f8b
          imagePullPolicy: IfNotPresent
          command: ["/bin/sh", "-c"]
          args:
            - |
              echo "streak-evaluator up $(date -u +%FT%TZ); target=30 nights"
              while true; do
                # soak-eval reads /results/{ledger,restarts}.tsv, writes streak.tsv,
                # and exits non-zero IFF the most recent night is non-green.
                if soak-eval --results-dir /results --target 30; then
                  : # last night green (or no nights yet) — nothing to alert
                else
                  # Non-green last night. Alert ONCE: key on the last streak.tsv
                  # date row so a persistent break does not re-fire every loop.
                  LAST="$(grep -v '^date' /results/streak.tsv 2>/dev/null | grep -v '^#' | tail -1 | cut -f1)"
                  SENT="$(cat /results/.last-alert 2>/dev/null || true)"
                  if [ -n "$LAST" ] && [ "$LAST" != "$SENT" ]; then
                    REASON="$(grep -v '^date' /results/streak.tsv | grep -v '^#' | tail -1 | cut -f4)"
                    MSG="tidalDB soak: night $LAST NON-GREEN — $REASON (streak reset)"
                    echo "ALERT: $MSG"
                    printf '%s\t%s\n' "$LAST" "$REASON" > "/results/ALERT-$LAST.txt"
                    echo "$LAST" > /results/.last-alert
                  fi
                fi
                # Bound the Retain volume: drop nightly summaries older than 31
                # days (the full window + 1) so successive 30-night runs never grow
                # the PVC without limit. ledger/restarts/streak are append/rewrite
                # and stay small.
                find /results -maxdepth 1 -name 'soak-*.json' -mtime +31 -delete 2>/dev/null || true
                sleep 300
              done              
          resources:
            requests: { cpu: 10m, memory: 32Mi }
            limits:   { cpu: 200m, memory: 128Mi }
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities: { drop: ["ALL"] }
          volumeMounts:
            - name: results
              mountPath: /results
        # ── HTTP read surface: serve the durable result dir ───────────────────
        - name: http
          image: busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662
          imagePullPolicy: IfNotPresent
          # busybox httpd: one-shot static file server rooted at /results.
          command: ["/bin/sh", "-c"]
          args:
            - |
              echo "http surface up $(date -u +%FT%TZ) on :8080 serving /results"
              exec httpd -f -p 8080 -h /results              
          ports:
            - name: http
              containerPort: 8080
          # Readiness gates the Service endpoint on the HTTP listener. The
          # result directory intentionally has no index file, so GET / returns
          # 404 even while the static server is healthy.
          readinessProbe:
            tcpSocket: { port: http }
            initialDelaySeconds: 5
            periodSeconds: 10
            failureThreshold: 3
          resources:
            requests: { cpu: 10m, memory: 16Mi }
            limits:   { cpu: 100m, memory: 64Mi }
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities: { drop: ["ALL"] }
          volumeMounts:
            - name: results
              mountPath: /results
              readOnly: true
      volumes:
        - name: results
          persistentVolumeClaim:
            claimName: tidal-soak-results