tidaldb/tidal-stress/k8s/soak-monitor.yaml

# Soak monitor — the always-on in-cluster observer for the 30-night soak.
#
# Two jobs, both writing to the SAME durable RWX result PVC the nightly CronJob
# writes its JSON summaries to, so EVERYTHING the operator needs to judge the
# 30-night streak lives in one place and survives the laptop session ending:
#
#   1. Restart watch: every 5 min, snapshot the tidaldb-{0,1,2} pod restart
#      counts + phase into /results/restarts.tsv. The GA bar is not just
#      "30 green soak verdicts" — it is ZERO unrecovered failures over the
#      window. An under-load pod restart during a soak night must be visible
#      even if the soak Job itself still passed, so we record it independently.
#
#   2. HTTP read surface: serve /results over HTTP on :8080 so the operator can
#      `kubectl port-forward deploy/tidal-soak-monitor 8080:8080 -n tidaldb-cluster`
#      and read the ledger / nightly summaries / restart log from a browser at
#      any time, from any machine, without exec'ing into a pod.
#
# The monitor does NOT generate load and does NOT gate anything — it is a passive
# recorder. The pass/fail signal is the CronJob's Job exit codes; this just makes
# the 30-night picture observable and durable.
#
# Apply: kubectl apply -f tidal-stress/k8s/soak-monitor.yaml
# Read:  kubectl port-forward deploy/tidal-soak-monitor 8080:8080 -n tidaldb-cluster
#        then open http://localhost:8080/ledger.tsv  (and /restarts.tsv, /)
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: tidal-soak-monitor
  namespace: tidaldb-cluster
  labels:
    app.kubernetes.io/name: tidal-soak
    app.kubernetes.io/part-of: tidaldb
automountServiceAccountToken: true
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: tidal-soak-monitor
  namespace: tidaldb-cluster
  labels:
    app.kubernetes.io/name: tidal-soak
    app.kubernetes.io/part-of: tidaldb
rules:
  # Read-only: pod restart counts + phase, and the nightly soak Job verdicts.
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["batch"]
    resources: ["jobs"]
    verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: tidal-soak-monitor
  namespace: tidaldb-cluster
  labels:
    app.kubernetes.io/name: tidal-soak
    app.kubernetes.io/part-of: tidaldb
subjects:
  - kind: ServiceAccount
    name: tidal-soak-monitor
    namespace: tidaldb-cluster
roleRef:
  kind: Role
  name: tidal-soak-monitor
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tidal-soak-monitor
  namespace: tidaldb-cluster
  labels:
    app.kubernetes.io/name: tidal-soak
    app.kubernetes.io/part-of: tidaldb
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: tidal-soak-monitor
  template:
    metadata:
      labels:
        app.kubernetes.io/name: tidal-soak-monitor
        app.kubernetes.io/part-of: tidaldb
    spec:
      serviceAccountName: tidal-soak-monitor
      securityContext:
        runAsNonRoot: true
        runAsUser: 1001
        runAsGroup: 1001
        fsGroup: 1001
        seccompProfile:
          type: RuntimeDefault
      containers:
        # ── Restart-watch sidecar: kubectl snapshot loop ──────────────────────
        - name: restart-watch
          image: bitnami/kubectl:latest
          imagePullPolicy: IfNotPresent
          command: ["/bin/sh", "-c"]
          args:
            - |
              echo "restart-watch up $(date -u +%FT%TZ)"
              # Header once (only if the file is new/empty).
              if [ ! -s /results/restarts.tsv ]; then
                printf 'ts_utc\tpod\trestarts\tphase\tready\n' >> /results/restarts.tsv
              fi
              while true; do
                TS="$(date -u +%FT%TZ)"
                kubectl get pods -n tidaldb-cluster \
                  -l app.kubernetes.io/name=tidaldb \
                  -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.containerStatuses[0].restartCount}{"\t"}{.status.phase}{"\t"}{.status.containerStatuses[0].ready}{"\n"}{end}' 2>/dev/null \
                  | while IFS="$(printf '\t')" read -r POD RC PH RD; do
                      [ -n "$POD" ] && printf '%s\t%s\t%s\t%s\t%s\n' "$TS" "$POD" "$RC" "$PH" "$RD" >> /results/restarts.tsv
                    done
                sleep 300
              done              
          resources:
            requests: { cpu: 10m, memory: 32Mi }
            limits:   { cpu: 100m, memory: 128Mi }
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities: { drop: ["ALL"] }
          volumeMounts:
            - name: results
              mountPath: /results
        # ── HTTP read surface: serve the durable result dir ───────────────────
        - name: http
          image: busybox:1.36
          imagePullPolicy: IfNotPresent
          # busybox httpd: one-shot static file server rooted at /results.
          command: ["/bin/sh", "-c"]
          args:
            - |
              echo "http surface up $(date -u +%FT%TZ) on :8080 serving /results"
              exec httpd -f -p 8080 -h /results              
          ports:
            - name: http
              containerPort: 8080
          resources:
            requests: { cpu: 10m, memory: 16Mi }
            limits:   { cpu: 100m, memory: 64Mi }
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities: { drop: ["ALL"] }
          volumeMounts:
            - name: results
              mountPath: /results
              readOnly: true
      volumes:
        - name: results
          persistentVolumeClaim:
            claimName: tidal-soak-results