# 30-NIGHT SOAK — the calendar half of the M11 GA bar.
#
# GA exit gate (docs/planning/milestone-11/phase-9.md §"Exit gate",
# docs/roadmap-to-cluster.md:115, guarantee-traceability.md G-C):
#   "Nightly suite green 30 consecutive days before GA."
#
# This CronJob IS that nightly. It fires once per night and runs a REAL sustained
# mixed read+write soak against the live 3-node cluster (round-robin across all
# three pods — full placement, every pod serves every shard group), with the
# m11p9 regression gates ARMED so a perf/correctness regression on any night
# makes the Job FAIL (non-zero exit) — that night is NOT green and the 30-night
# streak resets.
#
# PASS CRITERIA PER NIGHT (each must hold for the Job to exit 0):
#   --fail-on-knee     : no stage breached the built-in SLO (error rate >1% OR
#                        feed p99 >150ms — the capacity knee).
#   --max-p99-ms 150   : no stage's worst-op p99 exceeded 150ms (cluster network-
#                        hop SLO; in-process SLA is 50ms, the read-recall G1 is
#                        10ms — 150 is the soak regression tripwire, well above
#                        the measured 9.28ms read p99 / ~16ms cross-shard p99).
#   --max-error-pct 1  : no stage's error rate exceeded 1%.
# The streak passes when 30 CONSECUTIVE nightly Jobs have exited 0 with zero
# under-load pod restarts (the monitor records restarts; see soak-monitor.yaml).
#
# HISTORICAL RATE: 200 rps mixed (peach: feed/search/view/like/skip/item/embed).
# The 2026-06-19 one-shot sweep found 200 rps clean, but the subsequent nightly
# history was only 23 PASS / 32 FAIL and included repeated 4 GiB OOMKills. This
# CronJob is therefore parked (`suspend: true`). Resume it only after a controlled
# 100k/1536-D/200-rps memory profile passes under the corrected resource envelope,
# restart evidence is fail-closed, and the exact immutable images below exist.
# Adding nodes to RF3 full placement does not increase write capacity; the old
# ≥2.5x scaling goal requires Ref-B partitioned placement.
#
# WHY A CRONJOB, NOT ONE LONG JOB: the GA bar is literally "30 CONSECUTIVE DAYS"
# of an independent nightly verdict — a CronJob produces exactly that audit trail
# (30 dated Job objects, each PASS/FAIL), survives node reboots/evictions, and
# each night re-pulls fresh cluster state (a single 30-day Job would mask a
# mid-window regression and die on any one eviction). Each night is ~70 min of
# sustained load (a 1h hold + warm-up), bounded by activeDeadlineSeconds.
#
# Apply: kubectl apply -f tidal-stress/k8s/soak-nightly-cronjob.yaml
# Watch tonight's run: kubectl get jobs -n tidaldb-cluster -l app.kubernetes.io/name=tidal-soak
# Read a night's verdict: kubectl logs -n tidaldb-cluster job/<job-name>
apiVersion: batch/v1
kind: CronJob
metadata:
  name: tidal-soak-nightly
  namespace: tidaldb-cluster
  labels:
    app.kubernetes.io/name: tidal-soak
    app.kubernetes.io/part-of: tidaldb
spec:
  # 02:00 cluster-local nightly. Off-peak; one hour hold finishes well before any
  # morning activity. concurrencyPolicy Forbid: never overlap two soaks (they
  # would contend for the same 3 nodes and mutually depress p99 → false FAIL).
  schedule: "0 2 * * *"
  suspend: true
  concurrencyPolicy: Forbid
  startingDeadlineSeconds: 3600
  successfulJobsHistoryLimit: 30   # keep all 30 nights of PASS verdicts
  failedJobsHistoryLimit: 30       # keep every FAIL for streak-reset forensics
  jobTemplate:
    metadata:
      labels:
        app.kubernetes.io/name: tidal-soak
        app.kubernetes.io/part-of: tidaldb
    spec:
      backoffLimit: 0               # a failed night is a FAIL — do not silently retry
      activeDeadlineSeconds: 5400   # 90 min hard cap (1h hold + warm-up + margin)
      ttlSecondsAfterFinished: 2678400  # keep finished Job pods 31 days (full window)
      template:
        metadata:
          labels:
            app.kubernetes.io/name: tidal-soak
            app.kubernetes.io/part-of: tidaldb
        spec:
          restartPolicy: Never
          automountServiceAccountToken: false
          securityContext:
            runAsNonRoot: true
            runAsUser: 1000
            runAsGroup: 1000
            fsGroup: 1000              # so the soak user can write the RWX result PVC
            seccompProfile:
              type: RuntimeDefault
          containers:
            - name: soak
              image: registry.threesix.ai/tidal/stress:m12-fleet-remediation-20260815-r2@sha256:86bfa95f63ba01ed4afa120e41684cd8694743ec39aca72ef0bd68e5d9ef7f8b
              imagePullPolicy: IfNotPresent
              # entrypoint is tidal-stress; wrap it so we can stamp the result file
              # name with the date and append a ledger line regardless of verdict.
              command: ["/bin/sh", "-c"]
              args:
                - |
                  set -u
                  DATE="$(date -u +%Y-%m-%d)"
                  START_TS="$(date -u +%FT%TZ)"
                  OUT="/results/soak-$DATE.json"
                  OUT_TMP="/results/.soak-$DATE-$$.json"
                  TAB="$(printf '\t')"
                  if [ -f /results/ledger.tsv ] && grep -q "^${DATE}${TAB}" /results/ledger.tsv; then
                    echo "FATAL: ledger already contains $DATE; one calendar night cannot count twice" >&2
                    exit 124
                  fi
                  rm -f "$OUT_TMP"
                  echo "soak $DATE start $START_TS target=cluster image=m12-fleet-remediation-200rps"
                  tidal-stress \
                    --target https://tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 \
                    --target https://tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 \
                    --target https://tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 \
                    --ca-cert /etc/tidaldb/tls/ca.crt \
                    --skip-seed \
                    --corpus 100000 \
                    --embedding-dim 1536 \
                    --mix peach \
                    --users 50000 \
                    --ramp "200:3600" \
                    --json-summary "$OUT_TMP" \
                    --max-p99-ms 150 \
                    --max-error-pct 1 \
                    --fail-on-knee
                  LOAD_RC=$?
                  END_TS="$(date -u +%FT%TZ)"

                  # Extract the sustained stage's headline values for the ledger.
                  # The process exit remains authoritative, but missing/malformed
                  # evidence is independently fatal: an empty summary can never
                  # turn a completed process into a green night.
                  PASSED="$(grep -o '"passed"[: ]*[a-z]*' "$OUT_TMP" 2>/dev/null | head -1 | grep -o '[a-z]*$')"
                  P99="$(grep -o '"overall_p99_ms"[: ]*[0-9.]*' "$OUT_TMP" 2>/dev/null | tail -1 | grep -o '[0-9.]*$')"
                  ERRF="$(grep -o '"error_rate"[: ]*[0-9.]*' "$OUT_TMP" 2>/dev/null | tail -1 | grep -o '[0-9.]*$')"
                  SUMMARY_RC=0
                  case "$PASSED" in true|false) ;; *) SUMMARY_RC=126 ;; esac
                  printf '%s\n' "$P99" | grep -Eq '^[0-9]+([.][0-9]+)?$' || SUMMARY_RC=126
                  printf '%s\n' "$ERRF" | grep -Eq '^[0-9]+([.][0-9]+)?$' || SUMMARY_RC=126
                  if [ "$LOAD_RC" -eq 0 ] && [ "$PASSED" != "true" ]; then
                    SUMMARY_RC=126
                  fi
                  if [ "$SUMMARY_RC" -eq 0 ]; then
                    if ! mv "$OUT_TMP" "$OUT"; then
                      echo "FATAL: failed to publish $OUT atomically" >&2
                      SUMMARY_RC=125
                    fi
                  else
                    echo "FATAL: malformed or inconsistent JSON summary (passed=${PASSED:-?} p99=${P99:-?} err=${ERRF:-?})" >&2
                    mv "$OUT_TMP" "/results/soak-$DATE-invalid-$$.json" 2>/dev/null || true
                  fi

                  RC="$LOAD_RC"
                  if [ "$RC" -eq 0 ] && [ "$SUMMARY_RC" -ne 0 ]; then
                    RC="$SUMMARY_RC"
                  fi
                  SOAK_VERD="PASS"; [ "$RC" -ne 0 ] && SOAK_VERD="FAIL"
                  # Stage the candidate ledger separately. Nothing reads this
                  # path except the one-shot evaluator, so a killed Job cannot
                  # leave a provisional PASS in the durable public ledger.
                  PENDING_LEDGER_NAME=".ledger-$DATE-pending.tsv"
                  PENDING_LEDGER="/results/$PENDING_LEDGER_NAME"
                  if [ -f /results/ledger.tsv ]; then
                    cp /results/ledger.tsv "$PENDING_LEDGER" || {
                      echo "FATAL: failed to stage existing /results/ledger.tsv" >&2
                      exit 125
                    }
                    : > "$PENDING_LEDGER" || {
                      echo "FATAL: failed to create candidate ledger" >&2
                      exit 125
                    }
                  fi
                  if ! printf '%s\t%s\tsoak_rc=%s\tsummary_rc=%s\tgate_rc=pending\tpassed=%s\tp99_ms=%s\terr_frac=%s\timage=m12-fleet-remediation-200rps\tstart_utc=%s\tend_utc=%s\n' \
                    "$DATE" "$SOAK_VERD" "$RC" "$SUMMARY_RC" "${PASSED:-?}" "${P99:-?}" "${ERRF:-?}" "$START_TS" "$END_TS" \
                    >> "$PENDING_LEDGER"; then
                    echo "FATAL: failed to stage candidate ledger" >&2
                    rm -f "$PENDING_LEDGER"
                    exit 125
                  fi
                  echo "soak $DATE load end $END_TS verdict=$SOAK_VERD load_rc=$LOAD_RC summary_rc=$SUMMARY_RC p99=${P99:-?} err=${ERRF:-?}"

                  # Fail closed on the other half of the GA contract. The
                  # monitor samples once per minute; wait one full interval,
                  # then require fresh evidence bracketing both load boundaries
                  # for every expected cluster pod.
                  FINAL_RC="$RC"
                  if ! command -v soak-eval >/dev/null 2>&1; then
                    echo "FATAL: soak-eval missing from stress image" >&2
                    EVAL_RC=127
                  else
                    sleep 70
                    soak-eval \
                      --ledger-file "$PENDING_LEDGER_NAME" \
                      --no-write-streak \
                      --results-dir /results \
                      --target 30 \
                      --require-date "$DATE" \
                      --require-samples-through "$END_TS" \
                      --expected-pods 3
                    EVAL_RC=$?
                  fi
                  FINAL_VERD="$SOAK_VERD"
                  if [ "$EVAL_RC" -ne 0 ]; then
                    echo "soak $DATE: recovery-evidence gate failed rc=$EVAL_RC (see this Job log and /results/restarts.tsv)" >&2
                    FINAL_RC="$EVAL_RC"
                    FINAL_VERD="FAIL"
                  fi

                  # Convert the private candidate's last row into the combined
                  # load + recovery verdict. CronJob concurrency is Forbid, so
                  # this is the only writer; rename publishes one atomic ledger.
                  LEDGER_TMP="/results/.ledger-$DATE.tmp"
                  if ! {
                    sed '$d' "$PENDING_LEDGER"
                    printf '%s\t%s\tsoak_rc=%s\tsummary_rc=%s\tgate_rc=%s\tpassed=%s\tp99_ms=%s\terr_frac=%s\timage=m12-fleet-remediation-200rps\tstart_utc=%s\tend_utc=%s\n' \
                      "$DATE" "$FINAL_VERD" "$RC" "$SUMMARY_RC" "$EVAL_RC" "${PASSED:-?}" "${P99:-?}" "${ERRF:-?}" "$START_TS" "$END_TS"
                  } > "$LEDGER_TMP" || ! mv "$LEDGER_TMP" /results/ledger.tsv; then
                    echo "FATAL: failed to finalize /results/ledger.tsv" >&2
                    rm -f "$LEDGER_TMP" "$PENDING_LEDGER"
                    exit 125
                  fi
                  rm -f "$PENDING_LEDGER"

                  # Recompute streak.tsv from the finalized ledger. A non-zero
                  # result is expected on a failed night and must not mask FINAL_RC.
                  if command -v soak-eval >/dev/null 2>&1; then
                    soak-eval --results-dir /results --target 30 >/dev/null 2>&1 || true
                  fi
                  echo "soak $DATE final verdict=$FINAL_VERD load_rc=$LOAD_RC summary_rc=$SUMMARY_RC gate_rc=$EVAL_RC"
                  exit "$FINAL_RC"
              env:
                - name: TIDAL_API_KEY
                  valueFrom:
                    secretKeyRef:
                      name: tidaldb-credentials
                      key: TIDAL_API_KEY
                - name: TIDAL_STRESS_LOG
                  value: warn
              resources:
                requests:
                  cpu: 500m
                  memory: 512Mi
                limits:
                  cpu: "2"
                  memory: 2Gi
              securityContext:
                allowPrivilegeEscalation: false
                readOnlyRootFilesystem: true
                capabilities:
                  drop: ["ALL"]
              volumeMounts:
                - name: cluster-tls
                  mountPath: /etc/tidaldb/tls
                  readOnly: true
                - name: results
                  mountPath: /results
          volumes:
            - name: cluster-tls
              secret:
                secretName: tidaldb-cluster-tls
                items:
                  - key: ca.crt
                    path: ca.crt
            - name: results
              persistentVolumeClaim:
                claimName: tidal-soak-results