tidaldb/tidal-stress/k8s/soak-nightly-cronjob.yaml

# 30-NIGHT SOAK — the calendar half of the M11 GA bar.
#
# GA exit gate (docs/planning/milestone-11/phase-9.md §"Exit gate",
# docs/roadmap-to-cluster.md:115, guarantee-traceability.md G-C):
#   "Nightly suite green 30 consecutive days before GA."
#
# This CronJob IS that nightly. It fires once per night and runs a REAL sustained
# mixed read+write soak against the live 3-node cluster (round-robin across all
# three pods — full placement, every pod serves every shard group), with the
# m11p9 regression gates ARMED so a perf/correctness regression on any night
# makes the Job FAIL (non-zero exit) — that night is NOT green and the 30-night
# streak resets.
#
# PASS CRITERIA PER NIGHT (each must hold for the Job to exit 0):
#   --fail-on-knee     : no stage breached the built-in SLO (error rate >1% OR
#                        feed p99 >150ms — the capacity knee).
#   --max-p99-ms 150   : no stage's worst-op p99 exceeded 150ms (cluster network-
#                        hop SLO; in-process SLA is 50ms, the read-recall G1 is
#                        10ms — 150 is the soak regression tripwire, well above
#                        the measured 9.28ms read p99 / ~16ms cross-shard p99).
#   --max-error-pct 1  : no stage's error rate exceeded 1%.
# The streak passes when 30 CONSECUTIVE nightly Jobs have exited 0 with zero
# under-load pod restarts (the monitor records restarts; see soak-monitor.yaml).
#
# SUSTAINED RATE: 500 rps mixed (peach: feed/search/view/like/skip/item/embed) —
# deliberately BELOW the SLA knee. The rc12 read-SLA gate sustained 500 rps at
# p99 9.28ms / recall 0.9989 / 0% err / 0 under-load restarts, so 500 rps is a
# safe endurance rate that exercises BOTH read and write paths without parking
# the cluster at its ceiling (a soak proves stability over time, not peak — the
# 1-hour 100k-DAU peak run is the separate `--ramp 3900:3600` throughput gate).
#
# WHY A CRONJOB, NOT ONE LONG JOB: the GA bar is literally "30 CONSECUTIVE DAYS"
# of an independent nightly verdict — a CronJob produces exactly that audit trail
# (30 dated Job objects, each PASS/FAIL), survives node reboots/evictions, and
# each night re-pulls fresh cluster state (a single 30-day Job would mask a
# mid-window regression and die on any one eviction). Each night is ~70 min of
# sustained load (a 1h hold + warm-up), bounded by activeDeadlineSeconds.
#
# Apply: kubectl apply -f tidal-stress/k8s/soak-nightly-cronjob.yaml
# Watch tonight's run: kubectl get jobs -n tidaldb-cluster -l app.kubernetes.io/name=tidal-soak
# Read a night's verdict: kubectl logs -n tidaldb-cluster job/<job-name>
apiVersion: batch/v1
kind: CronJob
metadata:
  name: tidal-soak-nightly
  namespace: tidaldb-cluster
  labels:
    app.kubernetes.io/name: tidal-soak
    app.kubernetes.io/part-of: tidaldb
spec:
  # 02:00 cluster-local nightly. Off-peak; one hour hold finishes well before any
  # morning activity. concurrencyPolicy Forbid: never overlap two soaks (they
  # would contend for the same 3 nodes and mutually depress p99 → false FAIL).
  schedule: "0 2 * * *"
  concurrencyPolicy: Forbid
  startingDeadlineSeconds: 3600
  successfulJobsHistoryLimit: 30   # keep all 30 nights of PASS verdicts
  failedJobsHistoryLimit: 30       # keep every FAIL for streak-reset forensics
  jobTemplate:
    metadata:
      labels:
        app.kubernetes.io/name: tidal-soak
        app.kubernetes.io/part-of: tidaldb
    spec:
      backoffLimit: 0               # a failed night is a FAIL — do not silently retry
      activeDeadlineSeconds: 5400   # 90 min hard cap (1h hold + warm-up + margin)
      ttlSecondsAfterFinished: 2678400  # keep finished Job pods 31 days (full window)
      template:
        metadata:
          labels:
            app.kubernetes.io/name: tidal-soak
            app.kubernetes.io/part-of: tidaldb
        spec:
          restartPolicy: Never
          automountServiceAccountToken: false
          securityContext:
            runAsNonRoot: true
            runAsUser: 1000
            runAsGroup: 1000
            fsGroup: 1000              # so the soak user can write the RWX result PVC
            seccompProfile:
              type: RuntimeDefault
          containers:
            - name: soak
              image: registry.threesix.ai/tidal/stress:m12-rc7-seedretry
              imagePullPolicy: IfNotPresent
              # entrypoint is tidal-stress; wrap it so we can stamp the result file
              # name with the date and append a ledger line regardless of verdict.
              command: ["/bin/sh", "-c"]
              args:
                - |
                  set -u
                  DATE="$(date -u +%Y-%m-%d)"
                  OUT="/results/soak-$DATE.json"
                  echo "soak $DATE start $(date -u +%H:%M:%SZ) target=cluster image=m12-rc7-seedretry"
                  tidal-stress \
                    --target https://tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 \
                    --target https://tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 \
                    --target https://tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 \
                    --ca-cert /etc/tidaldb/tls/ca.crt \
                    --skip-seed \
                    --corpus 100000 \
                    --embedding-dim 1536 \
                    --mix peach \
                    --users 50000 \
                    --ramp "500:3600" \
                    --json-summary "$OUT" \
                    --max-p99-ms 150 \
                    --max-error-pct 1 \
                    --fail-on-knee
                  RC=$?
                  # Extract the verdict + headline p99/error from the JSON summary
                  # for a one-line ledger entry (grep, no jq dependency in the image).
                  # Fields are the real summary.rs keys: top-level "passed" (bool),
                  # per-stage "overall_p99_ms" (ms) and "error_rate" (FRACTION 0-1).
                  # Worst stage = max p99 over the run; we report the LAST stage's
                  # numbers (the sustained-rate stage, since the soak ramp is single-
                  # stage 500:3600 anyway) and trust "passed" for the verdict.
                  VERD="PASS"; [ "$RC" -ne 0 ] && VERD="FAIL"
                  PASSED="$(grep -o '"passed"[: ]*[a-z]*' "$OUT" 2>/dev/null | head -1 | grep -o '[a-z]*$')"
                  P99="$(grep -o '"overall_p99_ms"[: ]*[0-9.]*' "$OUT" 2>/dev/null | tail -1 | grep -o '[0-9.]*$')"
                  ERRF="$(grep -o '"error_rate"[: ]*[0-9.]*' "$OUT" 2>/dev/null | tail -1 | grep -o '[0-9.]*$')"
                  printf '%s\t%s\trc=%s\tpassed=%s\tp99_ms=%s\terr_frac=%s\timage=m12-rc7-seedretry\n' \
                    "$DATE" "$VERD" "$RC" "${PASSED:-?}" "${P99:-?}" "${ERRF:-?}" >> /results/ledger.tsv
                  echo "soak $DATE end $(date -u +%H:%M:%SZ) verdict=$VERD rc=$RC p99=${P99:-?} err=${ERR:-?}"
                  exit "$RC"                  
              env:
                - name: TIDAL_API_KEY
                  valueFrom:
                    secretKeyRef:
                      name: tidaldb-credentials
                      key: TIDAL_API_KEY
                - name: TIDAL_STRESS_LOG
                  value: warn
              resources:
                requests:
                  cpu: 500m
                  memory: 512Mi
                limits:
                  cpu: "2"
                  memory: 2Gi
              securityContext:
                allowPrivilegeEscalation: false
                readOnlyRootFilesystem: true
                capabilities:
                  drop: ["ALL"]
              volumeMounts:
                - name: cluster-tls
                  mountPath: /etc/tidaldb/tls
                  readOnly: true
                - name: results
                  mountPath: /results
          volumes:
            - name: cluster-tls
              secret:
                secretName: tidaldb-cluster-tls
                items:
                  - key: ca.crt
                    path: ca.crt
            - name: results
              persistentVolumeClaim:
                claimName: tidal-soak-results