diff --git a/tidal-stress/k8s/soak-nightly-cronjob.yaml b/tidal-stress/k8s/soak-nightly-cronjob.yaml index 88bb8f8..5bc0fa0 100644 --- a/tidal-stress/k8s/soak-nightly-cronjob.yaml +++ b/tidal-stress/k8s/soak-nightly-cronjob.yaml @@ -22,12 +22,17 @@ # The streak passes when 30 CONSECUTIVE nightly Jobs have exited 0 with zero # under-load pod restarts (the monitor records restarts; see soak-monitor.yaml). # -# SUSTAINED RATE: 500 rps mixed (peach: feed/search/view/like/skip/item/embed) — -# deliberately BELOW the SLA knee. The rc12 read-SLA gate sustained 500 rps at -# p99 9.28ms / recall 0.9989 / 0% err / 0 under-load restarts, so 500 rps is a -# safe endurance rate that exercises BOTH read and write paths without parking -# the cluster at its ceiling (a soak proves stability over time, not peak — the -# 1-hour 100k-DAU peak run is the separate `--ramp 3900:3600` throughput gate). +# SUSTAINED RATE: 200 rps mixed (peach: feed/search/view/like/skip/item/embed). +# RE-SCOPED 2026-06-19 from 500 rps. The original 500 rps cited the rc12 READ-SLA +# gate, but the peach mix is WRITE-heavy (view/like/skip ≈ 90% of ops) and the +# write path is structurally capped on this 3-node RF3 full-placement cluster +# (every follower applies every 1536-D write; T5). At 500 rps the soak FAILED +# every night (2026-06-17/18/19: p99 198-305ms, error 12-33%). A 2026-06-19 +# capacity sweep on a settled cluster measured the peach mix CLEAN (0.00% error, +# p99 schedule-lag <5ms, 0 follower restarts) at 100/150/200/250 rps; the knee is +# between 250 and 500. 200 rps is the measured sustainable rate with margin for a +# 1-hour × 30-consecutive-night endurance run. Raise only with more write capacity +# (≥5 nodes / partitioned placement) — see orchard9-k3sf cluster-state.yaml. # # WHY A CRONJOB, NOT ONE LONG JOB: the GA bar is literally "30 CONSECUTIVE DAYS" # of an independent nightly verdict — a CronJob produces exactly that audit trail @@ -82,7 +87,7 @@ spec: type: RuntimeDefault containers: - name: soak - image: registry.threesix.ai/tidal/stress:m12-soak-eval + image: registry.threesix.ai/tidal/stress:m12-rc7-seedretry imagePullPolicy: IfNotPresent # entrypoint is tidal-stress; wrap it so we can stamp the result file # name with the date and append a ledger line regardless of verdict. @@ -92,7 +97,7 @@ spec: set -u DATE="$(date -u +%Y-%m-%d)" OUT="/results/soak-$DATE.json" - echo "soak $DATE start $(date -u +%H:%M:%SZ) target=cluster image=m12-soak-eval" + echo "soak $DATE start $(date -u +%H:%M:%SZ) target=cluster image=m12-rc7-seedretry-200rps" tidal-stress \ --target https://tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 \ --target https://tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 \ @@ -103,7 +108,7 @@ spec: --embedding-dim 1536 \ --mix peach \ --users 50000 \ - --ramp "500:3600" \ + --ramp "200:3600" \ --json-summary "$OUT" \ --max-p99-ms 150 \ --max-error-pct 1 \ @@ -120,7 +125,7 @@ spec: PASSED="$(grep -o '"passed"[: ]*[a-z]*' "$OUT" 2>/dev/null | head -1 | grep -o '[a-z]*$')" P99="$(grep -o '"overall_p99_ms"[: ]*[0-9.]*' "$OUT" 2>/dev/null | tail -1 | grep -o '[0-9.]*$')" ERRF="$(grep -o '"error_rate"[: ]*[0-9.]*' "$OUT" 2>/dev/null | tail -1 | grep -o '[0-9.]*$')" - printf '%s\t%s\trc=%s\tpassed=%s\tp99_ms=%s\terr_frac=%s\timage=m12-soak-eval\n' \ + printf '%s\t%s\trc=%s\tpassed=%s\tp99_ms=%s\terr_frac=%s\timage=m12-rc7-seedretry-200rps\n' \ "$DATE" "$VERD" "$RC" "${PASSED:-?}" "${P99:-?}" "${ERRF:-?}" >> /results/ledger.tsv echo "soak $DATE end $(date -u +%H:%M:%SZ) verdict=$VERD rc=$RC p99=${P99:-?} err=${ERRF:-?}" # Refresh the streak immediately, CONSULTING restarts.tsv: a night