#!/usr/bin/env bash
# run-reliability.sh — the election-divergence reliability gate.
#
# Runs the cluster-mode e2e divergence suite N times SERIALLY and requires every
# iteration to be all-green. The rolling-restart-under-load false-quarantine was
# ~50% flaky, so a single green is meaningless — the bar is N consecutive passes
# (default 10). Any single failure stops the loop and prints the iteration so the
# fix can be root-caused, never masked with a wider budget.
#
# This is NOT a CI pipeline (the project bans CI/CD — see CLAUDE.md). It is a
# manual operator loop. Run it from the tidaldb workspace root.
#
# Usage:
#   scripts/run-reliability.sh            # 10 iterations (the gate)
#   scripts/run-reliability.sh 3          # quick 3-iteration smoke
#   N=20 scripts/run-reliability.sh       # via env
set -euo pipefail

ITERS="${1:-${N:-10}}"
TESTS=(cluster_reseed cluster_membership)

cd "$(dirname "$0")/.."

echo "── election-divergence reliability gate: ${ITERS} consecutive serial runs ──"
echo "    suite: ${TESTS[*]} (--features cluster-e2e, --test-threads=1)"

# Build once up front so per-iteration timing reflects the run, not the compile.
echo "[build] compiling the e2e test binaries…"
cargo test -p tidal-server --features cluster-e2e \
  $(printf -- '--test %s ' "${TESTS[@]}") --no-run

pass=0
for i in $(seq 1 "${ITERS}"); do
  start=$(date +%s)
  if cargo test -p tidal-server --features cluster-e2e \
       $(printf -- '--test %s ' "${TESTS[@]}") -- --test-threads=1 \
       >"/tmp/reliability-iter-${i}.log" 2>&1; then
    elapsed=$(( $(date +%s) - start ))
    pass=$(( pass + 1 ))
    echo "[iter ${i}/${ITERS}] PASS (${elapsed}s)"
  else
    elapsed=$(( $(date +%s) - start ))
    echo "[iter ${i}/${ITERS}] FAIL (${elapsed}s) — log: /tmp/reliability-iter-${i}.log"
    echo
    echo "── result lines ──"
    grep -E "test result|FAILED|panicked|reseed|quarantin" "/tmp/reliability-iter-${i}.log" | tail -30 || true
    echo
    echo "GATE FAILED after ${pass}/${ITERS} green. A single flake means the fix is"
    echo "incomplete — root-cause it back into the divergence seam; do NOT widen budgets."
    exit 1
  fi
done

echo
echo "GATE PASSED: ${pass}/${ITERS} consecutive serial runs all-green."