#!/usr/bin/env bash
# T3 kill loop v3 — HTTP-polling via port-forward to VIP (no exec into tidaldb pods).
# Measures actual election duration: T_kill → first stable new-leader response.
# Run alongside the T3 stress job (see ../k8s/stress-job-t3.yaml).
set -euo pipefail

NS=tidaldb-cluster
KILLS=10
RECOVERY_WAIT=30  # seconds between kill and next iteration (after pod Ready)

# Start port-forward to the client VIP service in background.
kubectl port-forward -n "$NS" svc/tidaldb 19500:9500 &>/tmp/t3-pf.log &
PF_PID=$!
trap 'kill "$PF_PID" 2>/dev/null || true' EXIT
sleep 2  # allow port-forward to establish

get_leader() {
  # Returns empty string on any failure (connection refused, timeout, non-200).
  curl -sf --max-time 2 "http://localhost:19500/cluster/status" 2>/dev/null \
    | python3 -c "
import sys, json
try:
    s = json.load(sys.stdin)
    ldr = s.get('leader', '') or ''
    # Only return if it looks like a valid pod name.
    if ldr.startswith('tidaldb-') and ldr.split('-')[-1].isdigit():
        print(ldr)
except Exception:
    pass
" 2>/dev/null || true
}

echo "T3 kill loop v3 — $(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "Gate: leader election < 10 000ms for all $KILLS kills"
echo ""

RESULTS=()

for i in $(seq 1 "$KILLS"); do
  echo "=== Kill $i / $KILLS at $(date -u +%H:%M:%SZ) ==="

  # Wait up to 30s for a stable leader.
  leader=""
  for _ in $(seq 1 60); do
    candidate=$(get_leader)
    if [[ -n "$candidate" ]]; then
      leader="$candidate"
      break
    fi
    sleep 0.5
  done

  if [[ -z "$leader" ]]; then
    echo "  SKIP: no stable leader after 30s"
    RESULTS+=("SKIP")
    continue
  fi
  echo "  Leader: $leader"

  # Kill with SIGKILL (no graceful drain).
  T_KILL_NS=$(python3 -c "import time; print(int(time.time_ns()))")
  kubectl delete pod "$leader" -n "$NS" --grace-period=0 --force 2>&1 || true

  # Poll at 200ms for a new, different leader.
  new_leader=""
  for _ in $(seq 1 150); do   # 150 × 200ms = 30s max
    sleep 0.2
    candidate=$(get_leader)
    if [[ -n "$candidate" ]] && [[ "$candidate" != "$leader" ]]; then
      new_leader="$candidate"
      break
    fi
  done

  T_ELECT_NS=$(python3 -c "import time; print(int(time.time_ns()))")

  if [[ -n "$new_leader" ]]; then
    elapsed_ms=$(( (T_ELECT_NS - T_KILL_NS) / 1000000 ))
    gate="PASS"
    [[ "$elapsed_ms" -gt 10000 ]] && gate="FAIL"
    echo "  Elected $new_leader in ${elapsed_ms}ms  [$gate]"
    RESULTS+=("${elapsed_ms}ms:$gate")
  else
    echo "  TIMEOUT: no new leader within 30s  [FAIL]"
    RESULTS+=("TIMEOUT:FAIL")
  fi

  # Wait for the killed pod to be Ready again.
  echo "  Waiting for $leader to recover..."
  kubectl wait pod "$leader" -n "$NS" --for=condition=Ready --timeout=120s 2>&1 || \
    echo "  WARNING: $leader did not recover within 120s"

  echo "  Waiting ${RECOVERY_WAIT}s before next kill..."
  sleep "$RECOVERY_WAIT"
done

echo ""
echo "=== T3 Results ==="
PASS=0; FAIL=0
for r in "${RESULTS[@]}"; do
  echo "  $r"
  [[ "$r" == *":PASS" ]] && PASS=$((PASS+1)) || FAIL=$((FAIL+1))
done
echo ""
echo "PASS: $PASS / $KILLS | FAIL/SKIP: $FAIL / $KILLS"
if [[ "$PASS" -eq "$KILLS" ]]; then
  echo "T3 GATE: PASS"
else
  echo "T3 GATE: FAIL"
fi