tidaldb/scripts/restore-fleet.sh

#!/usr/bin/env bash
# Restore selected tidalDB workloads parked after the 2026-08-12 fleet review.
#
# This script never recreates the retired public Ingresses or DNS records. It
# fails before mutation unless retained storage, immutable images, and scheduler
# headroom satisfy the selected workload's current resource contract.

set -euo pipefail
umask 077
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="${SCRIPT_DIR%/scripts}"
RESTORE_LOG_DIR="${TIDAL_RESTORE_LOG_DIR:-$REPO_ROOT/tmp/restore-logs}"
mkdir -p "$RESTORE_LOG_DIR"
RESTORE_STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
RESTORE_LOG_FILE="$RESTORE_LOG_DIR/restore-${RESTORE_STARTED_AT//:/}-$$.log"
exec > >(tee -a "$RESTORE_LOG_FILE") 2>&1
printf 'event=restore_start ts=%s pid=%s argv=' "$RESTORE_STARTED_AT" "$$"
printf '%q ' "$@"
printf 'log_file=%q\n' "${RESTORE_LOG_FILE##*/}"

export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/orchard9-k3sf.yaml}"

RESTORE_CLUSTER=0
RESTORE_STANDALONE=0
RESTORE_SOAK=0
ACCEPT_FAILED_GATE=0
CHECK_ONLY=0
STANDALONE_DATA_NODE=""
CLUSTER_DATA_NODE_0=""
CLUSTER_DATA_NODE_1=""
CLUSTER_DATA_NODE_2=""
PREFLIGHT_POD=""

usage() {
  cat <<'USAGE'
Usage:
  scripts/restore-fleet.sh --cluster [--check]
  scripts/restore-fleet.sh --standalone [--check]
  scripts/restore-fleet.sh --cluster --soak --accept-failed-gate [--check]

Options:
  --cluster              Restore the three-node RF3 cluster and soak monitor.
  --standalone           Restore the internal standalone store. Public routing
                         remains retired and must be provisioned separately.
  --soak                 Unsuspend the nightly endurance job after restoring the
                         cluster. Requires --accept-failed-gate.
  --accept-failed-gate   Acknowledge that the unchanged 200 rps gate passed only
                         23/55 measured nights and is not a release signal.
  --check                Run every preflight and print capacity; mutate nothing.
  -h, --help             Show this help.

Exactly one of --cluster or --standalone is required.

No-argument execution is intentionally a no-op. The old script could restore a
6-core cluster into a fleet with only ~2 cores free and then claim success via
public routes that no longer exist.
USAGE
}

die() {
  printf 'event=restore_error ts=%s message=%q\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >&2
  printf 'ERROR: %s\n' "$*" >&2
  exit 1
}

for arg in "$@"; do
  case "$arg" in
    --cluster) RESTORE_CLUSTER=1 ;;
    --standalone) RESTORE_STANDALONE=1 ;;
    --soak) RESTORE_SOAK=1 ;;
    --accept-failed-gate) ACCEPT_FAILED_GATE=1 ;;
    --check) CHECK_ONLY=1 ;;
    -h|--help) usage; exit 0 ;;
    *) usage >&2; die "unknown option: $arg" ;;
  esac
done

if [ "$RESTORE_CLUSTER" -eq 0 ] && [ "$RESTORE_STANDALONE" -eq 0 ]; then
  usage >&2
  exit 2
fi
if [ "$RESTORE_CLUSTER" -eq 1 ] && [ "$RESTORE_STANDALONE" -eq 1 ]; then
  die "choose exactly one data plane: --cluster or --standalone"
fi
if [ "$RESTORE_SOAK" -eq 1 ] && [ "$RESTORE_CLUSTER" -eq 0 ]; then
  die "--soak requires --cluster"
fi
if [ "$RESTORE_SOAK" -eq 1 ] && [ "$ACCEPT_FAILED_GATE" -eq 0 ]; then
  die "--soak requires --accept-failed-gate; the unchanged gate passed only 23/55 measured nights"
fi

for command in kubectl jq grep; do
  command -v "$command" >/dev/null 2>&1 || die "$command is required"
done
kubectl version --client >/dev/null
kubectl get namespace tidaldb tidaldb-cluster >/dev/null

KUBECONFIG_DISPLAY="$KUBECONFIG"
if [[ "$KUBECONFIG_DISPLAY" == "$HOME/"* ]]; then
  # Display a literal tilde so the runtime log never records workstation identity.
  # shellcheck disable=SC2088
  printf -v KUBECONFIG_DISPLAY '~/%s' "${KUBECONFIG_DISPLAY#"$HOME"/}"
fi
printf '==> kubeconfig: %s\n' "$KUBECONFIG_DISPLAY"
printf '==> context: %s\n' "$(kubectl config current-context)"
printf '==> selected: cluster=%s standalone=%s soak=%s check_only=%s\n' \
  "$RESTORE_CLUSTER" "$RESTORE_STANDALONE" "$RESTORE_SOAK" "$CHECK_ONLY"

check_claim() {
  local namespace="$1" claim="$2" phase volume policy node
  IFS=$'\t' read -r phase volume <<<"$(
    kubectl -n "$namespace" get "pvc/$claim" \
      -o jsonpath='{.status.phase}{"\t"}{.spec.volumeName}'
  )"
  [ "$phase" = "Bound" ] || die "$namespace/$claim is $phase, expected Bound"
  [ -n "$volume" ] || die "$namespace/$claim has no backing PV"
  policy="$(kubectl get "pv/$volume" -o jsonpath='{.spec.persistentVolumeReclaimPolicy}')"
  [ "$policy" = "Retain" ] || die "$namespace/$claim backing PV $volume uses $policy, expected Retain"
  node="$(
    kubectl get "pv/$volume" -o json |
      jq -r '[
        try (
          .spec.nodeAffinity.required.nodeSelectorTerms[].matchExpressions[]
          | select(.key == "kubernetes.io/hostname" and .operator == "In")
          | .values[]
        ) catch empty
      ] | unique | if length <= 1 then (.[]? // "") else error("PV has ambiguous hostname affinity") end'
  )"
  case "$namespace/$claim" in
    tidaldb/tidaldb-data) STANDALONE_DATA_NODE="$node" ;;
    tidaldb-cluster/data-tidaldb-0) CLUSTER_DATA_NODE_0="$node" ;;
    tidaldb-cluster/data-tidaldb-1) CLUSTER_DATA_NODE_1="$node" ;;
    tidaldb-cluster/data-tidaldb-2) CLUSTER_DATA_NODE_2="$node" ;;
  esac
  printf '    %-18s %-28s Bound / Retain (node=%s)\n' \
    "$namespace" "$claim" "${node:-network-backed}"
}

printf '==> retained storage\n'
if [ "$RESTORE_STANDALONE" -eq 1 ]; then
  check_claim tidaldb tidaldb-data
fi
if [ "$RESTORE_CLUSTER" -eq 1 ]; then
  check_claim tidaldb-cluster data-tidaldb-0
  check_claim tidaldb-cluster data-tidaldb-1
  check_claim tidaldb-cluster data-tidaldb-2
  check_claim tidaldb-cluster tidal-soak-results
fi

check_pod_template_images() {
  local namespace="$1" resource="$2" image images
  images="$(
    kubectl -n "$namespace" get "$resource" -o jsonpath='{range .spec.template.spec.initContainers[*]}{.image}{"\n"}{end}{range .spec.template.spec.containers[*]}{.image}{"\n"}{end}'
  )"
  [ -n "$images" ] || die "$namespace/$resource has no images"
  for image in $images; do
    case "$image" in
      *@sha256:*) ;;
      *) die "$namespace/$resource uses mutable image $image" ;;
    esac
  done
}

check_cronjob_images() {
  local namespace="$1" name="$2" image images
  images="$(
    kubectl -n "$namespace" get "cronjob/$name" -o jsonpath='{range .spec.jobTemplate.spec.template.spec.initContainers[*]}{.image}{"\n"}{end}{range .spec.jobTemplate.spec.template.spec.containers[*]}{.image}{"\n"}{end}'
  )"
  [ -n "$images" ] || die "$namespace/cronjob/$name has no images"
  for image in $images; do
    case "$image" in
      *@sha256:*) ;;
      *) die "$namespace/cronjob/$name uses mutable image $image" ;;
    esac
  done
}

printf '==> immutable workload images\n'
if [ "$RESTORE_STANDALONE" -eq 1 ]; then
  check_pod_template_images tidaldb deployment/tidaldb
fi
if [ "$RESTORE_CLUSTER" -eq 1 ]; then
  check_pod_template_images tidaldb-cluster statefulset/tidaldb
  check_pod_template_images tidaldb-cluster deployment/tidal-soak-monitor
fi
if [ "$RESTORE_SOAK" -eq 1 ]; then
  check_cronjob_images tidaldb-cluster tidal-soak-nightly
fi
printf '    all selected restore dependencies are digest-pinned\n'

require_parked() {
  local namespace="$1" resource="$2" replicas
  replicas="$(kubectl -n "$namespace" get "$resource" -o jsonpath='{.spec.replicas}')"
  [ "${replicas:-0}" -eq 0 ] || \
    die "$namespace/$resource already requests $replicas replicas; this restore only accepts the recorded parked state"
}

if [ "$RESTORE_STANDALONE" -eq 1 ]; then
  require_parked tidaldb deployment/tidaldb
fi
if [ "$RESTORE_CLUSTER" -eq 1 ]; then
  require_parked tidaldb-cluster statefulset/tidaldb
  require_parked tidaldb-cluster deployment/tidal-soak-monitor
  SOAK_SUSPENDED="$(kubectl -n tidaldb-cluster get cronjob/tidal-soak-nightly -o jsonpath='{.spec.suspend}')"
  [ "$SOAK_SUSPENDED" = "true" ] || \
    die "tidaldb-cluster/cronjob/tidal-soak-nightly must be suspended before restore"
fi

# One quantity implementation serves both live pod accounting and workload
# templates. The restore requirement therefore follows the manifest instead of
# duplicating CPU/memory constants that can silently drift.
# The `$q` names below are jq variables and must remain literal shell input.
# shellcheck disable=SC2016
RESOURCE_FILTERS='
  def cpu_m:
    tostring as $q |
    if $q == "" or $q == "null" then 0
    elif ($q | endswith("m")) then ($q | rtrimstr("m") | tonumber)
    elif ($q | endswith("u")) then (($q | rtrimstr("u") | tonumber) / 1000)
    elif ($q | endswith("n")) then (($q | rtrimstr("n") | tonumber) / 1000000)
    else (($q | tonumber) * 1000)
    end;
  def memory_bytes:
    tostring as $q |
    ($q | capture("^(?<n>[0-9]+(?:\\.[0-9]+)?)(?<u>[A-Za-z]*)$")) as $p |
    ($p.n | tonumber) as $n |
    if $p.u == "Ki" then $n * 1024
    elif $p.u == "Mi" then $n * 1048576
    elif $p.u == "Gi" then $n * 1073741824
    elif $p.u == "Ti" then $n * 1099511627776
    elif $p.u == "K" then $n * 1000
    elif $p.u == "M" then $n * 1000000
    elif $p.u == "G" then $n * 1000000000
    elif $p.u == "T" then $n * 1000000000000
    elif $p.u == "" then $n
    else error("unsupported memory quantity: " + $q)
    end;
  def pod_cpu:
    ([.containers[]? | (.resources.requests.cpu // "0") | cpu_m] | add // 0) as $regular |
    ([.initContainers[]? | (.resources.requests.cpu // "0") | cpu_m] | max // 0) as $init |
    ([ $regular, $init ] | max) + ((.overhead.cpu // "0") | cpu_m);
  def pod_memory:
    ([.containers[]? | (.resources.requests.memory // "0") | memory_bytes] | add // 0) as $regular |
    ([.initContainers[]? | (.resources.requests.memory // "0") | memory_bytes] | max // 0) as $init |
    ([ $regular, $init ] | max) + ((.overhead.memory // "0") | memory_bytes);
'

workload_request() {
  local namespace="$1" resource="$2" template_kind="$3"
  kubectl -n "$namespace" get "$resource" -o json |
    jq -c --arg template_kind "$template_kind" "$RESOURCE_FILTERS"'
      (
        if $template_kind == "cronjob"
        then .spec.jobTemplate.spec.template.spec
        else .spec.template.spec
        end
      )
      | {cpu_m: (pod_cpu | ceil), memory_bytes: (pod_memory | ceil)}
    '
}

capacity_snapshot() {
  jq -n \
    --slurpfile nodes <(kubectl get nodes -o json) \
    --slurpfile pods <(kubectl get pods -A -o json) "$RESOURCE_FILTERS"'
      ($pods[0].items
        | map(select(.spec.nodeName != null and .status.phase != "Succeeded" and .status.phase != "Failed"))
        | group_by(.spec.nodeName)
        | map({
            key: .[0].spec.nodeName,
            value: {
              cpu_m: (map(.spec | pod_cpu) | add // 0),
              memory_bytes: (map(.spec | pod_memory) | add // 0)
            }
          })
        | from_entries) as $used |
      $nodes[0].items
      | map({
          node: .metadata.name,
          ready: any(.status.conditions[]?; .type == "Ready" and .status == "True"),
          schedulable: ((.spec.unschedulable // false) | not),
          alloc_cpu_m: (.status.allocatable.cpu | cpu_m),
          alloc_memory_bytes: (.status.allocatable.memory | memory_bytes),
          used_cpu_m: ($used[.metadata.name].cpu_m // 0),
          used_memory_bytes: ($used[.metadata.name].memory_bytes // 0)
        }
        | . + {
            free_cpu_m: (.alloc_cpu_m - .used_cpu_m),
            free_memory_bytes: (.alloc_memory_bytes - .used_memory_bytes)
          })
    '
}

REQUIRED_CPU_M=0
REQUIRED_MEMORY_BYTES=0
DB_CPU_M=0
DB_MEMORY_BYTES=0

add_requirement() {
  local request="$1" replicas="$2" cpu memory
  cpu="$(printf '%s\n' "$request" | jq -r '.cpu_m')"
  memory="$(printf '%s\n' "$request" | jq -r '.memory_bytes')"
  REQUIRED_CPU_M=$((REQUIRED_CPU_M + cpu * replicas))
  REQUIRED_MEMORY_BYTES=$((REQUIRED_MEMORY_BYTES + memory * replicas))
}

if [ "$RESTORE_CLUSTER" -eq 1 ]; then
  DB_REQUEST="$(workload_request tidaldb-cluster statefulset/tidaldb pod)"
  MONITOR_REQUEST="$(workload_request tidaldb-cluster deployment/tidal-soak-monitor pod)"
  DB_CPU_M="$(printf '%s\n' "$DB_REQUEST" | jq -r '.cpu_m')"
  DB_MEMORY_BYTES="$(printf '%s\n' "$DB_REQUEST" | jq -r '.memory_bytes')"
  add_requirement "$DB_REQUEST" 3
  add_requirement "$MONITOR_REQUEST" 1
fi
if [ "$RESTORE_STANDALONE" -eq 1 ]; then
  DB_REQUEST="$(workload_request tidaldb deployment/tidaldb pod)"
  DB_CPU_M="$(printf '%s\n' "$DB_REQUEST" | jq -r '.cpu_m')"
  DB_MEMORY_BYTES="$(printf '%s\n' "$DB_REQUEST" | jq -r '.memory_bytes')"
  add_requirement "$DB_REQUEST" 1
fi
if [ "$RESTORE_SOAK" -eq 1 ]; then
  SOAK_REQUEST="$(workload_request tidaldb-cluster cronjob/tidal-soak-nightly cronjob)"
  add_requirement "$SOAK_REQUEST" 1
fi

CAPACITY="$(capacity_snapshot)"
printf '==> scheduler request headroom\n'
printf '%s\n' "$CAPACITY" | jq -r '.[] | "    \(.node): cpu=\(.free_cpu_m | floor)m free, memory=\((.free_memory_bytes / 1048576) | floor)Mi free, ready=\(.ready), schedulable=\(.schedulable)"'

FREE_CPU_M="$(printf '%s\n' "$CAPACITY" | jq '[.[] | select(.ready and .schedulable) | .free_cpu_m] | add // 0 | floor')"
FREE_MEMORY_BYTES="$(printf '%s\n' "$CAPACITY" | jq '[.[] | select(.ready and .schedulable) | .free_memory_bytes] | add // 0 | floor')"

[ "$FREE_CPU_M" -ge "$REQUIRED_CPU_M" ] || \
  die "selected restore needs ${REQUIRED_CPU_M}m CPU requests; only ${FREE_CPU_M}m is free"
[ "$FREE_MEMORY_BYTES" -ge "$REQUIRED_MEMORY_BYTES" ] || \
  die "selected restore needs $((REQUIRED_MEMORY_BYTES / 1048576))Mi memory requests; only $((FREE_MEMORY_BYTES / 1048576))Mi is free"

node_has_capacity() {
  local node="$1"
  printf '%s\n' "$CAPACITY" |
    jq -e --arg node "$node" --argjson cpu "$DB_CPU_M" --argjson memory "$DB_MEMORY_BYTES" \
      'any(.[]; .node == $node and .ready and .schedulable and .free_cpu_m >= $cpu and .free_memory_bytes >= $memory)' \
      >/dev/null
}

if [ "$RESTORE_CLUSTER" -eq 1 ]; then
  if [ -n "$CLUSTER_DATA_NODE_0" ] && [ -n "$CLUSTER_DATA_NODE_1" ] && [ -n "$CLUSTER_DATA_NODE_2" ]; then
    UNIQUE_DATA_NODES="$(
      jq -n --arg n0 "$CLUSTER_DATA_NODE_0" --arg n1 "$CLUSTER_DATA_NODE_1" --arg n2 "$CLUSTER_DATA_NODE_2" \
        '[$n0, $n1, $n2] | unique | length'
    )"
    [ "$UNIQUE_DATA_NODES" -eq 3 ] || die "RF3 data volumes are not pinned to three distinct nodes"
    for node in "$CLUSTER_DATA_NODE_0" "$CLUSTER_DATA_NODE_1" "$CLUSTER_DATA_NODE_2"; do
      node_has_capacity "$node" || \
        die "RF3 data volume node $node lacks ${DB_CPU_M}m CPU or $((DB_MEMORY_BYTES / 1048576))Mi memory"
    done
  elif [ -z "$CLUSTER_DATA_NODE_0$CLUSTER_DATA_NODE_1$CLUSTER_DATA_NODE_2" ]; then
    CLUSTER_NODES="$(
      printf '%s\n' "$CAPACITY" |
        jq --argjson cpu "$DB_CPU_M" --argjson memory "$DB_MEMORY_BYTES" \
          '[.[] | select(.ready and .schedulable and .free_cpu_m >= $cpu and .free_memory_bytes >= $memory)] | length'
    )"
    [ "$CLUSTER_NODES" -ge 3 ] || \
      die "RF3 topology needs three nodes with ${DB_CPU_M}m CPU and $((DB_MEMORY_BYTES / 1048576))Mi memory free each; only $CLUSTER_NODES qualify"
  else
    die "RF3 data volumes have mixed local and network-backed placement; manual scheduling proof required"
  fi
fi

if [ "$RESTORE_STANDALONE" -eq 1 ]; then
  if [ -n "$STANDALONE_DATA_NODE" ]; then
    node_has_capacity "$STANDALONE_DATA_NODE" || \
      die "standalone data volume node $STANDALONE_DATA_NODE lacks ${DB_CPU_M}m CPU or $((DB_MEMORY_BYTES / 1048576))Mi memory"
  else
    STANDALONE_TARGET_NODE="$(
      printf '%s\n' "$CAPACITY" |
        jq -r --argjson cpu "$DB_CPU_M" --argjson memory "$DB_MEMORY_BYTES" \
          'first(.[] | select(.ready and .schedulable and .free_cpu_m >= $cpu and .free_memory_bytes >= $memory) | .node) // ""'
    )"
    [ -n "$STANDALONE_TARGET_NODE" ] || \
      die "no node can place standalone request ${DB_CPU_M}m/$((DB_MEMORY_BYTES / 1048576))Mi"
    node_has_capacity "$STANDALONE_TARGET_NODE" || \
      die "selected standalone node $STANDALONE_TARGET_NODE lost capacity during preflight"
  fi
fi

printf '==> preflight passed: manifest-derived request %sm CPU and %sMi memory\n' \
  "$REQUIRED_CPU_M" "$((REQUIRED_MEMORY_BYTES / 1048576))"

if [ "$CHECK_ONLY" -eq 1 ]; then
  printf 'VERDICT: RESTORE PREFLIGHT PASSED (no resources changed)\n'
  printf 'event=restore_complete ts=%s mode=check verdict=passed\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
  exit 0
fi
MUTATION_STARTED=1
rollback_on_exit() {
  local rc=$?
  trap - EXIT
  set +e

  if [ -n "$PREFLIGHT_POD" ]; then
    kubectl -n tidaldb-cluster delete "pod/$PREFLIGHT_POD" \
      --ignore-not-found --wait=true >/dev/null 2>&1 ||
      printf 'WARNING: could not delete evaluator preflight pod %s\n' "$PREFLIGHT_POD" >&2
  fi
  if [ "$rc" -eq 0 ] || [ "$MUTATION_STARTED" -eq 0 ]; then
    exit "$rc"
  fi

  printf 'ERROR: restore failed rc=%s; returning selected workloads to the parked state\n' "$rc" >&2
  printf 'event=restore_rollback ts=%s rc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$rc" >&2
  if [ "$RESTORE_SOAK" -eq 1 ]; then
    kubectl -n tidaldb-cluster patch cronjob/tidal-soak-nightly \
      --type=merge -p '{"spec":{"suspend":true}}' >/dev/null ||
      printf 'ERROR: rollback could not suspend tidal-soak-nightly\n' >&2
  fi
  if [ "$RESTORE_CLUSTER" -eq 1 ]; then
    kubectl -n tidaldb-cluster scale deployment/tidal-soak-monitor --replicas=0 >/dev/null ||
      printf 'ERROR: rollback could not park tidal-soak-monitor\n' >&2
    kubectl -n tidaldb-cluster scale statefulset/tidaldb --replicas=0 >/dev/null ||
      printf 'ERROR: rollback could not park tidaldb StatefulSet\n' >&2
    kubectl -n tidaldb-cluster wait --for=delete pod \
      -l app.kubernetes.io/name=tidaldb --timeout=300s >/dev/null 2>&1 ||
      printf 'WARNING: rollback did not observe all cluster pods deleted within 300s\n' >&2
    kubectl -n tidaldb-cluster wait --for=delete pod \
      -l app.kubernetes.io/name=tidal-soak-monitor --timeout=300s >/dev/null 2>&1 ||
      printf 'WARNING: rollback did not observe soak monitor deletion within 300s\n' >&2
  fi
  if [ "$RESTORE_STANDALONE" -eq 1 ]; then
    kubectl -n tidaldb scale deployment/tidaldb --replicas=0 >/dev/null ||
      printf 'ERROR: rollback could not park standalone tidaldb\n' >&2
    kubectl -n tidaldb wait --for=delete pod \
      -l app.kubernetes.io/name=tidaldb --timeout=300s >/dev/null 2>&1 ||
      printf 'WARNING: rollback did not observe standalone pod deletion within 300s\n' >&2
  fi
  printf '==> rollback requested; inspect workload state before retrying\n' >&2
  exit "$rc"
}
trap rollback_on_exit EXIT

if [ "$RESTORE_CLUSTER" -eq 1 ]; then
  printf '==> restoring RF3 cluster\n'
  kubectl -n tidaldb-cluster scale statefulset/tidaldb --replicas=3
  kubectl -n tidaldb-cluster rollout status statefulset/tidaldb --timeout=900s
  kubectl -n tidaldb-cluster wait --for=condition=Ready \
    pod -l app.kubernetes.io/name=tidaldb --timeout=300s

  printf '==> restoring soak monitor (nightly job stays suspended)\n'
  kubectl -n tidaldb-cluster scale deployment/tidal-soak-monitor --replicas=1
  kubectl -n tidaldb-cluster rollout status deployment/tidal-soak-monitor --timeout=300s
fi

if [ "$RESTORE_STANDALONE" -eq 1 ]; then
  printf '==> restoring internal standalone store\n'
  kubectl -n tidaldb scale deployment/tidaldb --replicas=1
  kubectl -n tidaldb rollout status deployment/tidaldb --timeout=600s
  printf '    public Ingress and DNS remain retired by design\n'
fi

if [ "$RESTORE_SOAK" -eq 1 ]; then
  SOAK_IMAGE="$(
    kubectl -n tidaldb-cluster get cronjob/tidal-soak-nightly \
      -o jsonpath='{.spec.jobTemplate.spec.template.spec.containers[0].image}'
  )"
  PREFLIGHT_POD=tidal-soak-eval-preflight
  kubectl -n tidaldb-cluster delete "pod/$PREFLIGHT_POD" \
    --ignore-not-found --wait=true >/dev/null
  PREFLIGHT_OVERRIDES="$(
    jq -nc --arg image "$SOAK_IMAGE" '{
      spec: {
        automountServiceAccountToken: false,
        securityContext: {
          runAsNonRoot: true,
          runAsUser: 1000,
          runAsGroup: 1000,
          seccompProfile: {type: "RuntimeDefault"}
        },
        containers: [{
          name: "preflight",
          image: $image,
          imagePullPolicy: "IfNotPresent",
          command: ["soak-eval", "--help"],
          resources: {
            requests: {cpu: "10m", memory: "16Mi"},
            limits: {cpu: "100m", memory: "64Mi"}
          },
          securityContext: {
            allowPrivilegeEscalation: false,
            readOnlyRootFilesystem: true,
            capabilities: {drop: ["ALL"]}
          }
        }]
      }
    }'
  )"
  kubectl -n tidaldb-cluster run "$PREFLIGHT_POD" \
    --restart=Never --image="$SOAK_IMAGE" --overrides="$PREFLIGHT_OVERRIDES"
  kubectl -n tidaldb-cluster wait --for=jsonpath='{.status.phase}'=Succeeded \
    "pod/$PREFLIGHT_POD" --timeout=180s
  PREFLIGHT_HELP="$(kubectl -n tidaldb-cluster logs "$PREFLIGHT_POD")"
  printf '%s\n' "$PREFLIGHT_HELP"
  grep -q -- '--ledger-file' <<<"$PREFLIGHT_HELP" ||
    die "stress image soak-eval lacks --ledger-file; refusing the non-atomic nightly contract"
  grep -q -- '--no-write-streak' <<<"$PREFLIGHT_HELP" ||
    die "stress image soak-eval lacks --no-write-streak; refusing the non-atomic nightly contract"
  kubectl -n tidaldb-cluster delete "pod/$PREFLIGHT_POD" --wait=true >/dev/null
  PREFLIGHT_POD=""

  printf '==> unsuspending nightly soak after evaluator proof\n'
  kubectl -n tidaldb-cluster patch cronjob/tidal-soak-nightly \
    --type=merge -p '{"spec":{"suspend":false}}'
fi

printf '==> final state\n'
if [ "$RESTORE_CLUSTER" -eq 1 ]; then
  kubectl -n tidaldb-cluster get \
    statefulset/tidaldb deployment/tidal-soak-monitor cronjob/tidal-soak-nightly
fi
if [ "$RESTORE_STANDALONE" -eq 1 ]; then
  kubectl -n tidaldb get deployment/tidaldb
fi
printf 'event=restore_complete ts=%s mode=restore verdict=passed\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
printf 'VERDICT: SELECTED TIDALDB WORKLOADS RESTORED\n'