#!/usr/bin/env bash # Restore selected tidalDB workloads parked after the 2026-08-12 fleet review. # # This script never recreates the retired public Ingresses or DNS records. It # fails before mutation unless retained storage, immutable images, and scheduler # headroom satisfy the selected workload's current resource contract. set -euo pipefail umask 077 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="${SCRIPT_DIR%/scripts}" RESTORE_LOG_DIR="${TIDAL_RESTORE_LOG_DIR:-$REPO_ROOT/tmp/restore-logs}" mkdir -p "$RESTORE_LOG_DIR" RESTORE_STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" RESTORE_LOG_FILE="$RESTORE_LOG_DIR/restore-${RESTORE_STARTED_AT//:/}-$$.log" exec > >(tee -a "$RESTORE_LOG_FILE") 2>&1 printf 'event=restore_start ts=%s pid=%s argv=' "$RESTORE_STARTED_AT" "$$" printf '%q ' "$@" printf 'log_file=%q\n' "${RESTORE_LOG_FILE##*/}" export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/orchard9-k3sf.yaml}" RESTORE_CLUSTER=0 RESTORE_STANDALONE=0 RESTORE_SOAK=0 ACCEPT_FAILED_GATE=0 CHECK_ONLY=0 STANDALONE_DATA_NODE="" CLUSTER_DATA_NODE_0="" CLUSTER_DATA_NODE_1="" CLUSTER_DATA_NODE_2="" PREFLIGHT_POD="" usage() { cat <<'USAGE' Usage: scripts/restore-fleet.sh --cluster [--check] scripts/restore-fleet.sh --standalone [--check] scripts/restore-fleet.sh --cluster --soak --accept-failed-gate [--check] Options: --cluster Restore the three-node RF3 cluster and soak monitor. --standalone Restore the internal standalone store. Public routing remains retired and must be provisioned separately. --soak Unsuspend the nightly endurance job after restoring the cluster. Requires --accept-failed-gate. --accept-failed-gate Acknowledge that the unchanged 200 rps gate passed only 23/55 measured nights and is not a release signal. --check Run every preflight and print capacity; mutate nothing. -h, --help Show this help. Exactly one of --cluster or --standalone is required. No-argument execution is intentionally a no-op. The old script could restore a 6-core cluster into a fleet with only ~2 cores free and then claim success via public routes that no longer exist. USAGE } die() { printf 'event=restore_error ts=%s message=%q\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >&2 printf 'ERROR: %s\n' "$*" >&2 exit 1 } for arg in "$@"; do case "$arg" in --cluster) RESTORE_CLUSTER=1 ;; --standalone) RESTORE_STANDALONE=1 ;; --soak) RESTORE_SOAK=1 ;; --accept-failed-gate) ACCEPT_FAILED_GATE=1 ;; --check) CHECK_ONLY=1 ;; -h|--help) usage; exit 0 ;; *) usage >&2; die "unknown option: $arg" ;; esac done if [ "$RESTORE_CLUSTER" -eq 0 ] && [ "$RESTORE_STANDALONE" -eq 0 ]; then usage >&2 exit 2 fi if [ "$RESTORE_CLUSTER" -eq 1 ] && [ "$RESTORE_STANDALONE" -eq 1 ]; then die "choose exactly one data plane: --cluster or --standalone" fi if [ "$RESTORE_SOAK" -eq 1 ] && [ "$RESTORE_CLUSTER" -eq 0 ]; then die "--soak requires --cluster" fi if [ "$RESTORE_SOAK" -eq 1 ] && [ "$ACCEPT_FAILED_GATE" -eq 0 ]; then die "--soak requires --accept-failed-gate; the unchanged gate passed only 23/55 measured nights" fi for command in kubectl jq grep; do command -v "$command" >/dev/null 2>&1 || die "$command is required" done kubectl version --client >/dev/null kubectl get namespace tidaldb tidaldb-cluster >/dev/null KUBECONFIG_DISPLAY="$KUBECONFIG" if [[ "$KUBECONFIG_DISPLAY" == "$HOME/"* ]]; then # Display a literal tilde so the runtime log never records workstation identity. # shellcheck disable=SC2088 printf -v KUBECONFIG_DISPLAY '~/%s' "${KUBECONFIG_DISPLAY#"$HOME"/}" fi printf '==> kubeconfig: %s\n' "$KUBECONFIG_DISPLAY" printf '==> context: %s\n' "$(kubectl config current-context)" printf '==> selected: cluster=%s standalone=%s soak=%s check_only=%s\n' \ "$RESTORE_CLUSTER" "$RESTORE_STANDALONE" "$RESTORE_SOAK" "$CHECK_ONLY" check_claim() { local namespace="$1" claim="$2" phase volume policy node IFS=$'\t' read -r phase volume <<<"$( kubectl -n "$namespace" get "pvc/$claim" \ -o jsonpath='{.status.phase}{"\t"}{.spec.volumeName}' )" [ "$phase" = "Bound" ] || die "$namespace/$claim is $phase, expected Bound" [ -n "$volume" ] || die "$namespace/$claim has no backing PV" policy="$(kubectl get "pv/$volume" -o jsonpath='{.spec.persistentVolumeReclaimPolicy}')" [ "$policy" = "Retain" ] || die "$namespace/$claim backing PV $volume uses $policy, expected Retain" node="$( kubectl get "pv/$volume" -o json | jq -r '[ try ( .spec.nodeAffinity.required.nodeSelectorTerms[].matchExpressions[] | select(.key == "kubernetes.io/hostname" and .operator == "In") | .values[] ) catch empty ] | unique | if length <= 1 then (.[]? // "") else error("PV has ambiguous hostname affinity") end' )" case "$namespace/$claim" in tidaldb/tidaldb-data) STANDALONE_DATA_NODE="$node" ;; tidaldb-cluster/data-tidaldb-0) CLUSTER_DATA_NODE_0="$node" ;; tidaldb-cluster/data-tidaldb-1) CLUSTER_DATA_NODE_1="$node" ;; tidaldb-cluster/data-tidaldb-2) CLUSTER_DATA_NODE_2="$node" ;; esac printf ' %-18s %-28s Bound / Retain (node=%s)\n' \ "$namespace" "$claim" "${node:-network-backed}" } printf '==> retained storage\n' if [ "$RESTORE_STANDALONE" -eq 1 ]; then check_claim tidaldb tidaldb-data fi if [ "$RESTORE_CLUSTER" -eq 1 ]; then check_claim tidaldb-cluster data-tidaldb-0 check_claim tidaldb-cluster data-tidaldb-1 check_claim tidaldb-cluster data-tidaldb-2 check_claim tidaldb-cluster tidal-soak-results fi check_pod_template_images() { local namespace="$1" resource="$2" image images images="$( kubectl -n "$namespace" get "$resource" -o jsonpath='{range .spec.template.spec.initContainers[*]}{.image}{"\n"}{end}{range .spec.template.spec.containers[*]}{.image}{"\n"}{end}' )" [ -n "$images" ] || die "$namespace/$resource has no images" for image in $images; do case "$image" in *@sha256:*) ;; *) die "$namespace/$resource uses mutable image $image" ;; esac done } check_cronjob_images() { local namespace="$1" name="$2" image images images="$( kubectl -n "$namespace" get "cronjob/$name" -o jsonpath='{range .spec.jobTemplate.spec.template.spec.initContainers[*]}{.image}{"\n"}{end}{range .spec.jobTemplate.spec.template.spec.containers[*]}{.image}{"\n"}{end}' )" [ -n "$images" ] || die "$namespace/cronjob/$name has no images" for image in $images; do case "$image" in *@sha256:*) ;; *) die "$namespace/cronjob/$name uses mutable image $image" ;; esac done } printf '==> immutable workload images\n' if [ "$RESTORE_STANDALONE" -eq 1 ]; then check_pod_template_images tidaldb deployment/tidaldb fi if [ "$RESTORE_CLUSTER" -eq 1 ]; then check_pod_template_images tidaldb-cluster statefulset/tidaldb check_pod_template_images tidaldb-cluster deployment/tidal-soak-monitor fi if [ "$RESTORE_SOAK" -eq 1 ]; then check_cronjob_images tidaldb-cluster tidal-soak-nightly fi printf ' all selected restore dependencies are digest-pinned\n' require_parked() { local namespace="$1" resource="$2" replicas replicas="$(kubectl -n "$namespace" get "$resource" -o jsonpath='{.spec.replicas}')" [ "${replicas:-0}" -eq 0 ] || \ die "$namespace/$resource already requests $replicas replicas; this restore only accepts the recorded parked state" } if [ "$RESTORE_STANDALONE" -eq 1 ]; then require_parked tidaldb deployment/tidaldb fi if [ "$RESTORE_CLUSTER" -eq 1 ]; then require_parked tidaldb-cluster statefulset/tidaldb require_parked tidaldb-cluster deployment/tidal-soak-monitor SOAK_SUSPENDED="$(kubectl -n tidaldb-cluster get cronjob/tidal-soak-nightly -o jsonpath='{.spec.suspend}')" [ "$SOAK_SUSPENDED" = "true" ] || \ die "tidaldb-cluster/cronjob/tidal-soak-nightly must be suspended before restore" fi # One quantity implementation serves both live pod accounting and workload # templates. The restore requirement therefore follows the manifest instead of # duplicating CPU/memory constants that can silently drift. # The `$q` names below are jq variables and must remain literal shell input. # shellcheck disable=SC2016 RESOURCE_FILTERS=' def cpu_m: tostring as $q | if $q == "" or $q == "null" then 0 elif ($q | endswith("m")) then ($q | rtrimstr("m") | tonumber) elif ($q | endswith("u")) then (($q | rtrimstr("u") | tonumber) / 1000) elif ($q | endswith("n")) then (($q | rtrimstr("n") | tonumber) / 1000000) else (($q | tonumber) * 1000) end; def memory_bytes: tostring as $q | ($q | capture("^(?[0-9]+(?:\\.[0-9]+)?)(?[A-Za-z]*)$")) as $p | ($p.n | tonumber) as $n | if $p.u == "Ki" then $n * 1024 elif $p.u == "Mi" then $n * 1048576 elif $p.u == "Gi" then $n * 1073741824 elif $p.u == "Ti" then $n * 1099511627776 elif $p.u == "K" then $n * 1000 elif $p.u == "M" then $n * 1000000 elif $p.u == "G" then $n * 1000000000 elif $p.u == "T" then $n * 1000000000000 elif $p.u == "" then $n else error("unsupported memory quantity: " + $q) end; def pod_cpu: ([.containers[]? | (.resources.requests.cpu // "0") | cpu_m] | add // 0) as $regular | ([.initContainers[]? | (.resources.requests.cpu // "0") | cpu_m] | max // 0) as $init | ([ $regular, $init ] | max) + ((.overhead.cpu // "0") | cpu_m); def pod_memory: ([.containers[]? | (.resources.requests.memory // "0") | memory_bytes] | add // 0) as $regular | ([.initContainers[]? | (.resources.requests.memory // "0") | memory_bytes] | max // 0) as $init | ([ $regular, $init ] | max) + ((.overhead.memory // "0") | memory_bytes); ' workload_request() { local namespace="$1" resource="$2" template_kind="$3" kubectl -n "$namespace" get "$resource" -o json | jq -c --arg template_kind "$template_kind" "$RESOURCE_FILTERS"' ( if $template_kind == "cronjob" then .spec.jobTemplate.spec.template.spec else .spec.template.spec end ) | {cpu_m: (pod_cpu | ceil), memory_bytes: (pod_memory | ceil)} ' } capacity_snapshot() { jq -n \ --slurpfile nodes <(kubectl get nodes -o json) \ --slurpfile pods <(kubectl get pods -A -o json) "$RESOURCE_FILTERS"' ($pods[0].items | map(select(.spec.nodeName != null and .status.phase != "Succeeded" and .status.phase != "Failed")) | group_by(.spec.nodeName) | map({ key: .[0].spec.nodeName, value: { cpu_m: (map(.spec | pod_cpu) | add // 0), memory_bytes: (map(.spec | pod_memory) | add // 0) } }) | from_entries) as $used | $nodes[0].items | map({ node: .metadata.name, ready: any(.status.conditions[]?; .type == "Ready" and .status == "True"), schedulable: ((.spec.unschedulable // false) | not), alloc_cpu_m: (.status.allocatable.cpu | cpu_m), alloc_memory_bytes: (.status.allocatable.memory | memory_bytes), used_cpu_m: ($used[.metadata.name].cpu_m // 0), used_memory_bytes: ($used[.metadata.name].memory_bytes // 0) } | . + { free_cpu_m: (.alloc_cpu_m - .used_cpu_m), free_memory_bytes: (.alloc_memory_bytes - .used_memory_bytes) }) ' } REQUIRED_CPU_M=0 REQUIRED_MEMORY_BYTES=0 DB_CPU_M=0 DB_MEMORY_BYTES=0 add_requirement() { local request="$1" replicas="$2" cpu memory cpu="$(printf '%s\n' "$request" | jq -r '.cpu_m')" memory="$(printf '%s\n' "$request" | jq -r '.memory_bytes')" REQUIRED_CPU_M=$((REQUIRED_CPU_M + cpu * replicas)) REQUIRED_MEMORY_BYTES=$((REQUIRED_MEMORY_BYTES + memory * replicas)) } if [ "$RESTORE_CLUSTER" -eq 1 ]; then DB_REQUEST="$(workload_request tidaldb-cluster statefulset/tidaldb pod)" MONITOR_REQUEST="$(workload_request tidaldb-cluster deployment/tidal-soak-monitor pod)" DB_CPU_M="$(printf '%s\n' "$DB_REQUEST" | jq -r '.cpu_m')" DB_MEMORY_BYTES="$(printf '%s\n' "$DB_REQUEST" | jq -r '.memory_bytes')" add_requirement "$DB_REQUEST" 3 add_requirement "$MONITOR_REQUEST" 1 fi if [ "$RESTORE_STANDALONE" -eq 1 ]; then DB_REQUEST="$(workload_request tidaldb deployment/tidaldb pod)" DB_CPU_M="$(printf '%s\n' "$DB_REQUEST" | jq -r '.cpu_m')" DB_MEMORY_BYTES="$(printf '%s\n' "$DB_REQUEST" | jq -r '.memory_bytes')" add_requirement "$DB_REQUEST" 1 fi if [ "$RESTORE_SOAK" -eq 1 ]; then SOAK_REQUEST="$(workload_request tidaldb-cluster cronjob/tidal-soak-nightly cronjob)" add_requirement "$SOAK_REQUEST" 1 fi CAPACITY="$(capacity_snapshot)" printf '==> scheduler request headroom\n' printf '%s\n' "$CAPACITY" | jq -r '.[] | " \(.node): cpu=\(.free_cpu_m | floor)m free, memory=\((.free_memory_bytes / 1048576) | floor)Mi free, ready=\(.ready), schedulable=\(.schedulable)"' FREE_CPU_M="$(printf '%s\n' "$CAPACITY" | jq '[.[] | select(.ready and .schedulable) | .free_cpu_m] | add // 0 | floor')" FREE_MEMORY_BYTES="$(printf '%s\n' "$CAPACITY" | jq '[.[] | select(.ready and .schedulable) | .free_memory_bytes] | add // 0 | floor')" [ "$FREE_CPU_M" -ge "$REQUIRED_CPU_M" ] || \ die "selected restore needs ${REQUIRED_CPU_M}m CPU requests; only ${FREE_CPU_M}m is free" [ "$FREE_MEMORY_BYTES" -ge "$REQUIRED_MEMORY_BYTES" ] || \ die "selected restore needs $((REQUIRED_MEMORY_BYTES / 1048576))Mi memory requests; only $((FREE_MEMORY_BYTES / 1048576))Mi is free" node_has_capacity() { local node="$1" printf '%s\n' "$CAPACITY" | jq -e --arg node "$node" --argjson cpu "$DB_CPU_M" --argjson memory "$DB_MEMORY_BYTES" \ 'any(.[]; .node == $node and .ready and .schedulable and .free_cpu_m >= $cpu and .free_memory_bytes >= $memory)' \ >/dev/null } if [ "$RESTORE_CLUSTER" -eq 1 ]; then if [ -n "$CLUSTER_DATA_NODE_0" ] && [ -n "$CLUSTER_DATA_NODE_1" ] && [ -n "$CLUSTER_DATA_NODE_2" ]; then UNIQUE_DATA_NODES="$( jq -n --arg n0 "$CLUSTER_DATA_NODE_0" --arg n1 "$CLUSTER_DATA_NODE_1" --arg n2 "$CLUSTER_DATA_NODE_2" \ '[$n0, $n1, $n2] | unique | length' )" [ "$UNIQUE_DATA_NODES" -eq 3 ] || die "RF3 data volumes are not pinned to three distinct nodes" for node in "$CLUSTER_DATA_NODE_0" "$CLUSTER_DATA_NODE_1" "$CLUSTER_DATA_NODE_2"; do node_has_capacity "$node" || \ die "RF3 data volume node $node lacks ${DB_CPU_M}m CPU or $((DB_MEMORY_BYTES / 1048576))Mi memory" done elif [ -z "$CLUSTER_DATA_NODE_0$CLUSTER_DATA_NODE_1$CLUSTER_DATA_NODE_2" ]; then CLUSTER_NODES="$( printf '%s\n' "$CAPACITY" | jq --argjson cpu "$DB_CPU_M" --argjson memory "$DB_MEMORY_BYTES" \ '[.[] | select(.ready and .schedulable and .free_cpu_m >= $cpu and .free_memory_bytes >= $memory)] | length' )" [ "$CLUSTER_NODES" -ge 3 ] || \ die "RF3 topology needs three nodes with ${DB_CPU_M}m CPU and $((DB_MEMORY_BYTES / 1048576))Mi memory free each; only $CLUSTER_NODES qualify" else die "RF3 data volumes have mixed local and network-backed placement; manual scheduling proof required" fi fi if [ "$RESTORE_STANDALONE" -eq 1 ]; then if [ -n "$STANDALONE_DATA_NODE" ]; then node_has_capacity "$STANDALONE_DATA_NODE" || \ die "standalone data volume node $STANDALONE_DATA_NODE lacks ${DB_CPU_M}m CPU or $((DB_MEMORY_BYTES / 1048576))Mi memory" else STANDALONE_TARGET_NODE="$( printf '%s\n' "$CAPACITY" | jq -r --argjson cpu "$DB_CPU_M" --argjson memory "$DB_MEMORY_BYTES" \ 'first(.[] | select(.ready and .schedulable and .free_cpu_m >= $cpu and .free_memory_bytes >= $memory) | .node) // ""' )" [ -n "$STANDALONE_TARGET_NODE" ] || \ die "no node can place standalone request ${DB_CPU_M}m/$((DB_MEMORY_BYTES / 1048576))Mi" node_has_capacity "$STANDALONE_TARGET_NODE" || \ die "selected standalone node $STANDALONE_TARGET_NODE lost capacity during preflight" fi fi printf '==> preflight passed: manifest-derived request %sm CPU and %sMi memory\n' \ "$REQUIRED_CPU_M" "$((REQUIRED_MEMORY_BYTES / 1048576))" if [ "$CHECK_ONLY" -eq 1 ]; then printf 'VERDICT: RESTORE PREFLIGHT PASSED (no resources changed)\n' printf 'event=restore_complete ts=%s mode=check verdict=passed\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" exit 0 fi MUTATION_STARTED=1 rollback_on_exit() { local rc=$? trap - EXIT set +e if [ -n "$PREFLIGHT_POD" ]; then kubectl -n tidaldb-cluster delete "pod/$PREFLIGHT_POD" \ --ignore-not-found --wait=true >/dev/null 2>&1 || printf 'WARNING: could not delete evaluator preflight pod %s\n' "$PREFLIGHT_POD" >&2 fi if [ "$rc" -eq 0 ] || [ "$MUTATION_STARTED" -eq 0 ]; then exit "$rc" fi printf 'ERROR: restore failed rc=%s; returning selected workloads to the parked state\n' "$rc" >&2 printf 'event=restore_rollback ts=%s rc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$rc" >&2 if [ "$RESTORE_SOAK" -eq 1 ]; then kubectl -n tidaldb-cluster patch cronjob/tidal-soak-nightly \ --type=merge -p '{"spec":{"suspend":true}}' >/dev/null || printf 'ERROR: rollback could not suspend tidal-soak-nightly\n' >&2 fi if [ "$RESTORE_CLUSTER" -eq 1 ]; then kubectl -n tidaldb-cluster scale deployment/tidal-soak-monitor --replicas=0 >/dev/null || printf 'ERROR: rollback could not park tidal-soak-monitor\n' >&2 kubectl -n tidaldb-cluster scale statefulset/tidaldb --replicas=0 >/dev/null || printf 'ERROR: rollback could not park tidaldb StatefulSet\n' >&2 kubectl -n tidaldb-cluster wait --for=delete pod \ -l app.kubernetes.io/name=tidaldb --timeout=300s >/dev/null 2>&1 || printf 'WARNING: rollback did not observe all cluster pods deleted within 300s\n' >&2 kubectl -n tidaldb-cluster wait --for=delete pod \ -l app.kubernetes.io/name=tidal-soak-monitor --timeout=300s >/dev/null 2>&1 || printf 'WARNING: rollback did not observe soak monitor deletion within 300s\n' >&2 fi if [ "$RESTORE_STANDALONE" -eq 1 ]; then kubectl -n tidaldb scale deployment/tidaldb --replicas=0 >/dev/null || printf 'ERROR: rollback could not park standalone tidaldb\n' >&2 kubectl -n tidaldb wait --for=delete pod \ -l app.kubernetes.io/name=tidaldb --timeout=300s >/dev/null 2>&1 || printf 'WARNING: rollback did not observe standalone pod deletion within 300s\n' >&2 fi printf '==> rollback requested; inspect workload state before retrying\n' >&2 exit "$rc" } trap rollback_on_exit EXIT if [ "$RESTORE_CLUSTER" -eq 1 ]; then printf '==> restoring RF3 cluster\n' kubectl -n tidaldb-cluster scale statefulset/tidaldb --replicas=3 kubectl -n tidaldb-cluster rollout status statefulset/tidaldb --timeout=900s kubectl -n tidaldb-cluster wait --for=condition=Ready \ pod -l app.kubernetes.io/name=tidaldb --timeout=300s printf '==> restoring soak monitor (nightly job stays suspended)\n' kubectl -n tidaldb-cluster scale deployment/tidal-soak-monitor --replicas=1 kubectl -n tidaldb-cluster rollout status deployment/tidal-soak-monitor --timeout=300s fi if [ "$RESTORE_STANDALONE" -eq 1 ]; then printf '==> restoring internal standalone store\n' kubectl -n tidaldb scale deployment/tidaldb --replicas=1 kubectl -n tidaldb rollout status deployment/tidaldb --timeout=600s printf ' public Ingress and DNS remain retired by design\n' fi if [ "$RESTORE_SOAK" -eq 1 ]; then SOAK_IMAGE="$( kubectl -n tidaldb-cluster get cronjob/tidal-soak-nightly \ -o jsonpath='{.spec.jobTemplate.spec.template.spec.containers[0].image}' )" PREFLIGHT_POD=tidal-soak-eval-preflight kubectl -n tidaldb-cluster delete "pod/$PREFLIGHT_POD" \ --ignore-not-found --wait=true >/dev/null PREFLIGHT_OVERRIDES="$( jq -nc --arg image "$SOAK_IMAGE" '{ spec: { automountServiceAccountToken: false, securityContext: { runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000, seccompProfile: {type: "RuntimeDefault"} }, containers: [{ name: "preflight", image: $image, imagePullPolicy: "IfNotPresent", command: ["soak-eval", "--help"], resources: { requests: {cpu: "10m", memory: "16Mi"}, limits: {cpu: "100m", memory: "64Mi"} }, securityContext: { allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: ["ALL"]} } }] } }' )" kubectl -n tidaldb-cluster run "$PREFLIGHT_POD" \ --restart=Never --image="$SOAK_IMAGE" --overrides="$PREFLIGHT_OVERRIDES" kubectl -n tidaldb-cluster wait --for=jsonpath='{.status.phase}'=Succeeded \ "pod/$PREFLIGHT_POD" --timeout=180s PREFLIGHT_HELP="$(kubectl -n tidaldb-cluster logs "$PREFLIGHT_POD")" printf '%s\n' "$PREFLIGHT_HELP" grep -q -- '--ledger-file' <<<"$PREFLIGHT_HELP" || die "stress image soak-eval lacks --ledger-file; refusing the non-atomic nightly contract" grep -q -- '--no-write-streak' <<<"$PREFLIGHT_HELP" || die "stress image soak-eval lacks --no-write-streak; refusing the non-atomic nightly contract" kubectl -n tidaldb-cluster delete "pod/$PREFLIGHT_POD" --wait=true >/dev/null PREFLIGHT_POD="" printf '==> unsuspending nightly soak after evaluator proof\n' kubectl -n tidaldb-cluster patch cronjob/tidal-soak-nightly \ --type=merge -p '{"spec":{"suspend":false}}' fi printf '==> final state\n' if [ "$RESTORE_CLUSTER" -eq 1 ]; then kubectl -n tidaldb-cluster get \ statefulset/tidaldb deployment/tidal-soak-monitor cronjob/tidal-soak-nightly fi if [ "$RESTORE_STANDALONE" -eq 1 ]; then kubectl -n tidaldb get deployment/tidaldb fi printf 'event=restore_complete ts=%s mode=restore verdict=passed\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" printf 'VERDICT: SELECTED TIDALDB WORKLOADS RESTORED\n'