fix(m12-rc13): read-SLA collapse + WAL_RETENTION_SEGMENTS 16 + tidalctl S3 DR

Read-SLA fix (rc12→rc13 — cpu-cgroup starvation → multi-second p99 + churning
elections):
- offload.rs: add SEARCH_GATE semaphore (core_count+1 permits, 50ms shed to 429)
  so per-shard searches gate on CPU, not reactor threads; concurrent scatter_merge
  fan-out (join_all) replaces the serial blocking offload_region_read loop
- node.rs: scatter_merge → async; per-shard futures run via offload_search
  (each acquires one SEARCH_GATE permit, moves it into spawn_blocking so the
  permit is held for the search's full CPU lifetime)
- main.rs: explicit tokio runtime with worker_threads floored at 4, independent
  of the cgroup quota — keeps the control plane (heartbeat/election/apply) on its
  own workers even when quota < 4
- k8s statefulset: CPU limit 2→3 (was: available_parallelism()=2 → only 2 async
  workers; search burst starved the reactor)
- tidal/wal/compaction.rs: WAL_RETENTION_SEGMENTS 4→16 (64 MiB→256 MiB per-shard
  catch-up window; a briefly-down follower across a rolling restart streams up
  instead of forcing snapshot reseed; disk floor 768 MiB/pod, self-trimming)
- cluster_reseed.rs: OFFLINE_ITEMS 1800→5600 to exceed the new 16-segment
  retention window (19 segs > 17); fix sequential quarantine/reseed race via
  await_status_bool

tidalctl S3/R2 backup DR:
- tidalctl/Cargo.toml: aws-config, aws-sdk-s3, aws-credential-types, tokio, tempfile
- commands/s3.rs: S3Target + export_dir (upload every file, manifest last as
  atomicity marker) + import_to_dir (download prefix into temp staging dir)
- commands/backup.rs: run_backup/run_restore accept Option<&S3Target>; S3 export
  is additive after local fsync barrier; S3 import stages into TempDir then runs
  the unchanged verified restore on it
- main.rs: --s3-endpoint / --s3-bucket / --s3-prefix flags; all-or-nothing
  endpoint+bucket validation; usage updated

tidal-stress/k8s: recall-rc12-spread-job, soak-nightly-cronjob, soak-monitor,
soak-results-pvc, t5-readtput-job manifests

k8s/cluster/statefulset.yaml

@@ -84,7 +84,7 @@ spec:
               mountPath: /data
       containers:
         - name: tidaldb
-          image: registry.threesix.ai/tidal/server@sha256:2e1f7a0e2603fc135f6e79a8c376ca265f8ad74724904b1922df310e8e6da306  # m12-rc11 (= rc10 + corrected Bug 6: only force the re-baselining snapshot when frontier > baseline (divergent); at/below baseline keep frontier+1 so a caught-up node does NOT needlessly reseed-cascade)
+          image: registry.threesix.ai/tidal/server@sha256:ee0a8d8226c88102b7009605cb48768bda329ac3915ec68868e16cc74ca35d9a  # m12-rc13 (= rc12 + WAL_RETENTION_SEGMENTS 4->16: a follower briefly down across a rolling restart stream-catches-up from WAL instead of forcing a snapshot reseed-on-rejoin; per-shard catch-up window 64MiB->256MiB, worst-case 768MiB/pod retained WAL)
           imagePullPolicy: IfNotPresent
           # The image ENTRYPOINT is the bare binary. We override the command with
           # a tiny /bin/sh wrapper (the bookworm-slim runtime HAS a shell) so we
@@ -236,7 +236,14 @@ spec:
               cpu: "500m"
               memory: 1Gi
             limits:
-              cpu: "2"
+              # m12 read-SLA fix: 2->3. The cgroup cpu quota is what the engine's
+              # available_parallelism() reads (SEARCH_GATE / worker_threads sizing).
+              # At "2" a cross-shard search burst starved the async reactor + the
+              # election/heartbeat/apply loops (reads hung to the 30s route timeout;
+              # the starved control plane churned elections -> reseed self-exit).
+              # "3" leaves ~1 core for kubelet/system on the 4-core nodes; requests
+              # stay at 500m so the pod still schedules (server nodes alloc=3).
+              cpu: "3"
               memory: 4Gi           # m12/1536: 100k×1536 HNSW load peaks ~1.9Gi; 1M needs more headroom (nodes have 13Gi allocatable)
           securityContext:
             allowPrivilegeEscalation: false

tidal-server/src/cluster/node.rs

@@ -6546,26 +6546,43 @@ async fn broadcast_to_peers<B: serde::Serialize + Sync>(
 /// pre-m11p6 single read. The cross-group merge keeps each group's own diversity
 /// pass but does NOT re-diversify across groups (a cross-shard re-rank is the L4
 /// follow-up; disjoint groups make the score-merge sound for cardinality).
-fn scatter_merge<T, F>(
-    dbs: &[Arc<TidalDb>],
+async fn scatter_merge<T, F>(
+    dbs: Vec<Arc<TidalDb>>,
     limit: usize,
-    score: impl Fn(&T) -> f64,
+    score: impl Fn(&T) -> f64 + Send,
     per_db: F,
 ) -> std::result::Result<(Vec<T>, usize), ServerError>
 where
-    F: Fn(&TidalDb) -> std::result::Result<(Vec<T>, usize), ServerError>,
+    T: Send + 'static,
+    // `per_db` runs on a `spawn_blocking` thread, once per shard, possibly
+    // concurrently — it must be `Clone + Send + Sync + 'static` and own a clone
+    // of its captured query state. Each invocation gets its own `Arc<TidalDb>`.
+    F: Fn(Arc<TidalDb>) -> crate::offload::ShardSearch<T> + Clone + Send + Sync + 'static,
 {
-    // S=1: the one group's result is already ranked + limited by the engine.
-    // A single group's error IS the read's error — there is nothing to degrade to.
-    if let [only] = dbs {
-        return per_db(only);
+    // S=1: the one group's result is already ranked + limited by the engine. A
+    // single group's error IS the read's error — nothing to degrade to. Still
+    // gated + offloaded (one search, one permit) so it cannot pin the reactor.
+    if let [only] = dbs.as_slice() {
+        let only = Arc::clone(only);
+        let per_db = per_db.clone();
+        return crate::offload::offload_search(move || per_db(only)).await;
     }
+    // Fan every hosted group's search out CONCURRENTLY. `offload_search` gates
+    // each on the process-wide SEARCH_GATE (~= core count), so a wide fan-out
+    // never oversubscribes the cores; a shed shard returns Backpressure here and
+    // is degraded over exactly like any other per-shard error.
+    let futures = dbs.into_iter().map(|db| {
+        let per_db = per_db.clone();
+        crate::offload::offload_search(move || per_db(db))
+    });
+    let results = futures_util::future::join_all(futures).await;
+
     let mut merged: Vec<T> = Vec::new();
     let mut total = 0usize;
     let mut ok_groups = 0usize;
     let mut last_err: Option<ServerError> = None;
-    for (idx, db) in dbs.iter().enumerate() {
-        match per_db(db) {
+    for (idx, result) in results.into_iter().enumerate() {
+        match result {
             Ok((items, candidates)) => {
                 total = total.saturating_add(candidates);
                 merged.extend(items);
@@ -6931,20 +6948,19 @@ pub async fn feed(
 
     // m11p6: scatter the corpus-wide read over the LOCALLY hosted shard groups
     // and merge (see `scatter_merge`).
-    let retrieve = build_retrieve()?;
+    let retrieve = std::sync::Arc::new(build_retrieve()?);
     let dbs = node.hosted_dbs();
-    let (local_items, local_total) = offload_region_read(move || {
-        scatter_merge(
-            &dbs,
+    let (local_items, local_total) = scatter_merge(
+        dbs,
         limit,
         |it: &tidaldb::query::RetrieveResult| it.score,
-            |db| {
+        move |db: Arc<TidalDb>| {
             let r = db.retrieve(&retrieve).map_err(ServerError::Tidal)?;
             Ok((r.items, r.total_candidates))
         },
     )
-    })
-    .await?;
+    .await
+    .map_err(ClusterAppError)?;
 
     // m12p4: full placement / `S=1` → the local scatter already covers the whole
     // corpus; return it UNCHANGED (byte-for-byte the pre-m12p4 response). Under
@@ -7126,21 +7142,20 @@ pub async fn search(
     }
 
     // m11p6: scatter the search over the LOCALLY hosted shard groups and merge.
-    let search_query = build_search()?;
+    let search_query = std::sync::Arc::new(build_search()?);
     let dbs = node.hosted_dbs();
-    let (local_items, local_total) = offload_region_read(move || {
-        scatter_merge(
-            &dbs,
+    let (local_items, local_total) = scatter_merge(
+        dbs,
         limit as usize,
         |it: &tidaldb::query::SearchResultItem| it.score,
-            |db| {
+        move |db: Arc<TidalDb>| {
             db.reload_text_index().map_err(ServerError::Tidal)?;
             let r = db.search(&search_query).map_err(ServerError::Tidal)?;
             Ok((r.items, r.total_candidates))
         },
     )
-    })
-    .await?;
+    .await
+    .map_err(ClusterAppError)?;
 
     // m12p4: full placement / `S=1` → unchanged. Partial placement → fan out.
     let missing = node.missing_groups();
@@ -7268,16 +7283,15 @@ pub async fn vector_search(
     }
 
     // m11p6: scatter the probe over the LOCALLY hosted shard groups and merge.
-    let vector = req.vector.clone();
+    let vector = std::sync::Arc::new(req.vector.clone());
     let dbs = node.hosted_dbs();
-    let (local_items, local_total) = offload_region_read(move || {
-        scatter_merge(
-            &dbs,
+    let (local_items, local_total) = scatter_merge(
+        dbs,
         k,
         // Distance is "lower = better"; scatter_merge ranks by "higher =
         // better", so the merge key is the negated distance.
         |r: &tidaldb::storage::vector::VectorSearchResult| -f64::from(r.distance),
-            |db| {
+        move |db: Arc<TidalDb>| {
             let r = db
                 .vector_search_items(&vector, k, ef_search)
                 .map_err(ServerError::Tidal)?;
@@ -7285,8 +7299,8 @@ pub async fn vector_search(
             Ok((r, n))
         },
     )
-    })
-    .await?;
+    .await
+    .map_err(ClusterAppError)?;
 
     // m12p4: full placement / `S=1` → unchanged. Partial placement → fan out the
     // probe to each missing group (POST body carrying `shard=g`) and merge by

tidal-server/src/main.rs

@@ -109,12 +109,44 @@ struct StandaloneArgs {
     metrics: Option<String>,
 }
 
-#[tokio::main]
-async fn main() {
+/// Entry point.
+///
+/// We build the multi-thread runtime EXPLICITLY rather than via `#[tokio::main]`
+/// so the reactor's worker-thread floor is decoupled from the container's CPU
+/// cgroup quota. `#[tokio::main]` sizes `worker_threads` from
+/// `available_parallelism()`, which is quota-aware: with `limits.cpu="2"` it
+/// yields only 2 async workers, and a burst of CPU-bound shard searches on the
+/// blocking pool can then starve the reactor + the election/heartbeat/apply
+/// loops (the read-SLA collapse). Pinning `worker_threads` keeps the control
+/// plane responsive independent of the quota.
+///
+/// `max_blocking_threads` is DELIBERATELY left at tokio's default (512): the
+/// CPU-bound search admission is bounded by the `offload::SEARCH_GATE` semaphore,
+/// not by starving the blocking pool. Capping the blocking pool low is the one
+/// change that could DEADLOCK (a `spawn_blocking` task parked in a nested
+/// blocking wait would have no thread to make progress), so it is never lowered.
+fn main() {
+    // Floor the reactor at 4 workers (one per physical core on the 4-vCPU
+    // nodes), regardless of the cgroup quota. `enable_all()` matches
+    // `#[tokio::main]`'s I/O + time driver setup.
+    let worker_threads = std::thread::available_parallelism()
+        .map_or(4, std::num::NonZeroUsize::get)
+        .max(4);
+    let runtime = tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(worker_threads)
+        // max_blocking_threads intentionally left at the 512 default — see above.
+        .enable_all()
+        .build()
+        .unwrap_or_else(|e| {
+            eprintln!("error: build tokio runtime: {e}");
+            std::process::exit(1);
+        });
+    runtime.block_on(async {
         if let Err(err) = run().await {
             eprintln!("error: {err}");
             std::process::exit(1);
         }
+    });
 }
 
 async fn run() -> Result<()> {

tidal-server/src/offload.rs

@@ -47,13 +47,51 @@ const READ_BACKPRESSURE_RETRY_AFTER_MS: u64 = 50;
 /// generous headroom, capped well below the blocking-pool limit.
 static READ_GATE: OnceLock<Semaphore> = OnceLock::new();
 
+/// Process-wide cap on CONCURRENT single-db blocking reads admitted by
+/// [`offload_read`] (standalone reads + the single-group internal `?shard` hops).
+/// Each such request runs exactly ONE engine search, so a per-request permit is
+/// the right unit here. Re-sized DOWN from the m12p6 `*16` fan-out (which let a
+/// 2-quota pod admit 32 reads that, multiplied by the 3-shard serial scatter,
+/// became ~96 concurrent CPU-bound searches and a multi-second p99): a small
+/// multiple of the core count buffers a brief burst, then sheds as a fast 429.
 fn read_inflight_limit() -> usize {
-    std::thread::available_parallelism()
-        .map_or(8, std::num::NonZeroUsize::get)
-        .saturating_mul(16)
-        .clamp(16, 256)
+    core_parallelism()
+        .saturating_mul(READ_INFLIGHT_PER_CORE)
+        .clamp(MIN_READ_INFLIGHT, MAX_READ_INFLIGHT)
 }
 
+/// Buffer factor for [`read_inflight_limit`]: a few queued single-db reads per
+/// core absorb a burst without letting the blocking pool climb into a backlog.
+const READ_INFLIGHT_PER_CORE: usize = 4;
+const MIN_READ_INFLIGHT: usize = 4;
+const MAX_READ_INFLIGHT: usize = 64;
+
+/// The quota-aware core count. `available_parallelism` honours the cgroup CPU
+/// quota (e.g. `limits.cpu="3"` -> 3 even on a 4-core node), so this is the
+/// number of CPU-bound shard-searches the pod may run truly in parallel.
+fn core_parallelism() -> usize {
+    std::thread::available_parallelism().map_or(2, std::num::NonZeroUsize::get)
+}
+
+/// Process-wide cap on CONCURRENT CPU-bound shard-searches across ALL in-flight
+/// cross-shard reads. Sized to the core count (with a +1 of slack so one
+/// in-flight search blocked on the embedding-registry read-lock cannot idle a
+/// core), so the async reactor and the election/heartbeat/apply loops always
+/// retain CPU. EVERY per-shard search in [`offload_search`] acquires one
+/// permit; a 3-shard read therefore consumes up to 3 permits, and the gate — not
+/// a per-read counter — is what bounds total search parallelism. Excess sheds as
+/// a fast 429 (see `SEARCH_ADMIT_TIMEOUT_MS`).
+static SEARCH_GATE: OnceLock<Semaphore> = OnceLock::new();
+
+fn search_inflight_limit() -> usize {
+    core_parallelism().saturating_add(1).clamp(2, 16)
+}
+
+/// Max time a per-shard search waits for a [`SEARCH_GATE`] permit before shedding
+/// as backpressure. Same 50ms budget as the read/write gates: a search drains in
+/// worker-thread time, so a quick retry beats parking the request for seconds.
+const SEARCH_ADMIT_TIMEOUT_MS: u64 = READ_BACKPRESSURE_RETRY_AFTER_MS;
+
 /// Run a blocking READ-only query (RETRIEVE / SEARCH / text-index reload) on
 /// tokio's blocking pool, off the async reactor, and await its result.
 ///
@@ -105,6 +143,57 @@ where
     // `_permit` drops here, releasing the read slot.
 }
 
+/// Outcome of one shard's CPU-bound search inside a cross-shard scatter.
+///
+/// Either the engine's `(items, total_candidates)` slice, or a per-shard error
+/// to be degraded over (NOT to abort the whole read). A 429 here is a real
+/// per-shard shed, surfaced like any other shard error so the merge serves
+/// survivors.
+pub type ShardSearch<T> = std::result::Result<(Vec<T>, usize), ServerError>;
+
+/// Run one CPU-bound per-shard search under the process-wide [`SEARCH_GATE`].
+///
+/// Off the reactor: acquires a permit (50ms timeout -> fast `Backpressure` 429,
+/// never a hang), then `spawn_blocking`s the closure. The permit is moved INTO
+/// the blocking closure and dropped only when the search finishes, so the gate
+/// reflects searches actually burning a core, not merely admitted ones.
+///
+/// # Errors
+/// * `TidalError::Backpressure` (429) when the gate is saturated for 50ms.
+/// * `ServerError::Cluster` (500) if the gate is closed (defensive; the static
+///   lives for the process) or the blocking task panicked.
+pub async fn offload_search<F, T>(f: F) -> Result<T>
+where
+    F: FnOnce() -> Result<T> + Send + 'static,
+    T: Send + 'static,
+{
+    let gate = SEARCH_GATE.get_or_init(|| Semaphore::new(search_inflight_limit()));
+    let permit = match tokio::time::timeout(
+        std::time::Duration::from_millis(SEARCH_ADMIT_TIMEOUT_MS),
+        gate.acquire(),
+    )
+    .await
+    {
+        Ok(Ok(permit)) => permit,
+        Ok(Err(_closed)) => {
+            return Err(ServerError::Cluster("search admission gate closed".into()));
+        }
+        Err(_elapsed) => {
+            return Err(ServerError::Tidal(TidalError::Backpressure {
+                retry_after_ms: SEARCH_ADMIT_TIMEOUT_MS,
+            }));
+        }
+    };
+    // Move the permit into the blocking task so it is held for the search's whole
+    // CPU lifetime and released on the blocking thread when the search returns.
+    tokio::task::spawn_blocking(move || {
+        let _permit = permit;
+        f()
+    })
+    .await
+    .map_err(|e| ServerError::Cluster(format!("blocking search worker failed: {e}")))?
+}
+
 /// Configuration for the cluster write worker pool.
 ///
 /// Defaults are derived once at startup: `workers` from

tidal-server/tests/cluster_reseed.rs

@@ -86,15 +86,17 @@ const FAST_ELECTION_YAML: &str = "election:\n  heartbeat_interval_ms: 100\n  ele
 /// follower's frontier (a single segment is never deleted; the snapshot path
 /// only triggers once history is genuinely compacted away).
 ///
-/// Sizing (m12p6): the graceful-shutdown compaction now RETAINS the
-/// `WAL_RETENTION_SEGMENTS` (= 4) most-recent sealed segments so a briefly-down
-/// follower can stream-catch-up instead of reseeding. For this exit-gate to STILL
-/// force the snapshot-required path, the offline batch must compact the follower's
-/// frontier segment away DESPITE retention — i.e. produce more than
-/// `WAL_RETENTION_SEGMENTS + 1` segments. At ~56 KiB/item the 16 MiB segment holds
-/// ~292 items, so 1800 items ≈ 6 segments ⇒ the follower's frontier segment is
-/// well past the 4-segment retention window and is genuinely deleted.
-const OFFLINE_ITEMS: u64 = 1800;
+/// Sizing (m12 rolling-restart tuning): the graceful-shutdown compaction now
+/// RETAINS the `WAL_RETENTION_SEGMENTS` (= 16) most-recent sealed segments so a
+/// briefly-down follower can stream-catch-up instead of reseeding. For this
+/// exit-gate to STILL force the snapshot-required path, the offline batch must
+/// compact the follower's frontier segment away DESPITE retention — i.e. produce
+/// more than `WAL_RETENTION_SEGMENTS + 1` = 17 segments. At ~56 KiB/item the
+/// 16 MiB segment holds ~292 items, so 5600 items ≈ 19 segments ⇒ the follower's
+/// frontier segment is well past the 16-segment retention window and is genuinely
+/// deleted. (Sized off the constant + a 3-segment margin so a future `N` bump
+/// never silently un-sizes this gate.)
+const OFFLINE_ITEMS: u64 = 5600;
 
 /// The engine caps item metadata at 8 KiB per VALUE and 64 KiB TOTAL per item
 /// (a hard query-index-integrity invariant — never to be weakened). So the WAL
@@ -586,10 +588,18 @@ fn mp_quarantined_node_reseeds_without_wipe() {
         convergence_budget() + Duration::from_secs(10),
         "the restarted old leader quarantines on its divergent suffix",
     );
-    assert!(
-        status_bool(&cluster, LEADER, "reseed_required"),
-        "the quarantine latch must also write the reseed marker (m11p5 §2.4): {:?}",
-        cluster.local_status(LEADER)
+    // The quarantine flag (election_driver `quarantined.store`) and the reseed
+    // marker (`latch_reseed_marker`) are latched SEQUENTIALLY, not atomically, so
+    // there is a brief window where status reports quarantined=true before the
+    // marker surfaces. Poll for the marker (matching the quarantined check above)
+    // rather than a single-shot read, which races under concurrent-test load.
+    await_status_bool(
+        &cluster,
+        LEADER,
+        "reseed_required",
+        true,
+        convergence_budget() + Duration::from_secs(10),
+        "the quarantine latch must also write the reseed marker (m11p5 §2.4)",
     );
     println!("[quarantine] old leader quarantined AND latched reseed_required (no wipe)");
 

tidal-stress/k8s/recall-rc12-spread-job.yaml

@@ -0,0 +1,102 @@
+# rc12 read-SLA gate — true p99 + recall@10 vs a brute-force cosine oracle,
+# with read load SPREAD across all 3 nodes (full placement => any node serves any
+# read). recall.rs round-robins each probe across the --target list, so three
+# per-pod DNS targets give an even 1/3-per-node split (deterministic, unlike a
+# single ClusterIP Service which leans on kube-proxy connection balancing).
+# All three per-pod DNS names are cert SANs (wildcard *.tidaldb-peers + -0/-1/-2).
+#
+# Skip-seed: the 100k/1536 corpus already lives on the PVCs (a rolling restart
+# does not wipe them); the harness regenerates the SAME deterministic base
+# vectors locally to build the brute-force ground-truth oracle.
+#
+# Apply: kubectl apply -f tidal-stress/k8s/recall-rc12-spread-job.yaml
+# Watch: kubectl logs -f job/tidal-recall-rc12 -n tidaldb-cluster
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: tidal-recall-rc12
+  namespace: tidaldb-cluster
+  labels:
+    app.kubernetes.io/name: tidal-stress
+    app.kubernetes.io/part-of: tidaldb
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 7200
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: tidal-stress
+        app.kubernetes.io/part-of: tidaldb
+    spec:
+      restartPolicy: Never
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: stress
+          image: registry.threesix.ai/tidal/stress:m12-rc7-seedretry
+          imagePullPolicy: IfNotPresent
+          args:
+            - --target
+            - https://tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
+            - --target
+            - https://tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
+            - --target
+            - https://tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
+            - --ca-cert
+            - /etc/tidaldb/tls/ca.crt
+            - --verify-recall
+            - --skip-seed
+            - --seed-concurrency
+            - "32"
+            - --corpus
+            - "100000"
+            - --embedding-dim
+            - "1536"
+            - --recall-k
+            - "10"
+            - --recall-queries
+            - "1000"
+            - --read-p99-target-ms
+            - "10"
+            - --recall-target
+            - "0.95"
+            - --recall-ef-search
+            - "64"
+            - --ramp
+            - "100:30,200:30,300:30,500:30"
+          env:
+            - name: TIDAL_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: tidaldb-credentials
+                  key: TIDAL_API_KEY
+            - name: TIDAL_STRESS_LOG
+              value: warn
+          resources:
+            requests:
+              cpu: 500m
+              memory: 512Mi
+            limits:
+              cpu: "3"
+              memory: 2Gi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: cluster-tls
+              mountPath: /etc/tidaldb/tls
+              readOnly: true
+      volumes:
+        - name: cluster-tls
+          secret:
+            secretName: tidaldb-cluster-tls
+            items:
+              - key: ca.crt
+                path: ca.crt

tidal-stress/k8s/soak-monitor.yaml

@@ -0,0 +1,157 @@
+# Soak monitor — the always-on in-cluster observer for the 30-night soak.
+#
+# Two jobs, both writing to the SAME durable RWX result PVC the nightly CronJob
+# writes its JSON summaries to, so EVERYTHING the operator needs to judge the
+# 30-night streak lives in one place and survives the laptop session ending:
+#
+#   1. Restart watch: every 5 min, snapshot the tidaldb-{0,1,2} pod restart
+#      counts + phase into /results/restarts.tsv. The GA bar is not just
+#      "30 green soak verdicts" — it is ZERO unrecovered failures over the
+#      window. An under-load pod restart during a soak night must be visible
+#      even if the soak Job itself still passed, so we record it independently.
+#
+#   2. HTTP read surface: serve /results over HTTP on :8080 so the operator can
+#      `kubectl port-forward deploy/tidal-soak-monitor 8080:8080 -n tidaldb-cluster`
+#      and read the ledger / nightly summaries / restart log from a browser at
+#      any time, from any machine, without exec'ing into a pod.
+#
+# The monitor does NOT generate load and does NOT gate anything — it is a passive
+# recorder. The pass/fail signal is the CronJob's Job exit codes; this just makes
+# the 30-night picture observable and durable.
+#
+# Apply: kubectl apply -f tidal-stress/k8s/soak-monitor.yaml
+# Read:  kubectl port-forward deploy/tidal-soak-monitor 8080:8080 -n tidaldb-cluster
+#        then open http://localhost:8080/ledger.tsv  (and /restarts.tsv, /)
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: tidal-soak-monitor
+  namespace: tidaldb-cluster
+  labels:
+    app.kubernetes.io/name: tidal-soak
+    app.kubernetes.io/part-of: tidaldb
+automountServiceAccountToken: true
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: tidal-soak-monitor
+  namespace: tidaldb-cluster
+  labels:
+    app.kubernetes.io/name: tidal-soak
+    app.kubernetes.io/part-of: tidaldb
+rules:
+  # Read-only: pod restart counts + phase, and the nightly soak Job verdicts.
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["batch"]
+    resources: ["jobs"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: tidal-soak-monitor
+  namespace: tidaldb-cluster
+  labels:
+    app.kubernetes.io/name: tidal-soak
+    app.kubernetes.io/part-of: tidaldb
+subjects:
+  - kind: ServiceAccount
+    name: tidal-soak-monitor
+    namespace: tidaldb-cluster
+roleRef:
+  kind: Role
+  name: tidal-soak-monitor
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tidal-soak-monitor
+  namespace: tidaldb-cluster
+  labels:
+    app.kubernetes.io/name: tidal-soak
+    app.kubernetes.io/part-of: tidaldb
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: tidal-soak-monitor
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: tidal-soak-monitor
+        app.kubernetes.io/part-of: tidaldb
+    spec:
+      serviceAccountName: tidal-soak-monitor
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1001
+        runAsGroup: 1001
+        fsGroup: 1001
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        # ── Restart-watch sidecar: kubectl snapshot loop ──────────────────────
+        - name: restart-watch
+          image: bitnami/kubectl:latest
+          imagePullPolicy: IfNotPresent
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              echo "restart-watch up $(date -u +%FT%TZ)"
+              # Header once (only if the file is new/empty).
+              if [ ! -s /results/restarts.tsv ]; then
+                printf 'ts_utc\tpod\trestarts\tphase\tready\n' >> /results/restarts.tsv
+              fi
+              while true; do
+                TS="$(date -u +%FT%TZ)"
+                kubectl get pods -n tidaldb-cluster \
+                  -l app.kubernetes.io/name=tidaldb \
+                  -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.containerStatuses[0].restartCount}{"\t"}{.status.phase}{"\t"}{.status.containerStatuses[0].ready}{"\n"}{end}' 2>/dev/null \
+                  | while IFS="$(printf '\t')" read -r POD RC PH RD; do
+                      [ -n "$POD" ] && printf '%s\t%s\t%s\t%s\t%s\n' "$TS" "$POD" "$RC" "$PH" "$RD" >> /results/restarts.tsv
+                    done
+                sleep 300
+              done
+          resources:
+            requests: { cpu: 10m, memory: 32Mi }
+            limits:   { cpu: 100m, memory: 128Mi }
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities: { drop: ["ALL"] }
+          volumeMounts:
+            - name: results
+              mountPath: /results
+        # ── HTTP read surface: serve the durable result dir ───────────────────
+        - name: http
+          image: busybox:1.36
+          imagePullPolicy: IfNotPresent
+          # busybox httpd: one-shot static file server rooted at /results.
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              echo "http surface up $(date -u +%FT%TZ) on :8080 serving /results"
+              exec httpd -f -p 8080 -h /results
+          ports:
+            - name: http
+              containerPort: 8080
+          resources:
+            requests: { cpu: 10m, memory: 16Mi }
+            limits:   { cpu: 100m, memory: 64Mi }
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities: { drop: ["ALL"] }
+          volumeMounts:
+            - name: results
+              mountPath: /results
+              readOnly: true
+      volumes:
+        - name: results
+          persistentVolumeClaim:
+            claimName: tidal-soak-results

tidal-stress/k8s/soak-nightly-cronjob.yaml

@@ -0,0 +1,162 @@
+# 30-NIGHT SOAK — the calendar half of the M11 GA bar.
+#
+# GA exit gate (docs/planning/milestone-11/phase-9.md §"Exit gate",
+# docs/roadmap-to-cluster.md:115, guarantee-traceability.md G-C):
+#   "Nightly suite green 30 consecutive days before GA."
+#
+# This CronJob IS that nightly. It fires once per night and runs a REAL sustained
+# mixed read+write soak against the live 3-node cluster (round-robin across all
+# three pods — full placement, every pod serves every shard group), with the
+# m11p9 regression gates ARMED so a perf/correctness regression on any night
+# makes the Job FAIL (non-zero exit) — that night is NOT green and the 30-night
+# streak resets.
+#
+# PASS CRITERIA PER NIGHT (each must hold for the Job to exit 0):
+#   --fail-on-knee     : no stage breached the built-in SLO (error rate >1% OR
+#                        feed p99 >150ms — the capacity knee).
+#   --max-p99-ms 150   : no stage's worst-op p99 exceeded 150ms (cluster network-
+#                        hop SLO; in-process SLA is 50ms, the read-recall G1 is
+#                        10ms — 150 is the soak regression tripwire, well above
+#                        the measured 9.28ms read p99 / ~16ms cross-shard p99).
+#   --max-error-pct 1  : no stage's error rate exceeded 1%.
+# The streak passes when 30 CONSECUTIVE nightly Jobs have exited 0 with zero
+# under-load pod restarts (the monitor records restarts; see soak-monitor.yaml).
+#
+# SUSTAINED RATE: 500 rps mixed (peach: feed/search/view/like/skip/item/embed) —
+# deliberately BELOW the SLA knee. The rc12 read-SLA gate sustained 500 rps at
+# p99 9.28ms / recall 0.9989 / 0% err / 0 under-load restarts, so 500 rps is a
+# safe endurance rate that exercises BOTH read and write paths without parking
+# the cluster at its ceiling (a soak proves stability over time, not peak — the
+# 1-hour 100k-DAU peak run is the separate `--ramp 3900:3600` throughput gate).
+#
+# WHY A CRONJOB, NOT ONE LONG JOB: the GA bar is literally "30 CONSECUTIVE DAYS"
+# of an independent nightly verdict — a CronJob produces exactly that audit trail
+# (30 dated Job objects, each PASS/FAIL), survives node reboots/evictions, and
+# each night re-pulls fresh cluster state (a single 30-day Job would mask a
+# mid-window regression and die on any one eviction). Each night is ~70 min of
+# sustained load (a 1h hold + warm-up), bounded by activeDeadlineSeconds.
+#
+# Apply: kubectl apply -f tidal-stress/k8s/soak-nightly-cronjob.yaml
+# Watch tonight's run: kubectl get jobs -n tidaldb-cluster -l app.kubernetes.io/name=tidal-soak
+# Read a night's verdict: kubectl logs -n tidaldb-cluster job/<job-name>
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: tidal-soak-nightly
+  namespace: tidaldb-cluster
+  labels:
+    app.kubernetes.io/name: tidal-soak
+    app.kubernetes.io/part-of: tidaldb
+spec:
+  # 02:00 cluster-local nightly. Off-peak; one hour hold finishes well before any
+  # morning activity. concurrencyPolicy Forbid: never overlap two soaks (they
+  # would contend for the same 3 nodes and mutually depress p99 → false FAIL).
+  schedule: "0 2 * * *"
+  concurrencyPolicy: Forbid
+  startingDeadlineSeconds: 3600
+  successfulJobsHistoryLimit: 30   # keep all 30 nights of PASS verdicts
+  failedJobsHistoryLimit: 30       # keep every FAIL for streak-reset forensics
+  jobTemplate:
+    metadata:
+      labels:
+        app.kubernetes.io/name: tidal-soak
+        app.kubernetes.io/part-of: tidaldb
+    spec:
+      backoffLimit: 0               # a failed night is a FAIL — do not silently retry
+      activeDeadlineSeconds: 5400   # 90 min hard cap (1h hold + warm-up + margin)
+      ttlSecondsAfterFinished: 2678400  # keep finished Job pods 31 days (full window)
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: tidal-soak
+            app.kubernetes.io/part-of: tidaldb
+        spec:
+          restartPolicy: Never
+          automountServiceAccountToken: false
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 1000
+            runAsGroup: 1000
+            fsGroup: 1000              # so the soak user can write the RWX result PVC
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: soak
+              image: registry.threesix.ai/tidal/stress:m12-rc7-seedretry
+              imagePullPolicy: IfNotPresent
+              # entrypoint is tidal-stress; wrap it so we can stamp the result file
+              # name with the date and append a ledger line regardless of verdict.
+              command: ["/bin/sh", "-c"]
+              args:
+                - |
+                  set -u
+                  DATE="$(date -u +%Y-%m-%d)"
+                  OUT="/results/soak-$DATE.json"
+                  echo "soak $DATE start $(date -u +%H:%M:%SZ) target=cluster image=m12-rc7-seedretry"
+                  tidal-stress \
+                    --target https://tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 \
+                    --target https://tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 \
+                    --target https://tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 \
+                    --ca-cert /etc/tidaldb/tls/ca.crt \
+                    --skip-seed \
+                    --corpus 100000 \
+                    --embedding-dim 1536 \
+                    --mix peach \
+                    --users 50000 \
+                    --ramp "500:3600" \
+                    --json-summary "$OUT" \
+                    --max-p99-ms 150 \
+                    --max-error-pct 1 \
+                    --fail-on-knee
+                  RC=$?
+                  # Extract the verdict + headline p99/error from the JSON summary
+                  # for a one-line ledger entry (grep, no jq dependency in the image).
+                  # Fields are the real summary.rs keys: top-level "passed" (bool),
+                  # per-stage "overall_p99_ms" (ms) and "error_rate" (FRACTION 0-1).
+                  # Worst stage = max p99 over the run; we report the LAST stage's
+                  # numbers (the sustained-rate stage, since the soak ramp is single-
+                  # stage 500:3600 anyway) and trust "passed" for the verdict.
+                  VERD="PASS"; [ "$RC" -ne 0 ] && VERD="FAIL"
+                  PASSED="$(grep -o '"passed"[: ]*[a-z]*' "$OUT" 2>/dev/null | head -1 | grep -o '[a-z]*$')"
+                  P99="$(grep -o '"overall_p99_ms"[: ]*[0-9.]*' "$OUT" 2>/dev/null | tail -1 | grep -o '[0-9.]*$')"
+                  ERRF="$(grep -o '"error_rate"[: ]*[0-9.]*' "$OUT" 2>/dev/null | tail -1 | grep -o '[0-9.]*$')"
+                  printf '%s\t%s\trc=%s\tpassed=%s\tp99_ms=%s\terr_frac=%s\timage=m12-rc7-seedretry\n' \
+                    "$DATE" "$VERD" "$RC" "${PASSED:-?}" "${P99:-?}" "${ERRF:-?}" >> /results/ledger.tsv
+                  echo "soak $DATE end $(date -u +%H:%M:%SZ) verdict=$VERD rc=$RC p99=${P99:-?} err=${ERR:-?}"
+                  exit "$RC"
+              env:
+                - name: TIDAL_API_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: tidaldb-credentials
+                      key: TIDAL_API_KEY
+                - name: TIDAL_STRESS_LOG
+                  value: warn
+              resources:
+                requests:
+                  cpu: 500m
+                  memory: 512Mi
+                limits:
+                  cpu: "2"
+                  memory: 2Gi
+              securityContext:
+                allowPrivilegeEscalation: false
+                readOnlyRootFilesystem: true
+                capabilities:
+                  drop: ["ALL"]
+              volumeMounts:
+                - name: cluster-tls
+                  mountPath: /etc/tidaldb/tls
+                  readOnly: true
+                - name: results
+                  mountPath: /results
+          volumes:
+            - name: cluster-tls
+              secret:
+                secretName: tidaldb-cluster-tls
+                items:
+                  - key: ca.crt
+                    path: ca.crt
+            - name: results
+              persistentVolumeClaim:
+                claimName: tidal-soak-results

tidal-stress/k8s/soak-results-pvc.yaml

@@ -0,0 +1,25 @@
+# Durable result sink for the 30-night soak (GA bar: nightly suite green 30
+# consecutive days — docs/planning/milestone-11/phase-9.md "Exit gate").
+#
+# longhorn-rwx: RWX so the nightly soak CronJob pods (writers) and the always-on
+# soak-monitor (reader) can mount it concurrently, and Retain reclaim policy so
+# the 30 nights of JSON summaries survive a PVC delete / job churn / operator
+# session ending. Each night writes /results/soak-YYYY-MM-DD.json + appends one
+# line to /results/ledger.tsv (date<TAB>verdict<TAB>p99<TAB>err% <TAB>image).
+#
+# Apply: kubectl apply -f tidal-stress/k8s/soak-results-pvc.yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: tidal-soak-results
+  namespace: tidaldb-cluster
+  labels:
+    app.kubernetes.io/name: tidal-soak
+    app.kubernetes.io/part-of: tidaldb
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: longhorn-rwx
+  resources:
+    requests:
+      storage: 2Gi

tidal-stress/k8s/t5-readtput-job.yaml

@@ -0,0 +1,67 @@
+# T5 read-throughput (the measurable half): how many /vector_search ops/s the
+# 3-node cluster sustains within SLA, spread round-robin across all 3 pods.
+# (T5-as-written's 2.5x WRITE-scaling is structurally impossible on 3-node RF3
+# full placement — proven in docs/profiling/m12p4-t5-sharded-throughput.md.)
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: tidal-t5-readtput
+  namespace: tidaldb-cluster
+  labels: { app.kubernetes.io/name: tidal-stress, app.kubernetes.io/part-of: tidaldb }
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 7200
+  template:
+    metadata:
+      labels: { app.kubernetes.io/name: tidal-stress, app.kubernetes.io/part-of: tidaldb }
+    spec:
+      restartPolicy: Never
+      automountServiceAccountToken: false
+      securityContext: { runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000, seccompProfile: { type: RuntimeDefault } }
+      containers:
+        - name: stress
+          image: registry.threesix.ai/tidal/stress:m12-rc7-seedretry
+          imagePullPolicy: IfNotPresent
+          args:
+            - --target
+            - https://tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
+            - --target
+            - https://tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
+            - --target
+            - https://tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
+            - --ca-cert
+            - /etc/tidaldb/tls/ca.crt
+            - --verify-recall
+            - --skip-seed
+            - --corpus
+            - "100000"
+            - --embedding-dim
+            - "1536"
+            - --recall-k
+            - "10"
+            - --recall-queries
+            - "1000"
+            - --read-p99-target-ms
+            - "10"
+            - --recall-target
+            - "0.95"
+            - --recall-ef-search
+            - "64"
+            - --ramp
+            - "1000:20,2000:20,3000:20,3800:20"
+          env:
+            - name: TIDAL_API_KEY
+              valueFrom: { secretKeyRef: { name: tidaldb-credentials, key: TIDAL_API_KEY } }
+            - name: TIDAL_STRESS_LOG
+              value: warn
+          resources:
+            requests: { cpu: "1", memory: 512Mi }
+            limits: { cpu: "3", memory: 2Gi }
+          securityContext: { allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: { drop: ["ALL"] } }
+          volumeMounts:
+            - { name: cluster-tls, mountPath: /etc/tidaldb/tls, readOnly: true }
+      volumes:
+        - name: cluster-tls
+          secret:
+            secretName: tidaldb-cluster-tls
+            items: [ { key: ca.crt, path: ca.crt } ]

tidal/src/wal/compaction.rs

@@ -79,7 +79,26 @@ pub fn compact_wal(wal_dir: &Path, checkpoint_seq: u64) -> Result<CompactionResu
 /// segment ages past `N`). A follower further behind than this window still
 /// correctly falls back to snapshot reseed (the serving-side `compacted-below`
 /// safety check is untouched).
-pub const WAL_RETENTION_SEGMENTS: usize = 4;
+///
+/// # Sizing (m12 read-storm / rolling-restart tuning: 4 -> 16)
+///
+/// A 1536-dim embedding insert rides the WAL as one kind-2 blob ≈ 64 B header +
+/// 6144 B vector + ~64 B meta ≈ 6.27 KiB/seqno; a packed signal-event batch is
+/// 32 B/event up to 256 events/batch. At `N = 4` the per-shard catch-up window is
+/// `4 * 16 MiB = 64 MiB` ≈ only ~10.4 k embedding-seqnos, so a follower that fell
+/// a few thousand inserts behind (or was down across a rolling-restart) crossed
+/// the window and was forced into a full snapshot reseed. `N = 16` gives
+/// `16 * 16 MiB = 256 MiB` ≈ ~42.8 k embedding-seqnos (or ~8.4 M batched
+/// event-seqnos) of stream-catch-up headroom per shard.
+///
+/// # Disk bound
+///
+/// Retention is per-shard. With 3 shards hosted per pod under one 5 GiB PVC, the
+/// worst-case retained-WAL floor is `3 * 16 * 16 MiB = 768 MiB` (≈ 15 % of the
+/// PVC), bounded and self-trimming as segments age past `N`. The remaining
+/// ~4.2 GiB stays free for the signal ledger, HNSW graph, snapshots, and the live
+/// (uncompacted) segment per shard.
+pub const WAL_RETENTION_SEGMENTS: usize = 16;
 
 /// Lower `floor` so the `WAL_RETENTION_SEGMENTS` most-recent (highest-`first_seq`)
 /// segments always survive, regardless of the checkpoint. `segments` is sorted

tidalctl/Cargo.toml

@@ -36,6 +36,26 @@ serde_json = "1"
 # WAL segments and snapshot artifacts with, so a tidalctl-written manifest and an
 # engine-verified one agree.
 blake3 = "1"
+# Object-store (S3-compatible) export/import for the DR gate (m11p8 R2). Chosen
+# over `rust-s3` because the AWS SDK shares the workspace's existing
+# hyper/rustls/tokio tree (no native-tls / second TLS stack), and R2 is handled
+# with `.endpoint_url(<account>.r2.cloudflarestorage.com)` + `.force_path_style`.
+# `default-features = false` + `rustls` keeps the TLS stack aligned with the rest
+# of the workspace (reqwest is already `rustls-tls`); `behavior-version-latest`
+# pins the SDK behavior contract so an SDK minor bump can't silently change it.
+aws-config = { version = "1.8", default-features = false, features = ["behavior-version-latest", "rustls"] }
+# `rt-tokio` exposes `ByteStream::from_path` (stream a file body straight off
+# disk on `put_object` — no whole-file read into memory).
+aws-sdk-s3 = { version = "1.137", default-features = false, features = ["behavior-version-latest", "rustls", "rt-tokio"] }
+# `hardcoded-credentials` exposes `Credentials::from_keys` so the env-var creds
+# (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY) become a static SigV4 provider.
+aws-credential-types = { version = "1.2", features = ["hardcoded-credentials"] }
+# S3 transfers run on a locally-built current-thread runtime ONLY when an --s3-*
+# flag is given; the local-dir backup/restore path stays fully synchronous.
+tokio = { version = "1", default-features = false, features = ["rt", "macros"] }
+# Fresh staging dir for an S3 import (downloaded prefix -> temp dir -> the
+# UNCHANGED verified restore reads it). Reaped when the restore returns.
+tempfile = "3"
 
 [dev-dependencies]
 tidaldb = { path = "../tidal", features = ["test-utils"] }

tidalctl/src/commands/backup.rs

@@ -35,6 +35,7 @@ use std::{
 
 use serde::{Deserialize, Serialize};
 
+use crate::commands::s3::S3Target;
 use crate::{CliError, wal_state};
 
 /// Manifest filename written at the root of every backup.
@@ -80,7 +81,18 @@ const MANIFEST_VERSION: u32 = 1;
 /// `tidalctl backup --path <data-dir> --out <dest>`: copy the data dir to `dest`
 /// and write a BLAKE3 manifest. Refuses to overwrite a non-empty destination so a
 /// backup never clobbers an existing one.
-pub(crate) fn run_backup(src: &Path, out: &Path, pretty: bool) -> Result<(String, i32), CliError> {
+///
+/// When `s3` is `Some`, the export is ADDITIVE: after the local `out` dir and its
+/// `BACKUP_MANIFEST.json` are written and fsynced (the consistency barrier),
+/// every file under `out` is mirrored into the bucket with the manifest uploaded
+/// LAST as the export's atomicity marker. With `s3 == None` the behavior is
+/// byte-identical to the original local-only backup.
+pub(crate) fn run_backup(
+    src: &Path,
+    out: &Path,
+    s3: Option<&S3Target>,
+    pretty: bool,
+) -> Result<(String, i32), CliError> {
     if !src.is_dir() {
         return Err(CliError::new(format!(
             "source data dir does not exist: {}",
@@ -167,24 +179,71 @@ pub(crate) fn run_backup(src: &Path, out: &Path, pretty: bool) -> Result<(String
     fsync_dirs_recursive(out)
         .map_err(|e| CliError::new(format!("fsync backup dirs {}: {e}", out.display())))?;
 
-    let summary = serde_json::json!({
+    // ADDITIVE S3/R2 export. The local artifact above is the source of truth and
+    // is already durable; the export walks it and `put_object`s every file under
+    // the prefix, uploading BACKUP_MANIFEST.json LAST so the manifest's presence
+    // in the bucket signals a complete export (the object-store analogue of the
+    // local fsync barrier). A pure local backup (`s3 == None`) skips this entirely.
+    let mut summary = serde_json::json!({
         "backed_up": src.display().to_string(),
         "destination": out.display().to_string(),
         "checkpoint_seq": manifest.checkpoint_seq,
         "file_count": manifest.file_count,
         "total_bytes": manifest.total_bytes,
     });
+    if let Some(target) = s3 {
+        let uploaded = crate::commands::s3::export_dir(out, target)?;
+        summary["s3_export"] = serde_json::json!({
+            "endpoint": target.endpoint,
+            "bucket": target.bucket,
+            "prefix": target.prefix,
+            "objects_uploaded": uploaded,
+            "atomicity_marker": BACKUP_MANIFEST,
+        });
+    }
     Ok((render(&summary, pretty)?, 0))
 }
 
 /// `tidalctl restore --from <backup> --path <target>`: verify the backup's BLAKE3
 /// manifest, then copy every file into `target`. Refuses a non-empty target so a
 /// restore never overwrites a live data dir (the destructive-op guard).
+///
+/// When `s3` is `Some`, the import is ADDITIVE: the prefix is downloaded into a
+/// FRESH temp staging dir and then the UNCHANGED verified restore runs on that
+/// staging dir (BLAKE3 verification, `safe_join`, fresh-target placement). The
+/// `--from` flag is not required in S3 mode (the staging dir replaces it); with
+/// `s3 == None` the behavior is byte-identical to the original local restore.
 pub(crate) fn run_restore(
     target: &Path,
-    from: &Path,
+    from: Option<&Path>,
+    s3: Option<&S3Target>,
     pretty: bool,
 ) -> Result<(String, i32), CliError> {
+    // Resolve the directory the verified restore reads from. In S3 mode, download
+    // the prefix into a fresh temp staging dir and restore from THAT, unchanged;
+    // the TempDir guard is held for the whole restore so the staging bytes survive
+    // BLAKE3 verification + copy, then are reaped on return.
+    let staging = match s3 {
+        Some(target_s3) => {
+            let dir = tempfile::Builder::new()
+                .prefix("tidalctl-s3-restore-")
+                .tempdir()
+                .map_err(|e| CliError::new(format!("create S3 staging dir: {e}")))?;
+            let downloaded = crate::commands::s3::import_to_dir(dir.path(), target_s3)?;
+            Some((dir, downloaded))
+        }
+        None => None,
+    };
+    let (from, s3_downloaded): (&Path, Option<u64>) = match (&staging, from) {
+        (Some((dir, n)), _) => (dir.path(), Some(*n)),
+        (None, Some(local)) => (local, None),
+        (None, None) => {
+            return Err(CliError::new(
+                "restore requires --from <backup-dir> (or --s3-endpoint/--s3-bucket for an object-store import)",
+            ));
+        }
+    };
+
     let manifest_path = from.join(BACKUP_MANIFEST);
     let manifest_bytes = std::fs::read(&manifest_path).map_err(|e| {
         CliError::new(format!(
@@ -264,13 +323,25 @@ pub(crate) fn run_restore(
     fsync_dirs_recursive(target)
         .map_err(|e| CliError::new(format!("fsync target dirs {}: {e}", target.display())))?;
 
-    let summary = serde_json::json!({
+    let mut summary = serde_json::json!({
         "restored": target.display().to_string(),
         "from": from.display().to_string(),
         "checkpoint_seq": manifest.checkpoint_seq,
         "files_verified_and_restored": restored,
         "note": "point a stopped node at this dir; under ack=quorum followers catch up via the live stream",
     });
+    if let (Some(s3_target), Some(downloaded)) = (s3, s3_downloaded) {
+        summary["s3_import"] = serde_json::json!({
+            "endpoint": s3_target.endpoint,
+            "bucket": s3_target.bucket,
+            "prefix": s3_target.prefix,
+            "objects_downloaded": downloaded,
+            "staged_then_verified": true,
+        });
+    }
+    // The TempDir staging guard (if any) is dropped here, after the verified
+    // restore copied every byte into `target` — the staged download is reaped.
+    drop(staging);
     Ok((render(&summary, pretty)?, 0))
 }
 

tidalctl/src/commands/mod.rs

@@ -9,5 +9,6 @@ pub(crate) mod backup;
 pub(crate) mod diagnostics;
 pub(crate) mod paths;
 pub(crate) mod recover;
+pub(crate) mod s3;
 pub(crate) mod scope_stats;
 pub(crate) mod status;

tidalctl/src/commands/s3.rs

@@ -0,0 +1,522 @@
+//! Object-store (S3-compatible, targeting Cloudflare R2) export/import for the DR
+//! gate (m11p8).
+//!
+//! This is an ADDITIVE layer over the local-dir backup/restore in
+//! [`crate::commands::backup`]: it never reimplements the consistency-proven
+//! backup engine. EXPORT runs AFTER the local artifact (data files +
+//! `BACKUP_MANIFEST.json`) is written and fsynced, then mirrors that exact tree
+//! into the bucket. IMPORT downloads the prefix into a temp staging dir and hands
+//! it to the UNCHANGED verified restore (BLAKE3 + `safe_join` + fresh-target).
+//!
+//! ## Atomicity marker
+//!
+//! `BACKUP_MANIFEST.json` is uploaded LAST. The restore reads the manifest first
+//! and verifies every listed file's BLAKE3 against the bytes it downloaded, so a
+//! partial export (process died mid-upload) leaves the manifest absent and the
+//! import fails closed with "is the prefix a tidalctl backup?" rather than
+//! restoring a truncated tree. The manifest's presence is therefore the
+//! commit point of the whole export, exactly as the local fsync barrier is for
+//! the on-disk artifact.
+//!
+//! ## R2 compatibility
+//!
+//! R2 is S3-v4-signature compatible behind a custom endpoint
+//! (`https://<account-id>.r2.cloudflarestorage.com`). We pass that endpoint via
+//! `--s3-endpoint`, force path-style addressing (R2 does not do virtual-hosted
+//! `<bucket>.<endpoint>` buckets), and read credentials from the standard
+//! `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` env vars. The region is a
+//! placeholder (`auto`) R2 ignores but the `SigV4` signer requires.
+
+use std::path::{Path, PathBuf};
+
+use aws_credential_types::Credentials;
+use aws_sdk_s3::config::{BehaviorVersion, Region};
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::Client;
+
+use crate::CliError;
+
+/// The manifest filename — duplicated from [`crate::commands::backup`] (it is a
+/// private const there) so the export can guarantee it is the LAST object put.
+const BACKUP_MANIFEST: &str = "BACKUP_MANIFEST.json";
+
+/// R2 ignores the `SigV4` region but the signer requires a non-empty one.
+const R2_PLACEHOLDER_REGION: &str = "auto";
+
+/// The S3/R2 target for an export or import: a bucket plus an optional key
+/// prefix, against a custom endpoint. Built from the `--s3-endpoint/-bucket/
+/// -prefix` flags; `None` for a pure local-dir operation (the additive S3 layer
+/// is skipped entirely, the synchronous local path is byte-identical to before).
+#[derive(Debug, Clone)]
+pub(crate) struct S3Target {
+    pub endpoint: String,
+    pub bucket: String,
+    /// Forward-slash key prefix, normalized to no leading/trailing slash.
+    pub prefix: String,
+}
+
+impl S3Target {
+    /// Build a target from the three flags, requiring all-or-nothing: an S3
+    /// export/import needs an endpoint AND a bucket (the prefix may be empty, a
+    /// bucket-root export). A partial set is a usage error, not a silent
+    /// local-only fallback — an operator who passed `--s3-bucket` but forgot
+    /// `--s3-endpoint` must hear about it, not get a local-only backup they think
+    /// went to R2.
+    pub(crate) fn from_flags(
+        endpoint: Option<&str>,
+        bucket: Option<&str>,
+        prefix: Option<&str>,
+    ) -> Result<Option<Self>, CliError> {
+        match (endpoint, bucket) {
+            (None, None) => {
+                if prefix.is_some() {
+                    return Err(CliError::new(
+                        "--s3-prefix requires --s3-endpoint and --s3-bucket",
+                    ));
+                }
+                Ok(None)
+            }
+            (Some(endpoint), Some(bucket)) => Ok(Some(Self {
+                endpoint: endpoint.to_string(),
+                bucket: bucket.to_string(),
+                prefix: normalize_prefix(prefix.unwrap_or("")),
+            })),
+            (Some(_), None) => Err(CliError::new("--s3-endpoint given without --s3-bucket")),
+            (None, Some(_)) => Err(CliError::new("--s3-bucket given without --s3-endpoint")),
+        }
+    }
+}
+
+/// Strip leading/trailing slashes and collapse the empty-prefix case to `""`, so
+/// the prefix joins cleanly with a relative path via [`object_key`].
+fn normalize_prefix(prefix: &str) -> String {
+    prefix.trim_matches('/').to_string()
+}
+
+/// Map a backup-relative file path to its object key under the prefix.
+///
+/// PURE function (the unit-tested core of the key mapping): joins the normalized
+/// prefix with the relative path using a single `/`, always forward-slash
+/// separated regardless of host path separator, and never emits a leading slash
+/// or a doubled `//`. An empty prefix yields the bare relative path. The relative
+/// path is itself already forward-slash form (it is the manifest's `path`, or a
+/// path derived from one via the same `rel_to_slash` rendering the backup writes).
+///
+/// This is the inverse of [`relative_from_key`]: `relative_from_key(prefix,
+/// object_key(prefix, rel)) == rel` for any prefix and any non-empty rel — the
+/// round-trip the export/import pair depends on.
+pub(crate) fn object_key(prefix: &str, rel: &str) -> String {
+    let prefix = prefix.trim_matches('/');
+    let rel = rel.trim_start_matches('/');
+    if prefix.is_empty() {
+        rel.to_string()
+    } else {
+        format!("{prefix}/{rel}")
+    }
+}
+
+/// Recover the backup-relative path from a listed object key, given the prefix
+/// used to export it. The inverse of [`object_key`].
+///
+/// Returns `None` if `key` is not under `prefix` (a stray object the list
+/// returned that was not part of this backup) — the caller skips it rather than
+/// downloading something outside the artifact.
+pub(crate) fn relative_from_key(prefix: &str, key: &str) -> Option<String> {
+    let prefix = prefix.trim_matches('/');
+    if prefix.is_empty() {
+        return Some(key.trim_start_matches('/').to_string());
+    }
+    let with_slash = format!("{prefix}/");
+    key.strip_prefix(&with_slash)
+        .filter(|rest| !rest.is_empty())
+        .map(ToString::to_string)
+}
+
+/// Build an S3 client for an R2-compatible endpoint with path-style addressing
+/// and `SigV4` creds from the environment. Returns a `CliError` (mapped to exit 1)
+/// rather than panicking if creds are absent, so an operator who forgot to export
+/// `AWS_ACCESS_KEY_ID` gets a clean message, not a stack trace.
+async fn build_client(target: &S3Target) -> Result<Client, CliError> {
+    let access_key = std::env::var("AWS_ACCESS_KEY_ID").map_err(|_| {
+        CliError::new(
+            "AWS_ACCESS_KEY_ID is not set (required for the S3/R2 export; set it plus \
+             AWS_SECRET_ACCESS_KEY to your R2 token credentials)",
+        )
+    })?;
+    let secret_key = std::env::var("AWS_SECRET_ACCESS_KEY")
+        .map_err(|_| CliError::new("AWS_SECRET_ACCESS_KEY is not set (required for the S3/R2 export)"))?;
+
+    let creds = Credentials::from_keys(access_key, secret_key, None);
+
+    let conf = aws_config::defaults(BehaviorVersion::latest())
+        .region(Region::new(R2_PLACEHOLDER_REGION))
+        .endpoint_url(&target.endpoint)
+        .credentials_provider(creds)
+        .load()
+        .await;
+
+    let s3_conf = aws_sdk_s3::config::Builder::from(&conf)
+        // R2 buckets are addressed path-style (`<endpoint>/<bucket>/<key>`), not
+        // virtual-hosted (`<bucket>.<endpoint>`), so force it.
+        .force_path_style(true)
+        .build();
+    Ok(Client::from_conf(s3_conf))
+}
+
+/// Run an async S3 closure to completion on a local current-thread runtime.
+///
+/// tidalctl is otherwise fully synchronous; we build the runtime ONLY on the
+/// S3 path (when `--s3-*` is given) so the local-dir backup/restore stays a plain
+/// blocking call with zero runtime overhead.
+fn block_on<F, T>(fut: F) -> Result<T, CliError>
+where
+    F: std::future::Future<Output = Result<T, CliError>>,
+{
+    let rt = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .map_err(|e| CliError::new(format!("failed to build S3 transfer runtime: {e}")))?;
+    rt.block_on(fut)
+}
+
+/// EXPORT: mirror the just-written local backup dir into the bucket under the
+/// prefix, uploading `BACKUP_MANIFEST.json` LAST as the completion marker.
+///
+/// Called by `run_backup` after the local artifact is fully written and fsynced.
+/// `local_dir` is that local artifact dir; every regular file under it is
+/// `put_object`ed, manifest last.
+pub(crate) fn export_dir(local_dir: &Path, target: &S3Target) -> Result<u64, CliError> {
+    // Enumerate every regular file (relative paths), then partition the manifest
+    // out so it is uploaded strictly last.
+    let mut rel_files = Vec::new();
+    collect_rel_files(local_dir, Path::new(""), &mut rel_files)
+        .map_err(|e| CliError::new(format!("walk local backup dir: {e}")))?;
+
+    let manifest_present = rel_files.iter().any(|r| rel_to_slash(r) == BACKUP_MANIFEST);
+    if !manifest_present {
+        return Err(CliError::new(format!(
+            "local backup dir {} has no {BACKUP_MANIFEST}; refusing to export an artifact \
+             with no atomicity marker",
+            local_dir.display()
+        )));
+    }
+
+    block_on(async move {
+        let client = build_client(target).await?;
+
+        let mut uploaded = 0u64;
+        // Data files first; manifest deferred to the end.
+        for rel in &rel_files {
+            let rel_slash = rel_to_slash(rel);
+            if rel_slash == BACKUP_MANIFEST {
+                continue;
+            }
+            put_file(&client, target, local_dir, &rel_slash).await?;
+            uploaded += 1;
+        }
+        // The atomicity marker LAST: its presence signals a complete export.
+        put_file(&client, target, local_dir, BACKUP_MANIFEST).await?;
+        uploaded += 1;
+        Ok(uploaded)
+    })
+}
+
+/// IMPORT: download every object under the prefix into `staging_dir` (which must
+/// be a fresh temp dir), reconstructing the backup tree, then return so the
+/// caller hands `staging_dir` to the UNCHANGED verified restore.
+pub(crate) fn import_to_dir(staging_dir: &Path, target: &S3Target) -> Result<u64, CliError> {
+    block_on(async move {
+        let client = build_client(target).await?;
+
+        let keys = list_keys(&client, target).await?;
+        if keys.is_empty() {
+            return Err(CliError::new(format!(
+                "no objects under s3://{}/{} (is the prefix a tidalctl backup?)",
+                target.bucket, target.prefix
+            )));
+        }
+
+        let mut manifest_seen = false;
+        let mut downloaded = 0u64;
+        for key in &keys {
+            let Some(rel) = relative_from_key(&target.prefix, key) else {
+                continue; // stray object outside the prefix (defensive)
+            };
+            if rel == BACKUP_MANIFEST {
+                manifest_seen = true;
+            }
+            get_to_file(&client, target, key, staging_dir, &rel).await?;
+            downloaded += 1;
+        }
+
+        if !manifest_seen {
+            return Err(CliError::new(format!(
+                "prefix s3://{}/{} has no {BACKUP_MANIFEST}; the export is incomplete or this \
+                 is not a tidalctl backup (refusing to restore a partial artifact)",
+                target.bucket, target.prefix
+            )));
+        }
+        Ok(downloaded)
+    })
+}
+
+/// PUT one file from the local dir to its object key.
+async fn put_file(
+    client: &Client,
+    target: &S3Target,
+    local_dir: &Path,
+    rel_slash: &str,
+) -> Result<(), CliError> {
+    let key = object_key(&target.prefix, rel_slash);
+    let abs = local_dir.join(rel_to_native(rel_slash));
+    let body = ByteStream::from_path(&abs)
+        .await
+        .map_err(|e| CliError::new(format!("open {} for upload: {e}", abs.display())))?;
+    client
+        .put_object()
+        .bucket(&target.bucket)
+        .key(&key)
+        .body(body)
+        .send()
+        .await
+        .map_err(|e| CliError::new(format!("put_object {key}: {}", s3_err(&e))))?;
+    Ok(())
+}
+
+/// GET one object into the staging dir at its relative path.
+async fn get_to_file(
+    client: &Client,
+    target: &S3Target,
+    key: &str,
+    staging_dir: &Path,
+    rel: &str,
+) -> Result<(), CliError> {
+    let resp = client
+        .get_object()
+        .bucket(&target.bucket)
+        .key(key)
+        .send()
+        .await
+        .map_err(|e| CliError::new(format!("get_object {key}: {}", s3_err(&e))))?;
+    let bytes = resp
+        .body
+        .collect()
+        .await
+        .map_err(|e| CliError::new(format!("read object body {key}: {e}")))?
+        .into_bytes();
+
+    let dest = staging_dir.join(rel_to_native(rel));
+    if let Some(parent) = dest.parent() {
+        std::fs::create_dir_all(parent)
+            .map_err(|e| CliError::new(format!("mkdir {}: {e}", parent.display())))?;
+    }
+    std::fs::write(&dest, &bytes)
+        .map_err(|e| CliError::new(format!("write staged {}: {e}", dest.display())))?;
+    Ok(())
+}
+
+/// List every object key under the prefix, following continuation tokens so a
+/// backup with more than 1000 files (the page cap) lists fully.
+async fn list_keys(client: &Client, target: &S3Target) -> Result<Vec<String>, CliError> {
+    let list_prefix = if target.prefix.is_empty() {
+        String::new()
+    } else {
+        format!("{}/", target.prefix)
+    };
+
+    let mut keys = Vec::new();
+    let mut continuation: Option<String> = None;
+    loop {
+        let mut req = client.list_objects_v2().bucket(&target.bucket);
+        if !list_prefix.is_empty() {
+            req = req.prefix(&list_prefix);
+        }
+        if let Some(token) = &continuation {
+            req = req.continuation_token(token);
+        }
+        let resp = req
+            .send()
+            .await
+            .map_err(|e| CliError::new(format!("list_objects_v2: {}", s3_err(&e))))?;
+
+        for obj in resp.contents() {
+            if let Some(key) = obj.key() {
+                keys.push(key.to_string());
+            }
+        }
+        if resp.is_truncated().unwrap_or(false) {
+            continuation = resp.next_continuation_token().map(ToString::to_string);
+            if continuation.is_none() {
+                break;
+            }
+        } else {
+            break;
+        }
+    }
+    Ok(keys)
+}
+
+/// Render an AWS SDK error to a flat operator-readable string (the SDK's nested
+/// `Display` is verbose; `DisplayErrorContext` flattens the source chain).
+fn s3_err<E: std::error::Error + 'static>(e: &aws_sdk_s3::error::SdkError<E>) -> String {
+    format!("{}", aws_sdk_s3::error::DisplayErrorContext(e))
+}
+
+/// Recursively collect file paths relative to `root` (forward-slash form is
+/// applied later by `rel_to_slash`). Mirrors `backup::collect_files` but keeps
+/// the manifest IN (it is uploaded, last) — the only difference from the backup
+/// walk, which skips it.
+fn collect_rel_files(root: &Path, rel: &Path, out: &mut Vec<PathBuf>) -> std::io::Result<()> {
+    let dir = root.join(rel);
+    for entry in std::fs::read_dir(&dir)? {
+        let entry = entry?;
+        let child_rel = rel.join(entry.file_name());
+        let file_type = entry.file_type()?;
+        if file_type.is_dir() {
+            collect_rel_files(root, &child_rel, out)?;
+        } else if file_type.is_file() {
+            out.push(child_rel);
+        }
+    }
+    Ok(())
+}
+
+/// Render a relative path with forward slashes (portable object keys), matching
+/// the backup manifest's `rel_to_slash`.
+fn rel_to_slash(rel: &Path) -> String {
+    rel.components()
+        .map(|c| c.as_os_str().to_string_lossy().into_owned())
+        .collect::<Vec<_>>()
+        .join("/")
+}
+
+/// Turn a forward-slash relative key path back into a host-native `PathBuf`
+/// (the inverse of `rel_to_slash` for the local filesystem).
+fn rel_to_native(rel_slash: &str) -> PathBuf {
+    rel_slash.split('/').collect()
+}
+
+#[cfg(test)]
+#[allow(clippy::unwrap_used)]
+mod tests {
+    use super::*;
+
+    /// The pure key mapping: prefix + relative path -> object key, with no leading
+    /// slash and no doubled separator, for the prefix shapes an operator can pass
+    /// (`backups/node-A`, a trailing-slash form, the bucket-root empty prefix).
+    #[test]
+    fn object_key_joins_prefix_and_relative_path() {
+        assert_eq!(
+            object_key("backups/node-A", "wal/seg-1.seg"),
+            "backups/node-A/wal/seg-1.seg"
+        );
+        assert_eq!(
+            object_key("backups/node-A", "BACKUP_MANIFEST.json"),
+            "backups/node-A/BACKUP_MANIFEST.json"
+        );
+        // An empty prefix is a bucket-root export: the key is the bare rel path.
+        assert_eq!(object_key("", "wal/seg-1.seg"), "wal/seg-1.seg");
+        // Leading/trailing slashes on the prefix are normalized away (no `//`).
+        assert_eq!(
+            object_key("/backups/node-A/", "data.db"),
+            "backups/node-A/data.db"
+        );
+        // A stray leading slash on the rel path never produces `prefix//rel`.
+        assert_eq!(object_key("p", "/data.db"), "p/data.db");
+    }
+
+    /// `relative_from_key` is the exact inverse of `object_key`: it recovers the
+    /// backup-relative path from a listed key, and rejects a key outside the
+    /// prefix.
+    #[test]
+    fn relative_from_key_inverts_object_key() {
+        assert_eq!(
+            relative_from_key("backups/node-A", "backups/node-A/wal/seg-1.seg")
+                .as_deref(),
+            Some("wal/seg-1.seg")
+        );
+        // Empty prefix: the whole key is the relative path.
+        assert_eq!(
+            relative_from_key("", "BACKUP_MANIFEST.json").as_deref(),
+            Some("BACKUP_MANIFEST.json")
+        );
+        // A key NOT under the prefix is rejected (stray object the list returned).
+        assert_eq!(
+            relative_from_key("backups/node-A", "backups/node-B/data.db"),
+            None
+        );
+        // The prefix "directory marker" itself (key == prefix, no child) is None.
+        assert_eq!(relative_from_key("backups/node-A", "backups/node-A/"), None);
+    }
+
+    /// The full round-trip the export/import pair depends on: for every
+    /// (prefix, rel) an export produces a key, and importing that key recovers the
+    /// SAME rel — across the prefix shapes and a nested path.
+    #[test]
+    fn object_key_round_trips_through_relative_from_key() {
+        let cases = [
+            ("backups/node-A", "wal/seg-1.seg"),
+            ("backups/node-A", "BACKUP_MANIFEST.json"),
+            ("", "data.db"),
+            ("p", "a/b/c/deep.bin"),
+            ("dr/2026-06-17", "signals/keyspace/000123.fjall"),
+        ];
+        for (prefix, rel) in cases {
+            let key = object_key(prefix, rel);
+            let recovered = relative_from_key(prefix, &key);
+            assert_eq!(
+                recovered.as_deref(),
+                Some(rel),
+                "round-trip failed for prefix={prefix:?} rel={rel:?} (key={key:?})"
+            );
+        }
+    }
+
+    /// `normalize_prefix` strips slashes so `from_flags` stores a clean prefix.
+    #[test]
+    fn normalize_prefix_strips_surrounding_slashes() {
+        assert_eq!(normalize_prefix("/backups/node-A/"), "backups/node-A");
+        assert_eq!(normalize_prefix("backups/node-A"), "backups/node-A");
+        assert_eq!(normalize_prefix(""), "");
+        assert_eq!(normalize_prefix("/"), "");
+    }
+
+    /// `from_flags` is all-or-nothing on endpoint+bucket: a partial set is a usage
+    /// error (never a silent local-only fallback), and the no-flags case is `None`
+    /// (pure local backup). Assertions avoid `Result::unwrap` so the production
+    /// `CliError` need not derive `Debug`.
+    #[test]
+    fn from_flags_requires_endpoint_and_bucket_together() {
+        // No S3 flags: local-only, no target.
+        assert!(matches!(S3Target::from_flags(None, None, None), Ok(None)));
+        // Full set: a target with a normalized prefix.
+        let Ok(Some(t)) = S3Target::from_flags(
+            Some("https://acct.r2.cloudflarestorage.com"),
+            Some("dr"),
+            Some("/node-A/"),
+        ) else {
+            panic!("full flag set must build a target");
+        };
+        assert_eq!(t.bucket, "dr");
+        assert_eq!(t.prefix, "node-A");
+        // Empty prefix is allowed (bucket-root export).
+        let Ok(Some(t)) = S3Target::from_flags(Some("https://e"), Some("b"), None) else {
+            panic!("endpoint+bucket with no prefix must build a target");
+        };
+        assert_eq!(t.prefix, "");
+        // Partial sets are usage errors.
+        assert!(S3Target::from_flags(Some("https://e"), None, None).is_err());
+        assert!(S3Target::from_flags(None, Some("b"), None).is_err());
+        assert!(S3Target::from_flags(None, None, Some("p")).is_err());
+    }
+
+    /// `rel_to_native` is the inverse of `rel_to_slash` on a multi-segment path,
+    /// so a key listed from S3 lands at the right place in the staging tree.
+    #[test]
+    fn rel_to_native_round_trips_rel_to_slash() {
+        let p = PathBuf::from("wal").join("seg-1.seg");
+        let slash = rel_to_slash(&p);
+        assert_eq!(slash, "wal/seg-1.seg");
+        assert_eq!(rel_to_native(&slash), p);
+    }
+}

tidalctl/src/main.rs

@@ -77,6 +77,13 @@ struct CliArgs {
     out: Option<PathBuf>,
     /// Restore source (`restore --from <backup-dir>`).
     from: Option<PathBuf>,
+    /// Object-store (S3/R2) endpoint URL (`--s3-endpoint`), e.g.
+    /// `https://<account-id>.r2.cloudflarestorage.com`. Additive on backup/restore.
+    s3_endpoint: Option<String>,
+    /// Object-store bucket (`--s3-bucket`).
+    s3_bucket: Option<String>,
+    /// Object-store key prefix (`--s3-prefix`); empty ⇒ bucket-root.
+    s3_prefix: Option<String>,
 }
 
 enum Command {
@@ -143,6 +150,9 @@ fn parse_args(args: &[String]) -> Result<CliArgs, CliError> {
     let mut path: Option<PathBuf> = None;
     let mut out: Option<PathBuf> = None;
     let mut from: Option<PathBuf> = None;
+    let mut s3_endpoint: Option<String> = None;
+    let mut s3_bucket: Option<String> = None;
+    let mut s3_prefix: Option<String> = None;
     let mut pretty = false;
     let mut verify_only = false;
     let mut i = 2;
@@ -170,6 +180,27 @@ fn parse_args(args: &[String]) -> Result<CliArgs, CliError> {
                 }
                 from = Some(PathBuf::from(&args[i]));
             }
+            "--s3-endpoint" => {
+                i += 1;
+                if i >= args.len() {
+                    return Err(CliError::new("--s3-endpoint requires a value"));
+                }
+                s3_endpoint = Some(args[i].clone());
+            }
+            "--s3-bucket" => {
+                i += 1;
+                if i >= args.len() {
+                    return Err(CliError::new("--s3-bucket requires a value"));
+                }
+                s3_bucket = Some(args[i].clone());
+            }
+            "--s3-prefix" => {
+                i += 1;
+                if i >= args.len() {
+                    return Err(CliError::new("--s3-prefix requires a value"));
+                }
+                s3_prefix = Some(args[i].clone());
+            }
             "--pretty" => {
                 pretty = true;
             }
@@ -192,11 +223,15 @@ fn parse_args(args: &[String]) -> Result<CliArgs, CliError> {
         verify_only,
         out,
         from,
+        s3_endpoint,
+        s3_bucket,
+        s3_prefix,
     })
 }
 
 fn usage() -> String {
-    "Usage: tidalctl <command> --path <dir> [--out <dir>] [--from <dir>] [--pretty]\n\n\
+    "Usage: tidalctl <command> --path <dir> [--out <dir>] [--from <dir>] [--pretty]\n              \
+       [--s3-endpoint <url> --s3-bucket <b> [--s3-prefix <p>]]\n\n\
      Commands:\n  \
        status       Report WAL state, checkpoint, and directory layout\n  \
        paths        Report resolved directory paths and existence\n  \
@@ -208,6 +243,15 @@ fn usage() -> String {
      Backup/restore operate on a data dir AT REST (a stopped/drained node):\n  \
        tidalctl backup  --path /data/node --out /backups/node-A\n  \
        tidalctl restore --from /backups/node-A --path /data/node-new\n\n\
+     Object-store (S3/R2) export/import is ADDITIVE — pass --s3-endpoint +\n  \
+     --s3-bucket on backup to ALSO mirror the artifact to the bucket (manifest\n  \
+     uploaded last as the completion marker), or on restore to download the prefix\n  \
+     into a temp dir and run the verified restore on it. Creds from env\n  \
+     AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY:\n  \
+       AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... \\\n    \
+         tidalctl backup --path /data/node --out /backups/node-A \\\n      \
+           --s3-endpoint https://<account-id>.r2.cloudflarestorage.com \\\n      \
+           --s3-bucket tidaldb-dr --s3-prefix node-A\n\n\
      Exit codes: 0 = ok/empty, 1 = usage/internal error, 2 = degraded/unreadable\n  \
        (WAL, checkpoint, or a derived index exists but could not be read)."
         .to_string()
@@ -231,14 +275,25 @@ fn run(args: &[String]) -> Result<(String, i32), CliError> {
                 .out
                 .as_ref()
                 .ok_or_else(|| CliError::new("backup requires --out <dest-dir>"))?;
-            commands::backup::run_backup(&cli.path, out, cli.pretty)
+            // The S3 target (if any) is built from the three flags; all-or-nothing
+            // on endpoint+bucket so a half-set is a usage error, not a silent
+            // local-only backup the operator thinks went to R2.
+            let s3 = commands::s3::S3Target::from_flags(
+                cli.s3_endpoint.as_deref(),
+                cli.s3_bucket.as_deref(),
+                cli.s3_prefix.as_deref(),
+            )?;
+            commands::backup::run_backup(&cli.path, out, s3.as_ref(), cli.pretty)
         }
         Command::Restore => {
-            let from = cli
-                .from
-                .as_ref()
-                .ok_or_else(|| CliError::new("restore requires --from <backup-dir>"))?;
-            commands::backup::run_restore(&cli.path, from, cli.pretty)
+            // --from is required ONLY for a local restore; an S3 import stages the
+            // prefix into a temp dir and restores from that instead.
+            let s3 = commands::s3::S3Target::from_flags(
+                cli.s3_endpoint.as_deref(),
+                cli.s3_bucket.as_deref(),
+                cli.s3_prefix.as_deref(),
+            )?;
+            commands::backup::run_restore(&cli.path, cli.from.as_deref(), s3.as_ref(), cli.pretty)
         }
     }
 }