.woodpecker.yaml

@@ -3,32 +3,15 @@
 #   • event: push  → the m11p8 RELEASE GATE (rolling-upgrade tier-3 test) then a
 #     Kaniko image build. Deployment is MANUAL (kustomize, orchard9-k3sf ops repo).
 #   • event: cron  → the m11p9 NIGHTLY CONTINUOUS-CORRECTNESS run: the tier-3
-#     chaos suites (incl. the disk-full / slow-fsync / asymmetric-partition fault
+#     chaos suites (incl. the new disk-full / slow-fsync / asymmetric-partition
-#     classes) with elevated kill-points, plus the security/ops owner-tests (mTLS,
+#     fault classes) with elevated kill-points, then a tidal-stress soak with
-#     backup/restore round-trip, gap-free WAL archival). A nightly failure flags a
+#     PASS/FAIL regression gates on p99 + error rate. A nightly failure flags a
-#     correctness regression for that day.
+#     correctness or performance regression for that day.
 #
 # Per-step `when:` routes each step to its event; the workflow-level `when` admits
-# both. The cron pipeline requires a cron named "nightly" in the Woodpecker repo
+# both. Configure a cron named "nightly" in the Woodpecker repo settings to fire
-# settings (the same `tidal-server` binary serves standalone AND multi-process
+# the cron pipeline (the same `tidal-server` binary serves standalone AND
-# `cluster --region`, so one image covers both deployments).
+# multi-process `cluster --region`, so one image covers both deployments).
-#
-# ── Honesty note, 2026-08-30 ────────────────────────────────────────────────────
-# For 216 days this file described a nightly correctness programme that had NEVER
-# RUN: the "nightly" cron was never created, so four pipelines existed in total
-# and all were push events. Tier-3 chaos, the fault classes, the soak's p99 and
-# error-rate gates, mTLS and the PITR test produced exactly ZERO signal, while
-# reading to anyone opening this file like rigorous standing coverage. A gate
-# nobody runs is worse than no gate, because it makes the project look covered.
-#
-# The nightly SOAK step was REMOVED rather than scheduled. It is not a capacity
-# quibble — it is unfalsifiable here. Measured free capacity on the best node is
-# 1700m CPU (agent-1; server-1/2 have 435m/550m), and the step drove 1000 rps for
-# 600s while gating on p99 ≤ 250ms. On that hardware the gate fails from CPU
-# starvation, not from a regression, so scheduling it would manufacture a nightly
-# false alarm — the same fake-coverage defect inverted. Its command list now lives
-# in docs/runbooks/nightly-soak.md as a pre-release step, run where the capacity
-# to make its numbers mean something actually exists.
 when:
   branch: main
   event: [push, cron]
@@ -45,30 +28,6 @@ steps:
     image: rust:1-bookworm
     when:
       event: push
-    # ── Resource shape: HEAVY (declared once here, aliased by later steps) ─────
-    # This step spawns three real tidal-server processes, each with its own WAL,
-    # HNSW index, gRPC transport and tokio runtime. Under the namespace default
-    # (`ci-build-bounds`: limits 1500m/2Gi, requests 50m/128Mi) a prior pipeline
-    # pod was OOMKilled outright at 2Gi, and convergence starved past 12 min on
-    # 1.5 CPU against 21s locally.
-    #
-    # CALIBRATED 2026-08-30 against real node free capacity, NOT against the
-    # LimitRange max. `ci-build-bounds` permits up to cpu 3 / memory 6Gi, but
-    # permission is not capacity — measured free requests were agent-1 1700m/4193Mi,
-    # server-1 435m/2309Mi, server-2 550m/2257Mi. A `requests: cpu: "2"` step pod
-    # (the value this file's roadmap originally specified) fits on NO node and
-    # would sit Pending forever, which is a worse failure than being slow.
-    #
-    # So: requests are sized to actually schedule (fits agent-1 with ~700m CPU and
-    # ~2.1Gi memory of slack), while limits take the full LimitRange max. Limits do
-    # not affect scheduling, and the 6Gi limit is what eliminates the OOMKill; the
-    # 3 CPU limit doubles the old burst ceiling. Raising the LimitRange itself is
-    # explicitly NOT the fix here.
-    backend_options: &resources-heavy
-      kubernetes:
-        resources:
-          requests: { cpu: "1", memory: 2Gi }
-          limits: { cpu: "3", memory: 6Gi }
     # Budget headroom, matching the nightly steps below. The defaults are 60s boot
     # / 30s convergence (support/multiproc.rs:54,62), tuned for a developer
     # machine; this step spawns three real OS processes, drives a graceful
@@ -96,15 +55,6 @@ steps:
     image: woodpeckerci/plugin-kaniko
     when:
       event: push
-    # ── Resource shape: LIGHT (declared once here, aliased by later steps) ─────
-    # Kaniko build: single process, but it OOMKilled at the 2Gi namespace default.
-    # A build does not need the heavy shape's CPU floor, so it is sized separately
-    # to leave headroom for anything co-scheduled on the same node.
-    backend_options: &resources-light
-      kubernetes:
-        resources:
-          requests: { cpu: "500m", memory: 1Gi }
-          limits: { cpu: "2", memory: 4Gi }
     settings:
       repo: tidal/server
       dockerfile: docker/standalone/Dockerfile
@@ -136,19 +86,11 @@ steps:
     when:
       event: cron
       cron: nightly
-    # Same three-process shape as the release gate, so the same resource shape.
-    backend_options: *resources-heavy
     environment:
       TIDAL_QUORUM_KILLPOINTS: "25"
       TIDAL_ELECTION_KILLPOINTS: "15"
-      # Raised from 120/90 to match the release gate. These budgets were TIGHTER
+      TIDAL_TEST_BOOT_BUDGET_SECS: "120"
-      # than the gate's on the same constrained runner while running the same
+      TIDAL_TEST_CONVERGENCE_BUDGET_SECS: "90"
-      # three-process suites, so the step would have failed nightly for the
-      # identical budget-sensitivity reason measured on 2026-08-30 — not for a
-      # correctness regression, which is the only thing this step should ever
-      # report.
-      TIDAL_TEST_BOOT_BUDGET_SECS: "300"
-      TIDAL_TEST_CONVERGENCE_BUDGET_SECS: "180"
     commands:
       - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang
       # New m11p9 fault classes first (disk-full, slow-fsync, asymmetric partition).
@@ -162,6 +104,63 @@ steps:
       - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_lifecycle -- --nocapture --test-threads 1
       - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_runbook -- --nocapture --test-threads 1
 
+  # ── Nightly soak (cron) ─────────────────────────────────────────────────────
+  # A tidal-stress soak with PASS/FAIL regression gates (m11p9). By default it
+  # boots a local standalone server and soaks it for a bounded window, gating on
+  # error rate and p99 so a perf regression fails the step and shows on the trend
+  # line (the JSON summary is the archived artifact). Point TIDAL_SOAK_TARGET at
+  # the live Ref-A cluster and raise TIDAL_SOAK_SECS to 3600 for the GA-bar 1-hour
+  # 100k-DAU soak (the standing k3s-access caveat applies — same as m11p1–p3).
+  nightly-soak:
+    image: rust:1-bookworm
+    when:
+      event: cron
+      cron: nightly
+    environment:
+      TIDAL_SOAK_RPS: "1000"
+      TIDAL_SOAK_SECS: "600"
+      TIDAL_SOAK_MAX_P99_MS: "250"
+      TIDAL_SOAK_MAX_ERROR_PCT: "1"
+      # Soak target port — an UNCLAIMED slot in the project's reserved dev band
+      # (59520-59529): 59520=site, 59521=iknowyou, so the soak uses 59526. One
+      # source of truth for the literal (referenced by --listen AND the TARGET).
+      TIDAL_SOAK_PORT: "59526"
+    commands:
+      - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang curl
+      - cargo build -p tidal-server -p tidal-stress
+      # Boot a standalone target unless TIDAL_SOAK_TARGET points elsewhere (Ref-A).
+      # NOTE the `$${VAR}` escaping: Woodpecker substitutes a bare `${VAR}` in a
+      # command BEFORE the shell runs, and step `environment:` vars are NOT in that
+      # preprocessor namespace (they would blank to ""). `$${VAR}` passes a literal
+      # `${VAR}` to the container shell, which expands it from the runtime env —
+      # so the soak's ramp and gate thresholds are actually populated. Bare shell
+      # locals (`$PORT`, `$TARGET`, `$SRV`) are never touched by the preprocessor.
+      - |
+        PORT="$${TIDAL_SOAK_PORT}"
+        TARGET="$${TIDAL_SOAK_TARGET:-http://127.0.0.1:$PORT}"
+        if [ -z "$TIDAL_SOAK_TARGET" ]; then
+          mkdir -p /tmp/soak-data
+          ./target/debug/tidal-server standalone --listen "127.0.0.1:$PORT" \
+            --schema tidal-server/config/default-schema.yaml --data-dir /tmp/soak-data &
+          SRV=$!
+          # Reap the background server + scratch dir on ANY exit (success, gate
+          # failure, or the boot-failure exit below) so the step is idempotent.
+          trap 'kill "$SRV" 2>/dev/null; rm -rf /tmp/soak-data' EXIT
+          up=0
+          for i in $(seq 1 100); do
+            curl -sf "$TARGET/health/startup" >/dev/null 2>&1 && { up=1; break; } || sleep 0.3
+          done
+          # Fail FAST and unambiguously on a boot failure (bad schema, port bound)
+          # rather than running the soak against a dead target and mislabeling it
+          # as an error-rate regression on the trend line.
+          if [ "$up" != "1" ]; then echo "soak target failed to start at $TARGET"; exit 1; fi
+        fi
+        ./target/debug/tidal-stress --target "$TARGET" \
+          --ramp "$${TIDAL_SOAK_RPS}:$${TIDAL_SOAK_SECS}" --corpus 5000 --mix peach \
+          --json-summary soak-summary.json \
+          --max-error-pct "$${TIDAL_SOAK_MAX_ERROR_PCT}" --max-p99-ms "$${TIDAL_SOAK_MAX_P99_MS}" --fail-on-knee
+      - cat soak-summary.json
+
   # ── Nightly security + ops correctness (cron) ───────────────────────────────
   # The G-Sec and G-Op owner-tests run nightly too (not just on-demand): mTLS /
   # foreign-pod rejection / cert rotation (tidal-net mtls + cluster_security), the
@@ -172,8 +171,6 @@ steps:
     when:
       event: cron
       cron: nightly
-    # In-process and fast; the light shape is sufficient.
-    backend_options: *resources-light
     commands:
       - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang
       - cargo test -p tidal-net --test mtls -- --test-threads 1

API.md

@@ -1134,37 +1134,6 @@ The cluster node also exposes the membership/recovery verbs `POST /cluster/join`
 `/cluster/catchup`, `/cluster/reseed`, and `/cluster/reconcile`, plus the sharded
 data surface `POST /sharded/{items,embeddings,signals}` and `GET /sharded/{feed,search}`.
 
-#### `/sharded/*` writes are SINGLE-COPY and require an explicit opt-in
-
-`POST /sharded/{items,embeddings,signals}` hash-partitions by `entity_id` and
-applies the write to the **owning region's local store with no WAL append**. It
-does not ride the leader relay, so the data has **redundancy 1 regardless of the
-replication factor** — a cluster running RF3 with `ack: quorum` does *not*
-replicate these writes. That is by design: it buys parallel write throughput
-across shard owners (measured 3,669 signals/s vs ~90/s on the replicated path).
-
-Because the endpoint previously answered `201`/`204` with nothing saying so, it
-now **requires `x-tidal-ack: local`** and returns `400` without it, with a body
-naming the header and the replicating alternative:
-
-```bash
-# Rejected — 400, single-copy durability was never acknowledged.
-curl -X POST "$BASE/sharded/items" -H "authorization: Bearer $KEY" \
-  -d '{"entity_id": 7, "metadata": {"title": "..."}}'
-
-# Accepted — 201, and the caller has stated it accepts redundancy 1.
-curl -X POST "$BASE/sharded/items" -H "authorization: Bearer $KEY" \
-  -H 'x-tidal-ack: local' \
-  -d '{"entity_id": 7, "metadata": {"title": "..."}}'
-```
-
-For a **replicated** write use `POST /items`, `/embeddings`, or `/signals` — those
-ride the leader WAL relay and honor `x-tidal-ack: leader|quorum`. `local` is
-rejected there (`x-tidal-ack must be "leader" or "quorum"`), because those routes
-always replicate.
-
-`GET /sharded/{feed,search}` are **reads** and are unaffected — no header needed.
-
 ### Cluster-Node-Only Data Endpoints
 
 | Method | Path | Purpose |

CHANGELOG.md

@@ -14,85 +14,8 @@ change to a public API or a persisted format ships with a documented migration
 path in this file. This supersedes the `0.1.0` "no stability guarantees" note
 below.
 
-### Breaking
-
-**`/sharded/*` writes now require `x-tidal-ack: local` (wire-visible)**
-
-`POST /sharded/{items,embeddings,signals}` hash-partitions by `entity_id` and
-applies the write to the **owning region's local store with no WAL append**. It
-does not ride the leader relay, so the data has **redundancy 1 regardless of the
-replication factor** — an RF3 cluster configured `ack: quorum` does not replicate
-it. That is intentional (parallel write throughput across shard owners: 3,669
-signals/s measured, vs ~90/s on the replicated path), but the endpoint answered
-`201`/`204` with nothing at the call site, in the response, or in the OpenAPI
-saying so. An operator probed with `/sharded/embeddings`, found each embedding on
-exactly one of three nodes, and filed a durability incident that had to be
-retracted.
-
-The three write routes now return **400** unless the request carries
-`x-tidal-ack: local`, with a body naming the header and pointing at the
-replicating alternative. The existing `x-tidal-ack` header is reused — no second
-durability knob — and `local` remains rejected on `/items` / `/embeddings` /
-`/signals`, which always replicate. `GET /sharded/{feed,search}` are reads and are
-**not** affected. Nothing was rerouted: a `/sharded/*` write that opts in is
-byte-for-byte as before, and the surface never silently replicates.
-
-*Migration:* add `-H 'x-tidal-ack: local'` to acknowledge single-copy durability,
-or move the write to `POST /items` / `/embeddings` / `/signals` for a replicated
-one. Live traffic showed **0 requests** to `/sharded/*` over ~2h of production
-load, and the two in-repo callers were migrated with this change.
-
-### Changed
-
-**`/cluster/status` `regions[]` frontiers may now be `null` (wire-visible)**
-
-`applied_events` and `lag_events` are now `Option<u64>` — JSON `null` when the
-aggregating node has no frontier report for a region. Previously an unreachable
-peer was reported as `applied_events: 0, lag_events: <leader high-water-mark>`;
-because lag is `hwm − applied`, a 500 ms probe timeout was rendered as the
-leader's entire history as a deficit. Measured live: every node reported both
-peers `partitioned: true, reachable: false, lag_events: 13322258` while the **same
-response's** `shards[]` array showed all three replicas converged at identical
-frontiers with lag 0. An operator reading the obvious field concludes the cluster
-is dead.
-
-The root cause was a type that could not express "unknown", so the fix is at the
-type: `null` where the value is not known, and no value is derived from one that
-is not. That covers three former fabrications — the unreachable peer, a leader
-with no ack mark for a peer, and a region whose lag was computed against a leader
-high-water-mark the aggregator never learned. `reachable` and `partitioned` are
-unchanged and still distinguish a genuinely partitioned peer from an unknown one;
-`shards[]` is unchanged (it was the trustworthy half throughout).
-
-*Migration:* clients typed against `applied_events` / `lag_events` must accept
-`null`. `tidalctl` renders it as `NO REPORT` / `?` and no longer treats it as a
-deficit, so `tidalctl cluster-status && deploy` is usable as a gate again — a
-known deficit, an unreachable region, a `partitioned: true` region, a pending
-reseed, or a local shard lag still exit 2.
-
 ### Fixed
 
-**`/feed` and `/search` `rank` is now dense and ascending (wire-visible)**
-
-Under full placement — the production shape, every shard group on every node —
-`missing_groups()` is empty and both handlers return straight out of
-`scatter_merge`, which sorted and truncated but never re-stamped `rank`. Each
-hosted group ranks its own slice `1..k` locally, the slices were concatenated and
-score-sorted, and the per-group counters reached the wire as-is: live
-`/search?query=verification` returned ranks **`1, 1, 2`**. The sibling
-`merge_cross_shard` already re-stamped for the partial-placement path and
-documented exactly why; full placement returned before reaching it.
-
-`scatter_merge` now takes the same `set_rank` closure and stamps after `truncate`,
-so there is one rank-stamping mechanism rather than two. **Ordering is unchanged**
-— the stamp is a renumbering, not a re-sort, and scores were always correctly
-descending. `/vector_search` carries no rank field and passes a no-op. The `S=1`
-fast path still returns the engine's own already-dense ranks; both shapes are now
-asserted.
-
-*Migration:* a client that deduplicated or keyed on `rank` while working around
-the duplicates can drop the workaround. `rank` is `1..n` over the returned page.
-
 **`vector_search` distances now honor the documented `[0.0, 4.0]` contract (wire-visible)**
 
 The read path passed the caller's raw query vector straight to the index while the

demo/capability-inventory.md

@@ -26,38 +26,11 @@ touch several.
 pods started 2026-08-23 05:32–05:41 UTC, namespace `tidaldb-cluster`,
 repo commit recorded per-run in `E2E_BUILD_REVISION`.
 
-**Environment gate discovered during inventory — CORRECTED 2026-08-31.** As
+**Environment gate discovered during inventory.** The running image predates the
-written on 2026-08-23 this read: *the running image predates the observability
+observability commit (`4766f56`), so HTTP request metrics and structured logs are
-commit (`4766f56`), so HTTP request metrics and structured logs are absent by
+absent by construction, while the operator/data credential split from `388e445`
-construction.* That was true of the image this inventory was taken against and
+*is* live. The runbook asserted all three were pending; that was stale. See
-**is false now.** Both capabilities from `4766f56` are in the running image:
+`BUG-001`.
-
-| rev | image | live from | carries `4766f56`? |
-|---|---|---|---|
-| 29 | `m12-admin-gate-20260823` | 2026-08-23T05:32Z | no — the image this inventory was taken against |
-| 30 | `m12-vector-grow-20260830` | 2026-08-30T17:38Z | **yes** — `7c1c80d` descends from `4766f56` |
-| 31 | `m12-vsc-20260830` | 2026-08-30T19:59Z | **yes** — `8aa1fbb` |
-
-```bash
-kubectl -n tidaldb-cluster get controllerrevision -l app.kubernetes.io/name=tidaldb \
-  -o custom-columns=REV:.revision,WHEN:.metadata.creationTimestamp,IMAGE:'.data.spec.template.spec.containers[0].image'
-git merge-base --is-ancestor 4766f56 8aa1fbb   # succeeds
-```
-
-Current state of the three:
-
-- **Operator/data credential split** (`388e445`) — live since 2026-08-23, as
-  originally recorded.
-- **HTTP request metrics** — **live.** Measured 2026-08-31T02:19Z:
-  `tidaldb_http_requests_total` present on all three pods, 19/24/18 series
-  against `tidaldb_*` baselines of 821/878/812.
-- **Structured logs** — in the image since rev 30. The StatefulSet did not ask
-  for them: `JSON_LOGS` was added to `k8s/cluster/statefulset.yaml` on
-  2026-08-31 and an env change takes effect only on the next pod restart. The
-  restart is the sole outstanding item; no code and no image roll is needed.
-
-See `BUG-001` for the original finding and `docs/runbooks/deploy-verification.md`
-§9.1/§9.3 for live state.
 
 ## Inventory
 
@@ -341,28 +314,21 @@ See `BUG-001` for the original finding and `docs/runbooks/deploy-verification.md
 - **Coverage.** `existing-green`
 - **Evidence.** `tests/e2e/features/09-operator-authority.spec.ts`
 
-### CAP-015 — Observability features are asserted at their real current state
+### CAP-015 — Inert observability features are inert for a known reason
-
-> **Corrected 2026-08-31.** This entry described both features as "absent from
-> the running image". Half of that is now false and the other half was never an
-> image problem. Title and bullets updated; the spec itself was already inverted
-> for the metrics half on 2026-08-30.
 
 - **Area.** `tidaldb_http_requests_total`; `JSON_LOGS`.
-- **Business purpose.** A tripwire in the honest direction. Pinning what is
+- **Business purpose.** A tripwire in the honest direction. These are committed
-  *currently* true means the day reality moves, the test fails and names the
+  and unit-tested but absent from the running image. Asserting their *current
-  runbook section to update — instead of the runbook quietly rotting, which is
+  absence* means the day someone rolls the observability image, this test fails
-  precisely what happened to §9.2, then to §9.1, then to §9.3.
+  and tells them to update the runbook — instead of the runbook quietly rotting,
+  which is precisely what happened to §9.2.
 - **Personas.** Operator.
-- **Primary workflow.** Scrape each pod's `:9091` with a generous timeout and
+- **Primary workflow.** Scrape each pod's `:9091` with a generous timeout;
-  assert `tidaldb_http_*` series are **present** (inverted 2026-08-30 when the
+  assert zero `tidaldb_http_*` series and record it as expected-for-this-image.
-  metrics went live). Assert `JSON_LOGS` is **absent from the live StatefulSet**
+  Assert `JSON_LOGS` is absent from the StatefulSet env.
-  — still true, and still the correct assertion until the pods restart with the
-  env var now in the manifest, at which point it inverts too.
 - **Edge cases.** A short scrape timeout returns zero lines for *every* metric
   and would make this test pass for the wrong reason — so it also asserts the
-  ordinary `tidaldb_` series are present (>100), proving the scrape actually
+  ordinary `tidaldb_` series are present, proving the scrape actually worked.
-  worked.
 - **Permission boundary.** None.
 - **Dependencies.** vmagent exec; the StatefulSet spec.
 - **Observability.** Both counts, so "absent" is distinguishable from "unscraped".
@@ -472,36 +438,26 @@ See `BUG-001` for the original finding and `docs/runbooks/deploy-verification.md
 - **Evidence.** `tests/e2e/features/10-ranking-semantics.spec.ts` — 10/10 exact
   match on all four probes; self-distances 0.0148–0.0197 against a 0.05 tolerance.
 
-### CAP-020 — Rank is a dense sequence on standalone AND on the cluster
+### CAP-020 — Rank is a dense sequence on standalone, and duplicated on the cluster
 
 - **Area.** `rank` on `/feed` and `/search`; `scatter_merge`.
 - **Business purpose.** `rank` is the field a client paginates and displays on.
   Duplicated ranks silently corrupt any consumer that keys on it.
 - **Personas.** Application developer; operator.
-- **Primary workflow.** Assert `rank` is exactly `1..n` on a standalone node, on
+- **Primary workflow.** Assert `rank` is exactly `1..n` on a standalone node.
-  both cluster merge shapes, and on the deployed cluster.
+  Separately, assert the deployed cluster still returns duplicates, with the
+  root cause cited and a "delete this tripwire" instruction in the message.
 - **Edge cases.** A limit of 1–2 can be answered from one shard group and would
-  show no duplicate; the cluster checks use 12. They also assert scores are still
+  show no duplicate; the cluster check uses 12. The cluster check also asserts
-  correctly ordered — the stamp is a renumbering, never a re-sort, so an ordering
+  scores are still correctly ordered — that is what localises the fault to the
-  change would be a different and worse defect. The single-group `[only]` fast
+  missing rank stamp rather than to the merge's sort.
-  path bypasses the merge entirely and is asserted separately rather than assumed.
 - **Permission boundary.** Read-only against the cluster; nothing is written.
 - **Dependencies.** CAP-016's fixture (standalone half); the public ingress and
-  data bearer (deployed-cluster half).
+  data bearer (cluster half).
 - **Observability.** Both rank arrays and both score arrays are attached.
 - **Coverage.** `new-test`
-- **Evidence.** `tests/e2e/features/10-ranking-semantics.spec.ts` (engine, dense
+- **Evidence.** `tests/e2e/features/10-ranking-semantics.spec.ts` (dense 1..60);
-  1..60); `tidal-server/tests/cluster_sharding.rs`
+  `tests/e2e/features/11-ranking-integrity.spec.ts` (cluster `[1,1,1,2,2,3,4,3,4,5,6,5]`)
-  `mp_ranked_reads_stamp_dense_rank_on_both_merge_shapes` (dense 1..12 on `/feed`
-  and `/search`, all 3 nodes, at 3 groups AND 1 group);
-  `tests/e2e/features/11-ranking-integrity.spec.ts` (deployed cluster).
-
-  **History.** This capability was first recorded as *"dense on standalone, and
-  duplicated on the cluster"*: under full placement `/search` returned
-  `[1,1,1,2,2,3,4,3,4,5,6,5]` because `scatter_merge` sorted and truncated the
-  concatenated per-group slices without re-stamping `rank`. `scatter_merge` now
-  takes the same `set_rank` closure as `merge_cross_shard`, and the tripwires that
-  pinned the defect were replaced by the positive assertions above.
 
 ## Intentionally excluded
 
@@ -510,7 +466,7 @@ See `BUG-001` for the original finding and `docs/runbooks/deploy-verification.md
 | Failover under induced node loss | Deliberately destructive against the production cluster this suite verifies. Killing a voter to watch election is a game-day exercise, not a post-deploy check. | `tidal-server` cluster e2e suite (`cluster_runbook`, `cluster_reseed`) exercises election and reseed against ephemeral multi-process clusters. | Operator; revisit when a staging cluster exists. |
 | Restore from the Velero backup | Restoring over live data is unacceptable; a restore drill needs an isolated target namespace. | `docs/runbooks/disaster-recovery.md` manual drill; `restore-canary-*` Backup objects prove the restore path independently. | Operator; revisit at the next DR drill. |
 | Read/write throughput and latency SLA | Load generation against production would distort the very metrics the dashboard checks assert on. | `docs/ops/stress-test-*.md`, `tidal-stress`; nightly soak. | Operator; not a deploy gate. |
-| VictoriaLogs LogsQL query surface | **Justification corrected 2026-08-31 — the original was factually wrong.** It claimed `/select/logsql/query` returns "unsupported path requested" on this build; that path answers normally (`_time:15m AND unit:tidaldb-cluster \| stats count() n` → `{"n":"248"}`). The real reason to defer is that tidalDB's lines are not classified yet: pre-`JSON_LOGS` every line is stamped `level:info`, so a query surface test would assert a shape that is about to change. | `kubectl logs` filtering at source (CAP-011); `docs/ops/observability.md` §2 documents the working queries and their pre/post-roll meaning. | Operator; revisit immediately after the `JSON_LOGS` restart, not "when the image is rolled" — the image is already rolled. |
+| VictoriaLogs LogsQL query surface | `/select/logsql/query` returns "unsupported path requested" on this build, and the deployed image emits plain text so `level:error` cannot match anyway. | `kubectl logs` filtering at source (CAP-011), documented as the current method. | Operator; revisit when the observability image is rolled. |
 | Traefik rate-limit thresholds (200/400) | Proving the limit requires deliberately flooding a production ingress shared with 35 other services. | Middleware config asserted declaratively in `k8s/cluster/ingress.yaml`. | Operator; revisit with a dedicated test host. |
 | The `tidaldb` namespace standalone Deployment | Live `1/1` but nothing routes to it; the fleet record calls it superseded. Verifying it would legitimise a surface that should be retired. | Noted as open drift in the runbook's closing section. | Operator; revisit by retiring it. |
 

docs/ops/observability.md

@@ -5,7 +5,7 @@ Four surfaces, in the order you reach for them during an incident.
 | Question | Surface |
 |---|---|
 | How much traffic, and how much of it is failing? | Grafana → **tidalDB — usage, errors, and cluster health** |
-| What exactly failed, for which request? | VictoriaLogs, `level:error`, correlate on `request_id` — **only after the `JSON_LOGS` roll; see §2** |
+| What exactly failed, for which request? | VictoriaLogs, `level:error`, correlate on `request_id` |
 | Is the cluster converged right now? | `tidalctl cluster-status` / `tidalctl watch` |
 | What does a query actually return? | `tidalctl search` / `tidalctl feed` |
 
@@ -93,108 +93,14 @@ Tunables: `TIDAL_SERVER_LOG` (env-filter, e.g.
 `tidal_server=debug,tantivy=warn,info` to quiet a noisy dependency),
 `TIDAL_SERVICE_NAME`, `TIDAL_ENV`.
 
-`TIDAL_SERVICE_NAME` is not cosmetic here. `service` is one of the collector's
+```bash
-four `_stream_fields`, so its value is index identity. Plain-text lines carry no
-`service` and Vector substitutes the container name, `tidaldb`; the JSON emitter
-sets its own, defaulting to `tidal-server`. The cluster StatefulSet therefore
-pins `TIDAL_SERVICE_NAME=tidaldb` so the stream does not rename itself the moment
-the format changes. `TIDAL_ENV` is unset in the cluster, so `env` is absent from
-its lines rather than guessed — do not filter on it.
-
-### Querying the log store
-
-Two things silently break a LogsQL query against tidalDB, and both fail as an
-empty result — which reads exactly like a healthy cluster.
-
-**1. The namespace field is `unit`, not `kubernetes.pod_namespace`.** Vector's
-`normalize` transform does `del(.kubernetes)` and writes the namespace to `unit`
-(and the node to `host`, the pod and container to `k8s_pod` / `k8s_container`).
-A query keyed on the Kubernetes field matches nothing, forever. This document
-shipped that broken selector; measured against the live store on
-2026-08-31T02:24Z, same 15-minute window, same cluster:
-
-```
-_time:15m AND kubernetes.pod_namespace:tidaldb-cluster | stats count() n
-  -> {"n":"0"}        # the selector is wrong, not the cluster
-
-_time:15m AND unit:tidaldb-cluster | stats count() n
-  -> {"n":"248"}      # same window, correct field
-```
-
-**So always run the unfiltered count first.** Zero total lines means a broken
-selector. Only once that number is non-zero does a zero *error* count mean
-anything.
-
-**2. `level`, `request_id`, `target` and every other structured field exist only
-once `JSON_LOGS` is on.** Until the pods restart with it, Vector cannot parse the
-line, keeps the text as `msg`, and stamps `level = "info"` — so `level:error`
-matches nothing no matter how healthy or unhealthy the cluster is, and every WARN
-is filed as info. Pre-roll reading, same window:
-
-```
-_time:15m AND unit:tidaldb-cluster | stats by (level) count() n
-  -> {"level":"info","n":"248"}     # ONE bucket: nothing is being classified
-
-_time:15m AND unit:tidaldb-cluster AND level:error | stats count() n
-  -> {"n":"0"}                      # meaningless pre-roll, not reassuring
-
-_time:1h AND unit:tidaldb-cluster AND request_id:* | stats count() n
-  -> {"n":"0"}                      # the field does not exist yet
-```
-
-(The counts move — `_time:15m` is a sliding window, and two runs a minute apart
-returned 247 and 248. The load-bearing observation is the *number of buckets*,
-not the number of lines: one bucket means the classifier never ran.)
-
-The store itself is fine — `level:error` already works for services that emit
-JSON: `_time:15m AND level:error | stats by (service) count() n` returns
-`{"service":"relay","n":"189"}`, `{"service":"external-secrets","n":"2"}`.
-tidalDB was simply not speaking the format. Deploy status is tracked in
-`docs/runbooks/deploy-verification.md` §9.3.
-
-**Post-roll, these are the queries.** Each pairs with its own discriminator so a
-zero is readable:
-
-```
-# is anything arriving at all?  (run this first, every time)
-_time:15m AND unit:tidaldb-cluster | stats count() n
-
-# how is it being classified?  more than one bucket == JSON_LOGS is live
-_time:15m AND unit:tidaldb-cluster | stats by (level) count() n
-
 # errors, last 15 minutes
-_time:15m AND unit:tidaldb-cluster AND level:error
+_time:15m AND kubernetes.pod_namespace:tidaldb-cluster AND level:error
 
 # one request end to end
-_time:1h AND unit:tidaldb-cluster AND request_id:418
+_time:1h AND request_id:"418"
-
-# one pod
-_time:15m AND unit:tidaldb-cluster AND k8s_pod:tidaldb-0 AND level:error
-
-# search the message text — the field is _msg, NOT msg (see below)
-_time:15m AND unit:tidaldb-cluster AND _msg:rejected AND _msg:request
 ```
 
-Run them with:
-
-```bash
-kubectl -n observability exec deploy/vmagent -- sh -c \
-  'wget -qO- --timeout=25 --post-data="query=<QUERY>" \
-   http://victoria-logs:9428/select/logsql/query'
-```
-
-**The message field is `_msg`, not `msg`.** Vector's sink declares
-`_msg_field: msg`, so VictoriaLogs renames it on ingest. `msg:rejected` parses
-fine and matches nothing, forever — the same silent-zero failure as the namespace
-field above. Verified 2026-08-31: `_msg:compaction` returns `{"n":"229"}` while
-`msg:compaction` in the same position returns empty.
-
-**Keep double quotes out of the query.** The runner above already wraps the query
-in a double-quoted `--post-data="query=…"`, so a quoted phrase like
-`_msg:"rejected request"` terminates the shell quoting and `wget` dies with
-`bad address 'request | stats…'`. Bare tokens (`request_id:418`) and ANDed word
-filters (`_msg:rejected AND _msg:request`) need no quotes and cannot break.
-
 ---
 
 ## 3. Live debugging with `tidalctl`

docs/planning/ROADMAP.md

@@ -38,7 +38,7 @@ A single embeddable database can replace the 6-system content ranking stack by t
 | M9  | Community Sync & Revocation | Local embeddable profiles can opt into community personalization and safely leave/purge contributions | Community personalization, federated taste graphs, shared feeds — ✅ COMPLETE (2026-06-06) |
 | M10 | Governance & Agent Rights | Community rules and agent-scoped permissions control what signals influence ranking | User-owned AI personalization at scale, policy-compliant agents — ✅ COMPLETE (2026-06-06) |
 | M11 | Enterprise-Grade Cluster | The multi-process cluster becomes a system of record: fast unified log, quorum-durable, self-healing, horizontally scalable, secured, observable, chaos-tested | Paying-customer cluster deployments — **✅ COMPLETE (all nine phases, 2026-06-13)**: m11p1 ✅ + m11p2 ✅ + m11p3 ✅ + m11p4 ✅ (automatic failover: Raft-style election + term fencing + divergence quarantine — closes **G5**; v0.9 "Credible HA" wave complete 2026-06-12) + m11p5 ✅ (membership/discovery/elasticity: DNS peers, snapshot+stream reseed, kind-4 membership records, seed join, `k8s/cluster/` — 2026-06-12) + m11p7 ✅ (security hardening: gRPC mTLS default + zero-drop cert rotation, inter-node HTTP TLS + per-node signed tokens, admin audit log, per-principal rate limit — 2026-06-13) + m11p8 ✅ (observability + operations: completed `tidaldb_cluster_*` set incl. breaker/forwards/self-heal on the per-node `/metrics` listener + Grafana cluster row + alert group, request-id/tracing across hops, truthful status, self-driving heal, WAL PITR archival + `tidalctl` backup/restore, rolling-upgrade version handshake + Woodpecker release gate — closes **G-O** — 2026-06-13) + m11p9 ✅ (continuous correctness: new fault classes (disk-full / slow-fsync / asymmetric partition) as REAL faults behind a production-compiled-out `fault-injection` feature, first-class invariant checkers, `tidal-stress` soak with regression gates + JSON summary, a Woodpecker cron nightly chaos+soak pipeline, and a guarantee→test traceability matrix — closes the **G-C** apparatus; the 30-day-green calendar accrues nightly — 2026-06-13) + m11p6 ✅ (sharding × replication + rebalancing: ONE hash-routed + replicated write surface across S shard groups each at RF with its own elected leader; per-group rebalancing verbs (`/cluster/shards/{id}/transfer` + `/replicas`) and a `?shard=` admin selector; tier-3 3×3 kill-node exit gate — only the dead node's leaderships move, reads never stop, zero acked loss; ≥5,000/s + 2.5× scaling Ref-A-pending — 2026-06-13). **ALL NINE PHASES COMPLETE**; the v1.0 bar now waits only on the 30-day-green nightly calendar (m11p9) and the standing Ref-A/k3s throughput re-runs |
-| M12 | Vector Retrieval at Production Shape | Ranking-at-scale steers on measured numbers, not absent ones: ANN candidate generation in RETRIEVE, honored per-query `ef_search`, a recall/latency/memory frontier measured at 1536-D against an exact oracle, sharded ingestion, cluster elasticity, and multi-vector user preference | Relevant, bounded feeds as the corpus grows past the scan cap — **✅ COMPLETE (2026-06-14 / 2026-06-23)**: m12p1 ✅ (read-recall harness: `vector_search_items` k-NN probe + `POST /vector_search` + `tidal-stress --verify-recall` brute-force oracle) + m12p2 ✅ (ANN candidate-gen in RETRIEVE: `for_you` via preference vector, `related` via seed embedding, cached per-signal-type top-K for `trending`; graceful scan-fallback) + m12p3 ✅ (index tuning: per-query `ef_search` honored, dimension-aware brute→HNSW crossover, F16 validated <1%, Int8 rejected) + m12p4 ✅ (sharded ingestion: scatter-gather pool + cross-shard unified reads, 3-group `shards:`) + m12p5 ✅ (idle-readiness: leader heartbeat carries live frontier so followers converge readiness on idle) + m12p6 ✅ (TLS scale-up: real mTLS `kubectl scale 3→5` seed-join over k8s, two-tier cert-manager PKI) + multi-vector user preference (online k-means clusters with DP-means split, per-cluster decayed importance, top-M ANN fan-out; cold-start single-vector fallback). v1.0 bar shared with M11: 30-day-green nightly calendar + standing Ref-A/k3s throughput re-runs. Phase docs (backfilled 2026-08-30): [`docs/planning/milestone-12/`](milestone-12/README.md) |
+| M12 | Vector Retrieval at Production Shape | Ranking-at-scale steers on measured numbers, not absent ones: ANN candidate generation in RETRIEVE, honored per-query `ef_search`, a recall/latency/memory frontier measured at 1536-D against an exact oracle, sharded ingestion, cluster elasticity, and multi-vector user preference | Relevant, bounded feeds as the corpus grows past the scan cap — **✅ COMPLETE (2026-06-14 / 2026-06-23)**: m12p1 ✅ (read-recall harness: `vector_search_items` k-NN probe + `POST /vector_search` + `tidal-stress --verify-recall` brute-force oracle) + m12p2 ✅ (ANN candidate-gen in RETRIEVE: `for_you` via preference vector, `related` via seed embedding, cached per-signal-type top-K for `trending`; graceful scan-fallback) + m12p3 ✅ (index tuning: per-query `ef_search` honored, dimension-aware brute→HNSW crossover, F16 validated <1%, Int8 rejected) + m12p4 ✅ (sharded ingestion: scatter-gather pool + cross-shard unified reads, 3-group `shards:`) + m12p5 ✅ (idle-readiness: leader heartbeat carries live frontier so followers converge readiness on idle) + m12p6 ✅ (TLS scale-up: real mTLS `kubectl scale 3→5` seed-join over k8s, two-tier cert-manager PKI) + multi-vector user preference (online k-means clusters with DP-means split, per-cluster decayed importance, top-M ANN fan-out; cold-start single-vector fallback). v1.0 bar shared with M11: 30-day-green nightly calendar + standing Ref-A/k3s throughput re-runs |
 
 ### Embeddable → Distributed Path
 
@@ -154,7 +154,7 @@ The roadmap now has two tracks:
 
 **iknowyou / Aeries: IN PROGRESS (as of 2026-02-24)** — M1–M4 complete. M5 (Communication Brief) is in progress with core implementation live; acceptance validation pending.
 
-**Engine status:** M0–M12 **COMPLETE**. M9/M10 shipped 2026-06-06; M11 (Enterprise-Grade Cluster, all nine phases) closed 2026-06-13; M12 (Vector Retrieval at production shape) closed 2026-06-14 / 2026-06-23 — see the M11/M12 milestone rows above and *Implementation Status* below. Per-milestone phase records live under `docs/planning/milestone-<N>/`; [M4](milestone-4/README.md), [M6](milestone-6/README.md), and [M12](milestone-12/README.md) were backfilled 2026-08-30 and flag where the criteria above no longer match the shipped code.
+**Engine status:** M0–M12 **COMPLETE**. M9/M10 shipped 2026-06-06; M11 (Enterprise-Grade Cluster, all nine phases) closed 2026-06-13; M12 (Vector Retrieval at production shape) closed 2026-06-14 / 2026-06-23 — see the M11/M12 milestone rows above and *Implementation Status* below.
 
 **Next (engine):** isolate the Ref-A 100k/1536-D memory growth under the corrected 4 GiB request / 6 GiB canary limit. The 30-day nightly gate remains parked: its 200 rps history is 23 PASS / 32 FAIL with repeated 4 GiB OOMKills, so it is not a production-readiness signal yet. Resume only after the controlled profile passes and restart evidence is fail-closed. The old ≥2.5× write-scaling goal belongs to Ref-B (≥5 nodes **with partitioned placement**); adding nodes to full-placement RF3 does not scale writes. Deferred follow-ups: the `writer_agent` u16 interning on the WAL v3 envelope and the offline medoid-recluster tier for multi-vector preference.
 **Next (product):** iknowyou M5 acceptance pass, then M6 Closed Loop (session lifecycle + preference drift validation).
@@ -1135,9 +1135,6 @@ Then:
 
 ### Phases
 
-**Phase docs (backfilled 2026-08-30):** [`docs/planning/milestone-4/`](milestone-4/README.md)
-— per-phase record of what actually shipped, with the divergences from the criteria below.
-
 #### Phase 1: Session Schema and Lifecycle (m4p1)
 
 **Delivers:** `SessionId`, `AgentId`, `AgentPolicy`, and `SessionHandle` types in the schema and entities modules. Schema-level `session_policy()` for declaring per-agent allowed/denied signal lists, duration limits, and signal count caps. Session lifecycle APIs: `start_session`, `close_session`, `active_sessions`. WAL entries tagged with `session_id` for crash recovery of active sessions. Closed sessions archived to storage as frozen snapshots.
@@ -1877,9 +1874,6 @@ When/Then:
 
 ### Phases
 
-**Phase docs (backfilled 2026-08-30):** [`docs/planning/milestone-6/`](milestone-6/README.md)
-— per-phase record of what actually shipped, with the divergences from the criteria below.
-
 ---
 
 #### Phase 1: Cohort Engine + Cohort-Scoped Trending (m6p1)

docs/planning/milestone-12/README.md

@@ -1,73 +0,0 @@
-# Milestone 12 · Vector Retrieval at Production Shape (✅ COMPLETE 2026-06-14 / 2026-06-23)
-
-Milestone summary row: [ROADMAP · Milestone Summary, M12](../ROADMAP.md).
-Primary changelog record: [CHANGELOG.md](../../../CHANGELOG.md) (`[Unreleased]`).
-
-> **Backfilled index, not a plan.** M12 has no `## Milestone 12` prose section in
-> the ROADMAP — only a (detailed) summary-table row — and shipped without a
-> planning directory. These records were assembled after the fact (2026-08-30)
-> from the CHANGELOG, the M12 commits, the `docs/profiling/` evidence files, and
-> the shipped test suite. Unlike M4 and M6, M12's numbers **are** recorded: this
-> milestone's whole thesis was that ranking-at-scale must steer on measured
-> numbers, so it left measurements behind.
-
-## What the milestone proves
-
-The feed stays relevant *and* bounded as the corpus grows past the scan cap.
-Before M12, `for_you` and `related` fell back to scanning an arbitrary low-id
-slice of the universe, per-query `ef_search` was accepted and silently ignored,
-and the recall/latency/memory frontier at the production embedding shape
-(1536-D) was unmeasured. M12 closed **G1** (bounded, relevant candidate
-generation) and **G2** (measured ANN quality), then took the result to a real
-sharded, mTLS, elastic cluster.
-
-## Phases
-
-| Phase | Name | Record | Landed |
-|-------|------|--------|--------|
-| m12p1 | Read-recall harness — measurement truth | [phase-1.md](phase-1.md) | `bb21e69` 2026-06-14 |
-| m12p2 | ANN candidate generation in RETRIEVE | [phase-2.md](phase-2.md) | `bb21e69` 2026-06-14, cache fix `da5d2d4` |
-| m12p3 | Index tuning + recall/memory frontier | [phase-3.md](phase-3.md) | `bb21e69` 2026-06-14 |
-| m12p4 | Sharded ingestion | [phase-4.md](phase-4.md) | `31ee612` 2026-06-14 |
-| m12p5 | Idle-readiness convergence | [phase-5.md](phase-5.md) | `aa94fd9` 2026-06-14 |
-| m12p6 | TLS scale-up + HNSW graph persistence | [phase-6.md](phase-6.md) | `8e39ee1`, `4db3f1e`, `a039955`, `727fbfc`, `44ec878` 2026-06-14→16 |
-| — | Multi-vector user preference modelling | [multi-vector-preference.md](multi-vector-preference.md) | `6a937fc` 2026-06-23 |
-
-The last item is **not** numbered `m12p7`: the ROADMAP row lists it as a peer of
-the six phases without a phase number, and no commit or doc ever called it
-`m12p7`. Inventing one would put a fake identifier into the planning record.
-
-## Evidence files this milestone left behind
-
-| Evidence | File |
-|----------|------|
-| HNSW parameter sweep + F32/F16/Int8 frontier at 1536-D | [docs/profiling/usearch-tuning.md](../../profiling/usearch-tuning.md) |
-| Scale baselines, recall-harness section, honest labelling of mean-vs-p99 | [docs/profiling/scale-baselines.md](../../profiling/scale-baselines.md) |
-| Sharded-ingestion real-cluster run (T5) | [docs/profiling/m12p4-t5-sharded-throughput.md](../../profiling/m12p4-t5-sharded-throughput.md) |
-| Idle-readiness fix + elasticity under load (T4) | [docs/profiling/m12p5-idle-readiness-elasticity.md](../../profiling/m12p5-idle-readiness-elasticity.md) |
-| k3s deploy + recall findings, the 3-shard bug chain | [docs/profiling/m12-cluster-deploy-findings.md](../../profiling/m12-cluster-deploy-findings.md) |
-| Spec updated to shipped reality (per-query `ef_search` IMPLEMENTED, not deferred) | [docs/specs/07-vector-retrieval.md](../../specs/07-vector-retrieval.md) |
-| Multi-vector preference design | [docs/research/multi-vector-preference.md](../../research/multi-vector-preference.md) |
-
-## Tests
-
-`tidal/tests/m12p1_vector_recall.rs` (3), `m12p2_ann_retrieve.rs` (5),
-`m12p6_graph_persistence.rs` (1), `m12_reseed_term_marker.rs` (4),
-`m12_preference_event_time.rs` (1), plus
-`tidal-server/tests/vector_search.rs` (8) and
-`tidal-server/tests/cluster_region.rs` (14).
-
-## What M12 did NOT close — stated, not buried
-
-The ROADMAP records the honest residue, and it should not be read as complete:
-
-- The **≥2.5× write-scaling / ≥5,000 quorum-writes-per-second** gate remains
-  Ref-A/k3s-pending. m12p4 proved with data *why*: at fixed per-pod CPU,
-  full-placement sharding scales failover, not write throughput. The ≥2.5× goal
-  belongs to a ≥5-node **partitioned-placement** topology (Ref-B).
-- The **30-day nightly gate is parked**: 23 PASS / 32 FAIL at 200 rps with
-  repeated 4 GiB OOMKills. That is not a production-readiness signal yet.
-- The **Ref-A 100k/1536-D memory growth** is still unisolated under the corrected
-  4 GiB request / 6 GiB canary limit.
-- Deferred follow-ups: `writer_agent` u16 interning on the WAL v3 envelope, and
-  the offline medoid-recluster tier for multi-vector preference.

docs/planning/milestone-12/multi-vector-preference.md

@@ -1,80 +0,0 @@
-# Multi-vector user preference modelling (✅ COMPLETE 2026-06-23)
-
-Landed in `6a937fc`, with the docs/spec/API refresh in `4051077`. Design:
-[docs/research/multi-vector-preference.md](../../research/multi-vector-preference.md).
-Changelog: [CHANGELOG.md](../../../CHANGELOG.md). Milestone index:
-[README.md](README.md). Backfilled record.
-
-> **Deliberately not numbered.** The ROADMAP's M12 row lists this work as a peer
-> of m12p1–m12p6 with no phase number, and no commit or document ever called it
-> `m12p7`. It is filed under its own name rather than given an invented id.
-
-## The premise
-
-A warm user is many interests, not one averaged vector. Averaging jazz and
-powerlifting into a single centroid produces a vector that retrieves neither —
-and ANN candidate generation (m12p2) made that failure load-bearing, because the
-preference vector *is* the query.
-
-## What shipped
-
-1. **Online preference clustering** (`tidal/src/entities/multi_preference.rs`).
-   A warm user (≥ `COLD_START_N = 5` interactions) maintains up to `K_MAX`
-   preference clusters, built by online sequential k-means with a DP-means
-   threshold split: a new engagement updates its nearest cluster via a per-cluster
-   adaptive EMA, or — past the split threshold and under the cap — opens a new
-   cluster. At the cap the nearest cluster absorbs the engagement, so the
-   structure is bounded rather than unboundedly growing.
-2. **Interests that fade.** Per-cluster *importance* composes the canonical
-   forward-decay kernel anchored to each engagement's timestamp, so a stale
-   interest decays instead of persisting at full strength forever. Reusing the
-   canonical kernel is the same discipline `SessionHotState` follows: one decay
-   implementation, not per-tier copies.
-3. **Top-M ANN fan-out.** At query time `for_you` selects the top-`M` clusters by
-   current importance, issues `M` ANN queries
-   (`candidate_gen::ann_candidates_multi`), and merges by best (minimum)
-   distance; the personalization boost is the **max** cosine over all clusters —
-   a candidate that matches any one interest strongly is not diluted by the
-   interests it does not match.
-4. **Cold-start fallback preserved.** Users below the cold-start threshold keep
-   the single adaptive-LR vector (`tidal/src/entities/preference.rs`), so nothing
-   about the pre-existing behaviour changes for a new user.
-5. **Zero-migration persistence.** `MultiPreferenceVectors::checkpoint` /
-   `restore` (`multi_preference.rs:642,703`) serialize per-cluster
-   `[update_count:8 LE][importance_at_anchor:4 LE][anchor_ts:8 LE]` behind a
-   `FORMAT_VERSION` byte, with a first-byte discrimination trick so a **legacy
-   single-vector row whose `update_count` low byte happens to equal the format
-   version** is still read correctly as a cold-start user rather than
-   misparsed. That edge is covered by
-   `restore_rescues_legacy_row_whose_count_low_byte_equals_format_version`.
-
-## Evidence
-
-- Unit coverage in `multi_preference.rs`:
-  `checkpoint_restore_roundtrip_multi_cluster`,
-  `restore_reads_legacy_single_vector_rows_as_cold_start`,
-  `restore_renormalizes_torn_cluster_to_unit_length`,
-  `restore_drops_torn_tail_cluster_keeps_prefix`,
-  `restore_skips_dimension_mismatch`,
-  `restore_skips_dim_mismatched_multi_row_not_loaded_as_garbage`,
-  `checkpoint_skips_cold_row_for_user_also_in_clusters_no_demotion`.
-  The torn-row and dimension-mismatch cases matter: a checkpoint is read after a
-  crash, so "garbage in the tail" is the expected input, not the exceptional one.
-- `tidal/tests/m12_preference_event_time.rs` (1) — event-time anchoring.
-- `tidal/benches/multi_preference.rs`.
-
-## Side effect worth recording
-
-This work incidentally closed a limitation M6 had booked against M7: per-user
-preference `update_count` is now persisted and restored (see
-[milestone-6/phase-4.md](../milestone-6/phase-4.md) and
-[milestone-6/phase-6.md](../milestone-6/phase-6.md)). The `# Known Limitation`
-comment in `tidal/src/entities/preference.rs:37-45` still claims otherwise and is
-stale.
-
-## Deferred
-
-The **offline medoid-recluster tier** — periodically re-deriving cluster centroids
-from stored engagements rather than only updating them online — is recorded in the
-ROADMAP as an open follow-up. Online sequential k-means is order-dependent; a
-recluster pass is what would remove that dependence. Not built.

docs/planning/milestone-12/phase-1.md

@@ -1,53 +0,0 @@
-# m12p1 — Read-recall harness: measurement truth (✅ COMPLETE 2026-06-14)
-
-Landed in `bb21e69`. Changelog: [CHANGELOG.md](../../../CHANGELOG.md).
-Milestone index: [README.md](README.md). Backfilled record.
-
-## Why this came first
-
-G1 and G2 are numeric gates. Before m12p1 the project had no way to measure
-either: no pure ANN probe (every read path mixed profile scoring, fusion, and
-diversity into the result), and no ground truth to compare against. A gate you
-cannot measure is a gate you cannot pass — so the harness shipped before the
-tuning it exists to judge.
-
-## What shipped
-
-1. **A pure k-NN probe.** `TidalDb::vector_search_items(query, k, ef_search)`
-   returns raw HNSW nearest neighbours over the item content embedding slot with
-   **no** profile scoring, fusion, or diversity, so the result measures index
-   quality in isolation — the G2 metric. Exposed as `POST /vector_search` on the
-   standalone router and on the multi-process region node, which merges by
-   distance across hosted shard groups. A dimension-mismatched query is a **400,
-   not a 500**: caller error, not server fault.
-2. **The oracle.** `tidal-stress --verify-recall`
-   (`tidal-stress/src/recall.rs`) seeds the corpus with deterministic, id-keyed
-   embeddings (reproducible with `--skip-seed`), holds a brute-force cosine
-   ground truth in RAM, and ramps `/vector_search` probes **open-loop** with
-   coordinated-omission correction.
-3. **A verdict, not a number dump.** It reports per-stage **true p99** and mean
-   recall@k, plus the **read-knee** — the highest sustained QPS at which
-   `p99 ≤ target AND recall@k ≥ target` both hold — with a machine-readable JSON
-   summary (`tidal-stress/src/summary.rs`) and a `--fail-on-knee` PASS/FAIL exit
-   code. Knobs: `--recall-k`, `--recall-queries`, `--read-p99-target-ms`,
-   `--recall-target`, `--recall-ef-search`.
-4. **Repaired fabricated numbers.** The p99 column in
-   `docs/profiling/social-scale.md` was mean-as-p99 (the `social` bench is
-   Criterion, which reports mean only). That column and the `scale.rs` /
-   scale-baselines framing were corrected: every closed-loop number is now
-   labelled an *isolated per-op mean (regression tripwire)*, and only the
-   open-loop harness signs off p99/recall SLOs.
-
-## Evidence
-
-- **recall@10 = 0.9997** at 20k items / 1536-D against a real standalone server
-  (HNSW M=16 / ef=400 / F16 vs brute-force cosine) — far above the 0.95 G2 target.
-- `tidal/tests/m12p1_vector_recall.rs` (3), `tidal-server/tests/vector_search.rs` (8).
-- Harness section in [docs/profiling/scale-baselines.md](../../profiling/scale-baselines.md).
-- A 1536-D HNSW-vs-brute recall@10 bench added to `tidal/benches/vector.rs`.
-
-## Carried constraint
-
-The brute-force oracle needs roughly 6 GB RAM at 1M vectors, which is why the
-100k/1M exit-gate runs execute the same harness on the k3s cluster rather than a
-laptop.

docs/planning/milestone-12/phase-2.md

@@ -1,51 +0,0 @@
-# m12p2 — ANN candidate generation in RETRIEVE (✅ COMPLETE 2026-06-14)
-
-Landed in `bb21e69`; cache-invalidation fix `da5d2d4`. Changelog:
-[CHANGELOG.md](../../../CHANGELOG.md). Milestone index: [README.md](README.md).
-Backfilled record. This is the **G1 unblock**.
-
-## What shipped
-
-1. **`CandidateStrategy::Ann` is real.** It previously fell back to a scan with a
-   warning. The db layer now resolves the query vector — the user's **preference
-   vector** for `for_you`, the **seed item's embedding** (`similar_to`) for
-   `related` — and Stage 1 runs an `O(ef_search)` HNSW search over the item
-   content slot instead of scanning an arbitrary low-id slice of the universe
-   (`tidal/src/query/executor/candidate_gen.rs`,
-   `tidal/src/query/executor/pipeline.rs`). `for_you` and `related` are now `Ann`
-   profiles.
-2. **Graceful, not conditional-on-luck.** No vector registry, no preference
-   vector, or no seed ⇒ it degrades to a scan. Anonymous reads and cold-start
-   users still serve; every embedding-less schema and every pre-m12p2 caller is
-   unchanged.
-3. **`trending` stopped being an O(N) ledger scan.** `SignalRanked` is now a
-   cached per-signal-type top-K (`tidal/src/signals/ledger/hot_top_k.rs`): O(K)
-   on the served path, with a bounded O(N) rebuild only when stale. The cache is
-   sound because decay preserves relative order at a fixed λ, so it stays valid
-   until the next write. Small ledgers rebuild on any write (always fresh); large
-   ledgers throttle the rebuild off the read hot path (1 s). `trending` now uses
-   `SignalRanked(view)`, so it ranks the actually-viewed corpus at any id — not
-   whatever happened to sit in the low-id scan slice.
-4. **`related` over HTTP.** `GET /feed?profile=related&similar_to=<id>` resolves
-   the seed's embedding and runs ANN — "more like this" on the read surface
-   (`similar_to` on `FeedQuery`, threaded through all three feed handlers).
-5. **Harness support for measuring it.** `tidal-stress --feed-profile <name>`
-   forces every feed read to one profile (per-profile retrieve p99) and
-   `--seed-preferences` builds a preference vector per user so `for_you`
-   exercises ANN rather than the scan fallback.
-6. **Cache correctness under replication** (`da5d2d4`): the `SignalRanked` top-K
-   cache is invalidated on CRDT reconciliation. Without that, a follower could
-   serve a top-K assembled before reconciliation merged remote signal state.
-
-## Evidence
-
-- **trending retrieve p99 = 3.5–7.7 ms** under concurrent writes against a real
-  1536-D standalone server — inside the 10 ms G1 target — via the cached top-K.
-- `for_you` retrieve is ANN-backed at p99 ≈ 24 ms, dominated by the Stage-3
-  preference-boost recompute (a per-candidate embedding read). Recorded, not
-  hidden, and flagged forward to m12p3's index/score tuning as the risk
-  register's "materialized-score layer".
-- ANN candidate recall is the m12p1 `/vector_search` probe: 0.9997.
-- `tidal/tests/m12p2_ann_retrieve.rs` (5) — proves ANN, `related`, and `trending`
-  reach relevant items at high ids that a scan can never reach, plus cache
-  freshness.

docs/planning/milestone-12/phase-3.md

@@ -1,63 +0,0 @@
-# m12p3 — Index tuning + recall/latency/memory frontier (✅ COMPLETE 2026-06-14)
-
-Landed in `bb21e69`. Changelog: [CHANGELOG.md](../../../CHANGELOG.md).
-Milestone index: [README.md](README.md). Backfilled record. This is the **G2**
-work.
-
-## What shipped
-
-1. **Per-query `ef_search` — now actually honored.** `UsearchIndex::search` /
-   `filtered_search` respect a per-request `ef_search`
-   (`tidal/src/storage/vector/usearch_index.rs`). Pre-m12p3 the parameter was
-   accepted for trait compliance, logged a warning, and **ignored**, because
-   USearch 2.24 has no per-call beam argument. The override is race-free via an
-   `RwLock` epoch guard (`with_expansion`): searches that agree on `ef_search`
-   run in parallel under a shared guard, and only a query that changes the live
-   beam width takes the exclusive guard for its `(set, search)` window — not a
-   per-search mutex. `ef_search = 0` selects the slot default. The knob had been
-   plumbed end-to-end in m12p1; m12p3 is what makes it move recall.
-2. **Dimension-aware brute-force → HNSW crossover.** The exact `BruteForceIndex`
-   scans every vector under a read lock at `count × dim` cost, so a fixed 10,000
-   crossover meant a 15.4M-FMA scan at 1536-D — tens of milliseconds, blocking
-   writers. `usearch_min_vectors(dim)`
-   (`tidal/src/storage/vector/registry.rs`) now keeps a brute-force scan within
-   ~4M FMAs: ≈10,000 at or under 128-D (byte-compatible with pre-m12p3
-   behaviour), ≈2,600 at 1536-D. High-dimension mid-size slots flip to HNSW
-   *before* the scan blows the SLA rather than after.
-3. **`memory_usage()`** on `UsearchIndex` — the true graph + vector footprint
-   reported by USearch, not the `index_stats` lower bound, so pod sizing uses a
-   real number.
-4. **A grid-search harness.** `cargo run --release --example ann_grid_search`
-   (`tidal/examples/ann_grid_search.rs`) builds a `UsearchIndex` plus an exact
-   `BruteForceIndex` oracle over the same deterministic id-keyed corpus and
-   reports, per `(M, ef_construction, ef_search, quantization)` point: measured
-   recall@10 vs the oracle, mean and p99 search latency, build time, and true
-   footprint.
-
-## Measured frontier (1536-D, 100k clustered corpus, real exact oracle)
-
-The production default **M=16, ef_construction=400, F16 clears G1 and G2**:
-recall@10 **0.997** at p99 ≈ 1.4 ms raw ANN. `ef_search` is the latency lever —
-recall saturates by `ef_s=128`, where p99 ≈ 1.0 ms. **F16 costs 0.25% recall vs
-F32 for half the RAM** (≈5.2 GB per 1M true footprint including the graph).
-**Int8 is rejected** at 1536-D: recall 0.715, a 28% loss. Live
-`tidal-stress --verify-recall` against a real server measured recall@10 = 1.0000
-at 20k/1536-D at both the default beam and `--recall-ef-search 400`.
-
-Full table: [docs/profiling/usearch-tuning.md](../../profiling/usearch-tuning.md).
-1M command and baselines: [docs/profiling/scale-baselines.md](../../profiling/scale-baselines.md).
-
-## The measurement artifact this phase caught
-
-The recall corpus is now **clustered** (a Gaussian mixture) in both the grid
-harness and `tidal-stress` (`recall::embedding_for`). Uniform-random high-dim
-vectors are pathological for recall@k: under uniform data recall@10 fell from
-≈0.97 at 10k to ≈0.54 at 100k. That was a **measurement artifact, not an index
-regression** — at high dimension in a uniform shell, recall@10 measures
-impossible tie-breaking rather than index quality. Every number above uses the
-clustered corpus.
-
-## Spec follow-through
-
-[docs/specs/07-vector-retrieval.md](../../specs/07-vector-retrieval.md) was
-updated in the same wave: per-query `ef_search` is IMPLEMENTED, not deferred.

docs/planning/milestone-12/phase-4.md

@@ -1,34 +0,0 @@
-# m12p4 — Sharded ingestion (✅ COMPLETE 2026-06-14)
-
-Landed in `31ee612`. Evidence:
-[docs/profiling/m12p4-t5-sharded-throughput.md](../../profiling/m12p4-t5-sharded-throughput.md).
-Milestone index: [README.md](README.md). Backfilled record.
-
-## What shipped
-
-1. **Scatter-gather pool with cross-shard unified reads.** Writes hash-route
-   across a 3-group `shards:` topology; reads unify across groups (L4). The
-   "replicated XOR sharded" split — inherited from m11p6's shard-groups work — is
-   exercised end-to-end on the read path here.
-2. **A real run, not a simulation.** A 3-shard-group × RF=3 cluster on `kind`
-   (single node, 24 vCPU / 50 GB) under full mTLS with cert-manager-issued certs,
-   driven by **two** in-cluster `tidal-stress` generator pods: real binaries, real
-   WAL and quorum, real gRPC replication, real inter-node mTLS HTTP.
-3. **An HTTP/2 forwarding bug fixed.** A synthesized JSON body on a 204 relay
-   triggered an h2 `RST_STREAM`; a 204 forward must carry no body.
-
-## What the run actually proved — and did not
-
-With data rather than a hardware caveat: **at fixed per-pod CPU, full-placement
-sharding scales failover, not write throughput.** The headline gate
-(≥2.5× scaling AND ≥5,000 quorum writes/s) therefore remains **Ref-A/k3s-pending**,
-and the ≥2.5× target properly belongs to a ≥5-node **partitioned-placement**
-topology (Ref-B). Adding nodes to full-placement RF3 does not scale writes; the
-gate was mis-assigned, and m12p4 is what surfaced that.
-
-## Follow-on repair
-
-Deploying this layer to the real 3-shard k3s cluster exposed five 3-shard
-catch-up / read / reseed bugs, each root-caused and fixed in `m12-rc8` (with
-adversarial failover-safety review before implementation). Record:
-[docs/profiling/m12-cluster-deploy-findings.md](../../profiling/m12-cluster-deploy-findings.md).

docs/planning/milestone-12/phase-5.md

@@ -1,43 +0,0 @@
-# m12p5 — Idle-readiness convergence (✅ COMPLETE 2026-06-14)
-
-Landed in `aa94fd9`. Evidence:
-[docs/profiling/m12p5-idle-readiness-elasticity.md](../../profiling/m12p5-idle-readiness-elasticity.md).
-Milestone index: [README.md](README.md). Backfilled record.
-
-## The defect
-
-A snapshot-installed joiner served `503` on its readiness probe
-(`/health` → `region_health` → `is_ready`) until its sticky `converged` latch
-flipped — and pre-m12p5 the only thing that flipped that latch was observing new
-traffic. On an **idle** cluster there is no new traffic, so a caught-up follower
-stayed unready indefinitely. The worklog records an 11.5-hour hang from exactly
-this.
-
-This is the failure class worth naming: readiness derived from *activity* instead
-of from *state* is silently wrong precisely when the system is quiet.
-
-## What shipped
-
-1. **The leader heartbeat carries its live frontier** (`leader_last_seq`), so a
-   caught-up follower can compare its own applied frontier against the leader's
-   and flip `/health` Ready on an idle cluster — no ship and no status poll
-   required.
-2. **Cert SAN wildcard widened** for scale-to-5, removing the certificate-shaped
-   blocker in front of m12p6's scale-up test.
-
-## Evidence
-
-Proven locally over real OS processes, with the elasticity-under-load (T4)
-context recorded in the profiling doc. The 1M / 1536-D T4 run on k3s remains the
-project's standing Ref-A/k3s dependency — the machinery and the fix that unblock
-it are proven; the run itself is pending infrastructure access. That distinction
-is the honest one: the fix is verified, the scale run is not yet done.
-
-## Later reinforcement
-
-The readiness contract kept being sharpened after M12 closed, which is itself
-evidence the original latch was under-specified rather than merely buggy:
-`0919b0a` (seed-join learner auto-promotes after snapshot install and reports the
-caught-up frontier on the heartbeat), `7450cc7` ("readiness must prove
-convergence, not merely lack a marker"), and `fc1cc90` (refresh frontier gauges
-on the driver tick).

docs/planning/milestone-12/phase-6.md

@@ -1,47 +0,0 @@
-# m12p6 — TLS scale-up + HNSW graph persistence (✅ COMPLETE 2026-06-16)
-
-Landed across `8e39ee1`, `4db3f1e`, `a039955` (2026-06-14→15), then `727fbfc`
-and `44ec878` (2026-06-16). Changelog:
-[CHANGELOG.md](../../../CHANGELOG.md). Milestone index: [README.md](README.md).
-Backfilled record.
-
-## What shipped
-
-1. **Elasticity over REAL mTLS on k8s, for the first time.** A genuine
-   `kubectl scale 3→5` exercised seed-join over mTLS on `kind`: joiners flip
-   Ready in ~13 s via the m12p5 idle-readiness heartbeat, auto-promote to Voter,
-   reach full content parity at lag 0, and scale-down loses **zero acknowledged
-   DATA writes**.
-2. **A six-bug chain fixed to get there** — each one a separate way TLS turns a
-   working plaintext path into a silent failure:
-   - `https://` seed scheme (a plaintext seed URL against a TLS listener),
-   - rustls `CryptoProvider` install order (must be installed up front),
-   - a headless seed Service (so a joiner can resolve peers before joining),
-   - cold-handshake poll timeout,
-   - two-tier cert-manager PKI,
-   - the `grpc_tls_for` CA fallback for a joiner **not yet in the topology** —
-     the bootstrap paradox: a node needs TLS to join, and joining is what puts it
-     in the topology TLS is derived from.
-3. **HNSW graph persistence + bounded SIGTERM drain** (`a039955`). Boot loads the
-   persisted graph instead of rebuilding it, and a suspect graph is **skipped on
-   reseed-pending close** rather than trusted — the fail-closed choice: a
-   rebuild costs time, a silently corrupt graph costs correctness.
-4. **7th edge** (`44ec878`): correct reseed seqno alongside the suspect-graph
-   skip (rc11), plus the rc8+rc9 6-bug k3s 3-shard cluster repair (`727fbfc`).
-
-## Evidence
-
-- `tidal/tests/m12p6_graph_persistence.rs` (1),
-  `tidal/tests/m12_reseed_term_marker.rs` (4).
-- `tidal-server/tests/cluster_region.rs` (14).
-- [docs/profiling/m12-cluster-deploy-findings.md](../../profiling/m12-cluster-deploy-findings.md)
-  — the k3s deploy record, incl. the clean-slate 100k/1536 re-seed and the
-  workspace test tallies at rc8.
-
-## Carried forward
-
-The reseed/readiness surface kept needing repair after M12 closed —
-`c8ea05b`, `fe56d1b`, `973f073` (2026-06-18), `da736b8` (2026-08-20),
-`5b3cfe5` and `7450cc7` (2026-08-21). Read m12p6 as "the scale-up path was
-proven once, end to end, over real TLS", not as "the reseed state machine was
-finished".

docs/planning/milestone-4/README.md

@@ -1,85 +0,0 @@
-# Milestone 4 · Agent Memory (✅ COMPLETE 2026-02-21)
-
-Milestone spec, thesis, UAT scenario, and per-phase acceptance criteria:
-[ROADMAP · Milestone 4](../ROADMAP.md). Status row: `m4: Agent Session Layer`.
-
-> **These are backfilled records, not the original plans.** M4 shipped in the
-> squashed commit `39ada28` (2026-02-21) before this project kept per-milestone
-> planning directories, so the four phase docs here were written after the fact
-> (2026-08-30) from the ROADMAP acceptance criteria, the shipped source tree, and
-> the M4 test suite. Every claim below names the artifact that proves it. Where
-> the shipped surface diverges from the ROADMAP text, the phase doc says so
-> rather than restating the plan.
-
-## What the milestone proves
-
-An agent — not a human clicking a UI — can own a scoped memory lane inside the
-same embedded process: open a session bound to a `(user, agent, policy)` triple,
-write short-lived signals that the schema-declared policy accepts or rejects,
-read a live snapshot of that session's decayed state, blend the session into a
-`RETRIEVE` ranking, then close it and read the frozen archive. No Redis, no
-feature store, no policy middleware.
-
-## Phases
-
-| Phase | Name | Record |
-|-------|------|--------|
-| m4p1 | Session Schema and Lifecycle | [phase-1.md](phase-1.md) |
-| m4p2 | Session Signal Engine | [phase-2.md](phase-2.md) |
-| m4p3 | Policy Enforcement and Audit | [phase-3.md](phase-3.md) |
-| m4p4 | Session-Aware Ranking and M4 UAT | [phase-4.md](phase-4.md) |
-
-The four phases were strictly sequential (each needed the previous phase's write
-path), and all four landed in one commit; the phase split below is therefore a
-split of the *delivered surface*, traced from the ROADMAP's phase definitions to
-the modules and tests that implement each one.
-
-## Shipped surface (verified against the working tree)
-
-| Concern | Artifact |
-|---------|----------|
-| Session types | `tidal/src/session/types.rs` — `SessionId`, `AgentId`, `SessionHandle` |
-| Live session state | `tidal/src/session/state.rs` — per-session accumulators |
-| Session signal decay | `tidal/src/session/signal_state.rs` |
-| Snapshot / archive read model | `tidal/src/session/snapshot.rs` |
-| Policy evaluation | `tidal/src/session/policy.rs` |
-| Audit log | `tidal/src/session/audit.rs` |
-| Archive serialization | `tidal/src/session/serde/mod.rs`, `serde/start_record.rs` |
-| Policy declaration in schema | `tidal/src/schema/validation/policies.rs` — `AgentPolicy` |
-| `db.*` session API | `tidal/src/db/sessions.rs` |
-| WAL durability | `WalCommand::SessionStart` / `SessionSignal` / `SessionClose` in `tidal/src/wal/writer.rs` |
-
-## Evidence
-
-| Claim | Proof |
-|-------|-------|
-| Full agent workflow works end to end | `tidal/tests/m4_uat.rs` — 12 `#[test]` steps, "M4 User Acceptance Test: Agent Session Layer". Added by `39ada28`, the M4 commit. |
-| Sessions survive a crash | `tidal/tests/session_durability.rs` — 10 tests incl. `active_session_state_restored_after_crash`, `metadata_survives_crash`, `wal_replay_restores_signal_counts_exactly`. Added one commit later by `192c473` (M5), so read it as M4-surface hardening rather than M4 itself. |
-| Session semantics still hold after later reworks | `tidal/tests/review_pass2_zone_a_sessions.rs`, `tidal/tests/review_pass2_query_for_session.rs` — added by `9728194` (M0–M10 review pass 2, 2026-06-09) |
-| Sessions survive M7 crash hardening | `tidal/tests/m7_crash_m6.rs`, `tidal/tests/m7_crash_invariant.rs` (session-touching cases) |
-| Recorded at close | ROADMAP status row: 607 lib + 12 m4_uat + prior UATs passing |
-
-All 49 M4 acceptance criteria in [ROADMAP · Milestone 4](../ROADMAP.md) are
-marked `[x]`.
-
-## Naming drift worth knowing
-
-The ROADMAP's M4 text names errors `LumenError::PolicyViolation` /
-`LumenError::SessionExpired`. The shipped enum is `TidalError` — the `Lumen`
-prefix is a pre-rename artifact left in the planning prose. Real variants:
-`TidalError::PolicyViolation` and `TidalError::SessionExpired`
-(`tidal/src/schema/error.rs:123,130`), plus the session-internal
-`PolicyViolation { kind: PolicyViolationKind }` re-exported from
-`tidal/src/lib.rs`.
-
-## What M4 deliberately did not do
-
-The ROADMAP's "Deferred to Later Milestones" list for M4 held nine items. Their
-real fates:
-
-- Cross-session aggregation → shipped in m6p4 (see [milestone-6/phase-4.md](../milestone-6/phase-4.md)).
-- Session signal influence on the global preference vector → shipped in m6p4.
-- Semantic hint matching → M4 shipped keyword matching; embeddings arrived with M5.
-- RLHF training-data export, per-agent QPS rate limiting, session TTL sweeper → shipped in M7 (m7p2/m7p4).
-- User revocation of agent-contributed signals, multi-agent sessions → shipped in M9/M10; the read-path and profile-override fields on `AgentPolicy` (`allowed_read_signals`, `denied_read_signals`, `allowed_user_attributes`, `denied_user_attributes`, `allowed_profile_overrides`) were added by commit `d8e4083` for that work, not by M4.
-- Session forking and merging → still not built. No commit, no test, no issue; it remains an open idea, not a shipped feature.

docs/planning/milestone-4/phase-1.md

@@ -1,64 +0,0 @@
-# m4p1 — Session Schema and Lifecycle (✅ COMPLETE 2026-02-21)
-
-Phase spec and acceptance criteria: [ROADMAP · Milestone 4 · Phase 1](../ROADMAP.md).
-Milestone index: [README.md](README.md). Backfilled record — see the README's note
-on provenance.
-
-## What shipped
-
-1. **Session identity types** (`tidal/src/session/types.rs`). `SessionId` is a
-   `u64` newtype (`Copy`, `Hash`, `Ord`, `Display` as `session:<n>`) handed out
-   monotonically by `start_session`; `from_raw` exists for deserialization.
-   `AgentId` is a validated `String` newtype: 1–64 bytes, `[a-z0-9_-]` only —
-   uppercase, spaces, empty, and 65-byte inputs are all rejected at construction.
-2. **`SessionHandle` as the capability** (`tidal/src/session/state.rs:151`).
-   Carries `id`, `user_id`, `agent_id`, `policy_name`, `started_at: Instant`, and
-   a `closed: Arc<AtomicBool>` shared with the live `SessionState`.
-   `close_session` takes the handle **by value**, so use-after-close is a compile
-   error in the common case; the shared `closed` flag is the runtime
-   defence-in-depth for handles cloned into another thread.
-3. **Policy declaration in schema** (`tidal/src/schema/validation/policies.rs`).
-   `AgentPolicy` with `allowed_signals`, `denied_signals`, `max_session_duration`,
-   `max_signals_per_session` (`0` = unlimited), registered through
-   `SchemaBuilder::session_policy(name, policy)` and read back via
-   `Schema::session_policy(name)`. Validated at schema build time.
-4. **Lifecycle API** (`tidal/src/db/sessions.rs`). `start_session(user_id,
-   agent_id, policy_name, metadata) -> Result<SessionHandle>`,
-   `close_session(handle) -> Result<SessionSummary>`, and
-   `active_sessions() -> Vec<…>`. An undeclared policy name is refused at
-   `start_session`, not at first write.
-5. **WAL durability for the lifecycle**. `WalCommand::SessionStart` /
-   `SessionClose` (`tidal/src/wal/writer.rs:101,124`) journal session boundaries
-   on the same stream as signals; replay restores start-without-close as an
-   active session and start-with-close as an archive.
-6. **Archive keyspace.** `Tag::Session = 0x07` (`tidal/src/storage/keys.rs:29`)
-   holds session snapshots and audit logs, so a closed session is readable after
-   process restart, not just after close.
-
-## Evidence
-
-| Criterion | Proof |
-|-----------|-------|
-| Schema accepts and returns a declared policy | `m4_uat.rs::step1_schema_with_session_policy_builds` |
-| Start → active_sessions → close → not active | `m4_uat.rs::step2_session_start_and_close` |
-| Multiple concurrent sessions tracked independently | `m4_uat.rs::step12_active_sessions_tracking` |
-| Undeclared policy name rejected | `m4_uat.rs::step10_invalid_policy_name_rejected` |
-| `AgentId` format enforcement | `m4_uat.rs::step11_agent_id_validation` |
-| Active session restored after crash | `session_durability.rs::active_session_state_restored_after_crash` |
-| Session metadata survives crash | `session_durability.rs::metadata_survives_crash` |
-| Closed session not resurrected as active | `review_pass2_zone_a_sessions.rs::closed_session_is_not_restored_as_active` |
-
-## Divergence from the plan
-
-- **Error naming.** The ROADMAP says `LumenError::SessionExpired`; the shipped
-  enum is `TidalError::SessionExpired` (`tidal/src/schema/error.rs:130`).
-- **Session-ID reuse across restart.** The doc comment on `SessionId` still says
-  uniqueness is "not guaranteed across restarts". That is now stale: the review
-  pass-2 remediation (`9728194`) added
-  `review_pass2_zone_a_sessions.rs::reopen_does_not_reissue_archived_session_id`,
-  which proves a reopened database does not hand an archived id to a new session.
-  The guarantee is real; only the comment lags.
-- **`AgentPolicy` grew after M4.** The five read-path and profile-override fields
-  (`allowed_read_signals`, `denied_read_signals`, `allowed_user_attributes`,
-  `denied_user_attributes`, `allowed_profile_overrides`) were added by `d8e4083`
-  for M9/M10 governance. M4 shipped only the four write-path fields.

docs/planning/milestone-4/phase-2.md

@@ -1,86 +0,0 @@
-# m4p2 — Session Signal Engine (✅ COMPLETE 2026-02-21)
-
-Phase spec and acceptance criteria: [ROADMAP · Milestone 4 · Phase 2](../ROADMAP.md).
-Milestone index: [README.md](README.md). Backfilled record.
-
-## What shipped
-
-1. **`session_signal()` write path** (`tidal/src/db/sessions.rs`).
-   `db.session_signal(&handle, signal_type, entity_id, weight, timestamp,
-   Option<annotation>)` validates the session is open, evaluates policy
-   (m4p3), folds the weight into the per-signal-type accumulator, bumps the
-   session counters, and journals `WalCommand::SessionSignal`
-   (`tidal/src/wal/writer.rs:109`).
-2. **`SessionHotState` — reuse, not a second decay implementation**
-   (`tidal/src/session/signal_state.rs`). Session scores delegate the running
-   arithmetic to the canonical `forward_decay_step` kernel, the same kernel
-   `HotSignalState` uses, so the two tiers cannot drift
-   (`CODING_GUIDELINES.md` §3). Session decay is deliberately aggressive:
-   `DEFAULT_SESSION_LAMBDA = ln(2)/300s` — a five-minute half-life, matching
-   session timescales rather than the multi-day content half-lives.
-   Score/timestamp/count are three atomics updated by a CAS loop; the struct is
-   intentionally *not* cache-line padded, because sessions are not on the
-   200-entity hot ranking path.
-3. **Windowed counters per session signal type.** Each
-   `SessionSignalState` owns a `BucketedCounter::with_start_time(now_ns)`, so a
-   snapshot reports a 1-hour window count alongside the decayed score.
-4. **Snapshot read model** (`tidal/src/session/snapshot.rs`).
-   `db.session_snapshot(session_id) -> SessionSnapshot` carries
-   `signals_written`, `signals_rejected`, `overrides_rejected`, `duration_ms`,
-   `metadata`, timestamped `annotations`, `reward_velocity`,
-   `signaled_entities`, the `audit_log` + `audit_truncated` flag, per-signal-type
-   `signals: HashMap<String, SignalSnapEntry>`, and `started_at_ns` /
-   `closed_at_ns`. Active sessions decay lazily to wall-clock read time;
-   archived sessions are frozen at `close_session`.
-5. **Isolation by construction.** Session signals never touch the global item
-   ledger, the user preference vector, or the interaction-weight ledger. A
-   session's influence is read-time only, via the m4p4 `FOR SESSION` path.
-6. **Bounded memory, stated in constants** (`tidal/src/session/audit.rs`):
-   `MAX_ANNOTATIONS = 100`, `MAX_AUDIT_ENTRIES = 10_000`,
-   `MAX_CLOSED_SESSIONS = 10_000` with `EVICT_BATCH_SIZE = 1_000`, plus a
-   `MAX_SIGNALED_ENTITIES` structural cap on the distinct-entity set. The
-   entity cap is independent of `max_signals_per_session` (which may be `0` =
-   unlimited) so an adversarial session cannot grow the boost set without bound;
-   past the cap, new distinct entities are dropped and the boost degrades
-   gracefully instead of the process growing.
-
-## Evidence
-
-| Criterion | Proof |
-|-----------|-------|
-| Accepted writes update counts and audit | `m4_uat.rs::step3_session_signal_and_audit` |
-| Annotations captured, entity recorded in `signaled_entities` | `m4_uat.rs::step6_session_annotations_and_snapshot` |
-| Archived snapshot readable after close | `m4_uat.rs::step7_closed_session_snapshot` |
-| Archived snapshot readable after close **and reopen** | `session_durability.rs::archived_session_readable_after_close_and_reopen` |
-| Per-signal windowed counts surface in the snapshot | `session_durability.rs::per_signal_snapshot_shows_windowed_counts` |
-| Annotation timestamps preserved; annotations survive crash | `session_durability.rs::annotation_timestamps_preserved`, `annotations_survive_crash` |
-| WAL replay reproduces the accumulators exactly | `session_durability.rs::wal_replay_restores_signal_counts_exactly`, `wal_replay_reproduces_identical_window_1h` |
-| Two sessions never see each other's entities | `m4_uat.rs::step9_session_isolation` |
-
-## Divergence from the plan
-
-- **Where the decay constant lives.** The ROADMAP framed session decay as
-  per-signal-type schema decay reused verbatim. Shipped behaviour is a single
-  session-tier lambda (`DEFAULT_SESSION_LAMBDA`, 5-minute half-life) captured at
-  `SessionSignalState` construction — one knob for the whole session tier rather
-  than per-type specs.
-- **The performance criteria have a harness but no recorded numbers.** The
-  ROADMAP asserts `session_signal` < 200 µs, `session_snapshot` < 50 µs, and
-  50,000 session signals/s "(benchmarked)". The harness is real —
-  `tidal/benches/session.rs` (added one commit later by `192c473`) benches
-  exactly `session_signal`, `session_snapshot_100_signals`, and
-  `retrieve_1k_items/{without_session,with_session}` — but no run of it is
-  recorded anywhere in `docs/`, unlike the M11/M12 numbers in
-  `docs/profiling/`. So the functional criteria are proven by the tests above;
-  the three µs/throughput figures are asserted, measurable on demand
-  (`cargo bench -p tidaldb --bench session`), and **not currently evidenced**.
-
-## Fixed later, worth recording
-
-The out-of-order arm of the session decay update originally clamped `dt` to zero
-and folded a late weight in at full value, silently over-crediting stale
-activity. That is why `SessionHotState::on_signal` now delegates to
-`forward_decay_step`: the kernel folds the *pre-decayed* weight
-(`weight × exp(-λ·age)`) and refuses to regress `last_update_ns`. The
-divergence, and the reason the kernel exists, are documented in the doc comment
-at `tidal/src/session/signal_state.rs`.

docs/planning/milestone-4/phase-3.md

@@ -1,75 +0,0 @@
-# m4p3 — Policy Enforcement and Audit (✅ COMPLETE 2026-02-21)
-
-Phase spec and acceptance criteria: [ROADMAP · Milestone 4 · Phase 3](../ROADMAP.md).
-Milestone index: [README.md](README.md). Backfilled record.
-
-## What shipped
-
-1. **Policy evaluation on the write path** (`tidal/src/session/policy.rs`).
-   Every `session_signal` call is checked before any state mutation. The
-   evaluator returns a typed `PolicyViolation { kind, .. }` rather than a
-   stringly-typed reason. The four kinds M4 shipped:
-
-   | `PolicyViolationKind` | Trigger |
-   |-----------------------|---------|
-   | `Denied` | signal type appears in `denied_signals` |
-   | `NotAllowed` | `allowed_signals` is non-empty and the type is absent from it |
-   | `CountCap` | `signals_written` reached `max_signals_per_session` |
-   | `Expired` | wall clock passed `max_session_duration` |
-
-2. **Error surface** (`tidal/src/db/sessions.rs:405-417`). `Expired` maps to
-   `TidalError::SessionExpired`; every other kind maps to
-   `TidalError::PolicyViolation` (`tidal/src/schema/error.rs:123`). A rejected
-   write mutates no session signal state — only the rejection counter and the
-   audit log advance.
-3. **Bounded audit log** (`tidal/src/session/audit.rs`). `AuditEntry` carries
-   `timestamp_ns`, the signal type, an `accepted` flag, an `AuditKind`, and an
-   optional `reason`. It is a `VecDeque` capped at `MAX_AUDIT_ENTRIES = 10_000`;
-   past the cap the oldest entries are evicted and the snapshot's
-   `audit_truncated` flag is set, so a reader can tell a complete log from a
-   truncated one instead of silently believing a partial record.
-4. **`db.session_audit(session_id)`** reads the live log for an active session;
-   for a closed session the log ships inside the archived `SessionSnapshot`
-   (`audit_log` field), so it is still readable after close and after restart.
-
-## Evidence
-
-| Criterion | Proof |
-|-----------|-------|
-| A denied signal is rejected, counted, and audited with a reason | `m4_uat.rs::step4_policy_rejects_denied_signal` — asserts `signals_rejected == 1`, `signals_written == 0`, `audit.len() == 1`, `!audit[0].accepted`, `audit[0].reason.is_some()` |
-| A signal absent from a non-empty allow list is rejected | `m4_uat.rs::step5_policy_rejects_non_allowed_signal` |
-| Accepted writes are audited as accepted | `m4_uat.rs::step3_session_signal_and_audit` |
-| Truncation is observable, not silent | `session_durability.rs::audit_truncation_marker_set_when_cap_exceeded` |
-| Audit survives close | `m4_uat.rs::step7_closed_session_snapshot`, `session_durability.rs::archived_session_readable_after_close_and_reopen` |
-
-## Divergence from the plan
-
-- **Typed kinds, not free-text reasons.** The ROADMAP specified
-  `AuditEntry = { timestamp, signal_type, outcome: Accepted | Rejected(reason) }`
-  and a `reason: String` on the error. Shipped design keeps the human-readable
-  reason but adds `PolicyViolationKind` and `AuditKind` enums, so callers branch
-  on a variant instead of matching on prose. This is strictly better and is what
-  M9/M10 later extended.
-- **The `< 1 µs` policy-evaluation criterion has no recorded measurement.** The
-  ROADMAP marks it `[x]` "(benchmarked)". It is a `HashMap` lookup on the write
-  path, which makes the claim plausible, but no benchmark isolates it and no
-  recorded run exists. `tidal/benches/session.rs::session_signal` measures the
-  whole write including WAL, not policy evaluation alone. Treat the figure as
-  unevidenced.
-- **The property test in the ROADMAP criteria is not present as a property
-  test.** "For any sequence of allowed and denied signal writes, the audit log
-  exactly matches the write outcomes and no denied signal modifies session
-  state" is proven by the concrete `m4_uat.rs` steps 3–5 and by
-  `session_durability.rs`, not by a `proptest`. The invariant is covered; the
-  stated *form* (randomized property test) was not built.
-
-## Extended later
-
-M9/M10 governance (`d8e4083`) added seven further `PolicyViolationKind` variants
-— `CommunityWriteDenied`, `CommunityWriteNotAllowed`, `ReadDenied`,
-`ReadNotAllowed`, `AttributeReadDenied`, `AttributeReadNotAllowed`,
-`ProfileOverrideNotAllowed` — and the matching `AuditKind` variants
-(`ReadDenied`, `AttributeReadDenied`, `ProfileOverrideRejected`) plus the
-`overrides_rejected` counter on `SessionSnapshot`. The M4 mechanism is the one
-they extended: read-path and community policy reuse this evaluator and this
-audit log rather than adding a parallel one.

docs/planning/milestone-4/phase-4.md

@@ -1,98 +0,0 @@
-# m4p4 — Session-Aware Ranking and M4 UAT (✅ COMPLETE 2026-02-21)
-
-Phase spec and acceptance criteria: [ROADMAP · Milestone 4 · Phase 4](../ROADMAP.md).
-Milestone index: [README.md](README.md). Backfilled record.
-
-## What shipped
-
-1. **`FOR SESSION` on both query surfaces.**
-   `RetrieveBuilder::for_session(session_id)` and the SEARCH equivalent
-   (`tidal/src/query/search/executor.rs:265` — `with_session(context, snapshot)`).
-   The db layer loads the session snapshot and derives a `SessionContext`
-   (`tidal/src/db/query_ops.rs:172,440`), so an archived session works exactly
-   like a live one — frozen values instead of decayed-to-now values.
-2. **`SessionContext` as the ranking input** (`tidal/src/session/snapshot.rs:54`).
-   Built by `SessionContext::from_snapshot`: annotation text is split on
-   whitespace, lowercased, and de-duplicated through a `HashSet` into
-   `keywords`; `reward_velocity` is the current (or frozen) score of the
-   `reward` signal; session `metadata` rides along.
-3. **The boost formula** (`ProfileExecutor::session_boost`,
-   `tidal/src/ranking/executor/mod.rs:605-627`):
-
-   ```
-   hint_score = matched_keywords / total_keywords     // [0,1]
-   vel_norm   = reward_velocity / (reward_velocity+1) // Michaelis-Menten saturation
-   boost      = hint_score * 0.3 + vel_norm * 0.2
-   ```
-
-   A keyword matches if any of the candidate's metadata **values** contains it,
-   case-insensitively. The boost is **additive** and applied after base scoring,
-   before min-max normalization, so it layers onto personalization instead of
-   replacing it. Keywords are lowercased once per query
-   (`lowered_session_keywords`), not once per candidate.
-4. **Session state travels back with the results.** `Results.session_snapshot:
-   Option<SessionSnapshot>` is populated whenever `for_session` is present, so
-   an agent gets ranked items and its own session state in one round trip.
-5. **A ranking reason code.** `ReasonCode::SessionContext`
-   (`tidal/src/ranking/reason.rs:71,232`) makes a session-influenced result
-   explainable rather than mysterious.
-6. **The M4 UAT** (`tidal/tests/m4_uat.rs`) — 12 `#[test]` functions, one per
-   scenario step, covering lifecycle, signals, policy accept/reject, annotations,
-   snapshot, archive, `FOR SESSION` ranking, isolation, and `AgentId` validation.
-
-## Evidence
-
-| Criterion | Proof |
-|-----------|-------|
-| `FOR SESSION` query returns results and attaches the snapshot | `m4_uat.rs::step8_for_session_ranking_boost` |
-| Keyword hints move ranking | `session_durability.rs::hint_keywords_boost_matching_items`; unit test `ranking/executor/mod.rs::session_boost_keyword_match_is_case_insensitive` |
-| Empty keyword set is a no-op (no divide-by-zero, no phantom boost) | unit test `ranking/executor/mod.rs::session_boost_empty_keywords_is_noop` |
-| Sessions do not leak across each other in ranking inputs | `m4_uat.rs::step9_session_isolation` |
-| Archived session usable as query context | `m4_uat.rs::step7_closed_session_snapshot` |
-| RETRIEVE and SEARCH behave identically on a swept session | `review_pass2_query_for_session.rs::retrieve_and_search_degrade_identically_on_missing_session` |
-
-## Divergence from the plan
-
-- **Missing session degrades, it does not error — deliberately reversed.** The
-  ROADMAP criterion reads: "When `for_session` references a non-existent
-  session, `LumenError::Query("session not found")` returned." That behaviour
-  shipped and was then **removed on purpose** by the M0–M10 review pass 2
-  (`9728194`): `FOR SESSION <swept-id>` is a well-formed query, and
-  `CODING_GUIDELINES.md` §6 ("graceful degradation, never failure") says it must
-  execute without the boost. Before the fix RETRIEVE returned
-  `Err(SessionNotFound)` while the structurally identical SEARCH degraded — a
-  surface-specific outage the moment the session sweeper ran. Both surfaces now
-  degrade. The ROADMAP criterion is stale; the current behaviour is correct.
-- **The `< 5 ms` session-context overhead figure is unevidenced.**
-  `tidal/benches/session.rs` has exactly the right benchmark
-  (`retrieve_1k_items/{without_session,with_session}`), but no run of it is
-  recorded in `docs/`. Measurable on demand
-  (`cargo bench -p tidaldb --bench session`); not currently measured.
-
-## Known dead field — recorded, not silently tolerated
-
-`SessionContext.signaled_entities` is populated by
-`SessionContext::from_snapshot` and documented "for entity-level boost", and
-`tidal/src/session/audit.rs` says the set "feeds the FOR SESSION entity-level
-boost". **No such boost exists.** `session_boost` reads only `keywords` and
-`reward_velocity`; a repo-wide search finds `signaled_entities` reached from a
-`SessionContext` only in a test fixture that sets it to `HashSet::new()`. The
-field on `SessionSnapshot` is genuinely used — but by cross-session preference
-aggregation (`tidal/src/db/sessions.rs:547`, m6p4), never by ranking.
-
-Consequences worth stating plainly:
-
-- The set is computed and cloned on every `FOR SESSION` query for no effect.
-- `m4_uat.rs::step8_for_session_ranking_boost` is weaker than its comments
-  suggest. With no annotation set, `hint_score` is 0 and every candidate
-  receives the same uniform `vel_norm * 0.2`, which cannot reorder anything, so
-  its assertion (`rank_with <= rank_without`) holds trivially. The keyword half
-  of the boost *is* genuinely proven — by
-  `session_durability.rs::hint_keywords_boost_matching_items` and the
-  case-insensitivity unit test — so the mechanism works; only the
-  entity-identity claim in step 8's comments is unbacked.
-
-Not fixed here: this record is a planning-history backfill and does not touch
-Rust source. Either the entity boost should be implemented (a session that
-rewarded entity 5 arguably should rank entity 5 up) or the field and the two doc
-comments should go. Both are real changes needing a real owner.

docs/planning/milestone-6/README.md

@@ -1,72 +0,0 @@
-# Milestone 6 · Full Surface Coverage (✅ COMPLETE 2026-02-23)
-
-Milestone spec, thesis, UAT scenario, and per-phase acceptance criteria:
-[ROADMAP · Milestone 6](../ROADMAP.md).
-
-> **These are backfilled records, not the original plans.** M6 shipped in the
-> squashed commit `213b8ef` (2026-02-23, "complete M6-M7 + Enterprise Readiness
-> milestones") before this project kept per-milestone planning directories. The
-> six phase docs here were written after the fact (2026-08-30) from the ROADMAP
-> acceptance criteria, the shipped source tree, and the M6 test suite. Every
-> claim names the artifact that proves it, and where the shipped surface diverges
-> from the ROADMAP text the phase doc says so instead of restating the plan.
-
-## What the milestone proves
-
-Every use case in [USE_CASES.md](../../../USE_CASES.md), every sort mode, every
-filter, and every feedback loop resolves inside one query engine — cohort-scoped
-trending, social-graph scoping and collaborative filtering, the full sort
-surface, collections and saved searches, scoped SEARCH plus autocomplete, and
-notification capping — with no application-side ranking logic left over.
-
-## Phases
-
-| Phase | Name | Record | Primary test |
-|-------|------|--------|--------------|
-| m6p1 | Cohort Engine + Cohort-Scoped Trending | [phase-1.md](phase-1.md) | `tidal/tests/m6_cohort.rs` (12) |
-| m6p2 | Social Graph + Collaborative Filtering | [phase-2.md](phase-2.md) | `tidal/tests/m6_social.rs` (8) |
-| m6p3 | Full Sort Modes + Live Content + Engagement Filters | [phase-3.md](phase-3.md) | `tidal/tests/m6p3_sorts.rs` (9), `m6p3_filters.rs` (3), `m6p3_edge_cases.rs` (3) |
-| m6p4 | Collections + Watch History + Saved Searches | [phase-4.md](phase-4.md) | `tidal/tests/m6p4_collections.rs` (10) |
-| m6p5 | Query Composition + SUGGEST Autocomplete | [phase-5.md](phase-5.md) | `tidal/tests/m6p5_scope.rs` (13) |
-| m6p6 | Notification Capping + Adaptive Preferences + M6 UAT | [phase-6.md](phase-6.md) | `tidal/tests/m6_uat.rs` (9), `m6p6_creator_profile.rs` (6) |
-
-Test counts are `#[test]` functions in the working tree today; the ROADMAP status
-rows record the smaller counts these files had at close (e.g. m6p1 closed at
-"9 m6_cohort"). The suites grew with later hardening — most visibly
-`tidal/tests/m6_crash_surfaces.rs` (4) and `tidal/tests/m7_crash_m6.rs`, added by
-M7 to crash-fence the state M6 introduced.
-
-## Shipped surface (verified against the working tree)
-
-| Concern | Artifact |
-|---------|----------|
-| Cohort definitions, predicates | `tidal/src/cohort/types.rs` |
-| Cohort membership resolution | `tidal/src/cohort/resolver.rs` |
-| Per-cohort signal aggregation | `tidal/src/cohort/ledger.rs` |
-| Cohort checkpoint/restore | `tidal/src/cohort/checkpoint.rs` |
-| Cohort db API | `tidal/src/db/cohorts.rs` |
-| Social-graph filter | `tidal/src/query/executor/social_filter.rs` |
-| Co-engagement / collaborative filtering | `tidal/src/entities/co_engagement.rs` |
-| Collections | `tidal/src/entities/collection.rs`, `tidal/src/db/collections.rs` |
-| Saved searches | `tidal/src/session/saved_search.rs` |
-| SUGGEST autocomplete | `tidal/src/query/suggest.rs` |
-| Notification capping | `tidal/src/db/notification_tracker.rs` |
-| Adaptive preference learning rate | `tidal/src/entities/preference.rs` |
-| Sort surface | `tidal/src/ranking/profile.rs` (`Sort`), `tidal/src/ranking/executor/tests/sort_tests.rs` |
-| Benchmarks added by M6 | `tidal/benches/social.rs`, `tidal/benches/sort.rs` |
-
-All 62 M6 acceptance criteria in [ROADMAP · Milestone 6](../ROADMAP.md) are
-marked `[x]`. Recorded at close: 1,082 total (835 lib + 247 integration), 9
-`m6_uat` passing; re-verified 2026-02-24 at 1,206 lib + 70 M6 integration tests.
-
-## Divergences found while writing these records
-
-Three ROADMAP criteria describe behaviour the shipped code deliberately does not
-have. Each is documented in the source itself, and each is recorded in the
-relevant phase doc rather than smoothed over here:
-
-1. Co-engagement eviction is **minimum-weight**, not LRU (phase-2).
-2. The `related` blend is **additive with a 0.3 co-engagement term**, not the
-   convex combination `0.6/0.3/0.1` (phase-2).
-3. Per-user preference `update_counts` were documented as unpersisted and
-   "deferred to M7"; they are in fact persisted now, and by M12, not M7 (phase-6).

docs/planning/milestone-6/phase-1.md

@@ -1,46 +0,0 @@
-# m6p1 — Cohort Engine + Cohort-Scoped Trending (✅ COMPLETE 2026-02-23)
-
-Phase spec and acceptance criteria: [ROADMAP · Milestone 6 · Phase 1](../ROADMAP.md).
-Milestone index: [README.md](README.md). Spec: [docs/specs/05-cohorts.md](../../specs/05-cohorts.md).
-Backfilled record.
-
-## What shipped
-
-1. **Cohort definitions as data** (`tidal/src/cohort/types.rs`). `CohortDef {
-   name, predicate }` with a five-arm `Predicate` enum — `Eq { field, value }`,
-   `Any { field, values }`, `Range { field, lo, hi }` (inclusive, `f64`-parsed),
-   `And(Vec<Self>)`, `Or(Vec<Self>)`. Both types are `Serialize`/`Deserialize`,
-   which is what makes cohort state checkpointable rather than rebuild-only.
-2. **`db.define_cohort(CohortDef)`** (`tidal/src/db/cohorts.rs:19`) registers a
-   named cohort at runtime, not only at schema build time.
-3. **Membership resolution at write time** (`tidal/src/cohort/resolver.rs`).
-   User metadata is evaluated against every registered predicate; the result is
-   cached and invalidated when the user's metadata is rewritten, so a signal
-   write pays a lookup rather than a predicate sweep.
-4. **Per-cohort signal aggregation** (`tidal/src/cohort/ledger.rs` —
-   `CohortSignalLedger`). Same decay / windowed-count / velocity semantics as the
-   global ledger, keyed by cohort as well as entity and signal type, updated on
-   the same write.
-5. **Query surface.** `RetrieveBuilder::cohort(name)` scopes signal reads to the
-   cohort ledger; `cohort_predicate(Predicate)` answers ad-hoc cohorts by
-   resolving matching users at query time (slower, no pre-materialized state).
-   A `cohort_trending` built-in profile ships in `tidal/src/ranking/builtins.rs`.
-6. **Durability** (`tidal/src/cohort/checkpoint.rs`). Cohort ledger state is
-   checkpointed alongside the global ledger and restored on open.
-
-## Evidence
-
-- `tidal/tests/m6_cohort.rs` — 12 `#[test]` functions, including the divergence
-  case the ROADMAP calls for: cohort-scoped trending must differ from global
-  trending when cohort velocity diverges from global velocity.
-- `tidal/tests/m6_crash_surfaces.rs` and `tidal/tests/m7_crash_m6.rs` — M7
-  crash-fences the cohort state this phase introduced.
-- `tidal/src/testing/crash_injector.rs` references `CohortSignalLedger`, i.e.
-  the cohort tier is a first-class target of fault injection, not an untested
-  side structure.
-
-## Note on the "< 50 ms at 500 items / 20 users" criterion
-
-Marked `[x]` in the ROADMAP with no recorded run. The functional criteria are
-proven by `m6_cohort.rs`; the latency figure has no in-repo measurement. Unlike
-M11/M12, this milestone published no `docs/profiling/` record for its own gates.

docs/planning/milestone-6/phase-2.md

@@ -1,59 +0,0 @@
-# m6p2 — Social Graph Extension + Collaborative Filtering (✅ COMPLETE 2026-02-23)
-
-Phase spec and acceptance criteria: [ROADMAP · Milestone 6 · Phase 2](../ROADMAP.md).
-Milestone index: [README.md](README.md). Backfilled record.
-
-## What shipped
-
-1. **Reverse relationship index.** Given a creator, retrieve their inbound
-   follower set (`RoaringBitmap`-backed), maintained on every relationship write
-   and persisted so it survives restart.
-2. **`FilterExpr::social_graph(user_id, depth)`**
-   (`tidal/src/query/executor/social_filter.rs`,
-   `tidal/src/storage/indexes/filter/expr.rs`). Depth 1 constrains candidates to
-   items from followed creators; depth 2 expands through the resolved follow
-   graph. When combined with a trending profile, velocity reads are scoped to the
-   resolved subgraph (`tidal/src/ranking/executor/scoring.rs`).
-3. **Co-engagement index** (`tidal/src/entities/co_engagement.rs`). On a positive
-   engagement (like, or completion ≥ 0.8) pairwise edges are recorded between the
-   engaged item and the user's last `USER_RECENT_CAPACITY = 50` positively
-   engaged items; edge weight increments per co-occurrence. Edges are
-   **asymmetric** — `(A,B)` and `(B,A)` are separate entries, and scoring keys on
-   `(seed_item, candidate)`. Bounded at `DEFAULT_CO_ENGAGEMENT_CAPACITY = 50_000`
-   pairs.
-4. **Collaborative-filtering boost in `related`.** The ranking executor takes an
-   optional `CoEngagementIndex` (`with_co_engagement`,
-   `tidal/src/ranking/executor/mod.rs:212`) and folds a co-engagement term into
-   the score for `related`-style queries, with a `co_engagement` entry in the
-   reason snapshot so the contribution is explainable.
-
-## Evidence
-
-- `tidal/tests/m6_social.rs` — 8 `#[test]` functions.
-- `tidal/benches/social.rs` — social-graph benchmarks added by the same commit;
-  numbers recorded in [docs/profiling/social-scale.md](../../profiling/social-scale.md).
-- `tidal/tests/m7p3_social_scale.rs` — M7 re-tests this surface at scale.
-
-## Divergence from the plan — two, both self-documented in source
-
-1. **Eviction is minimum-weight, not LRU.** The ROADMAP (and the `0.1.0`
-   CHANGELOG entry) say "LRU eviction". The shipped policy is a weight-based
-   batch eviction: when the edge count exceeds capacity, edges are removed in
-   ascending weight order until the count is back at capacity. The module header
-   states the reasoning explicitly and says it "is NOT a true LRU policy" —
-   strongest co-occurrence edges carry the most recommendation value
-   (Sarwar et al., 2001), so signal strength beats recency here. The behaviour is
-   deliberate and better justified than the plan; only the plan's wording is
-   wrong.
-2. **The `related` blend is additive, not a convex combination.** The ROADMAP
-   criterion reads `final = embedding_sim × 0.6 + co_engagement × 0.3 +
-   signal_score × 0.1`. The effective formula is
-   `base_signal_score + boost_sum + co_eng_score × 0.3`
-   (`tidal/src/ranking/executor/mod.rs:775-796`). The comment there explains why:
-   embedding similarity is not available as a per-candidate scoring signal
-   because ANN retrieval uses the embedding for candidate *selection*, not
-   re-scoring. Folding `embedding_sim` in multiplicatively needs per-candidate ANN
-   distances plumbed through the executor context. That plumbing arrived with M12
-   (`for_you` / `related` ANN candidate generation, see
-   [milestone-12/phase-2.md](../milestone-12/phase-2.md)), so the blend is now
-   revisitable — it has not been revisited.

docs/planning/milestone-6/phase-3.md

@@ -1,41 +0,0 @@
-# m6p3 — Full Sort Mode Coverage + Live Content + Engagement Filters (✅ COMPLETE 2026-02-23)
-
-Phase spec and acceptance criteria: [ROADMAP · Milestone 6 · Phase 3](../ROADMAP.md).
-Milestone index: [README.md](README.md). Backfilled record.
-
-## What shipped
-
-1. **The sort surface completed** (`tidal/src/ranking/profile.rs`, `Sort` enum).
-   M6 added `AlphabeticalAsc`, `AlphabeticalDesc`, `Shortest`, `Longest`,
-   `MostCommented { window }`, `MostShared { window }`, `LiveViewerCount`, and
-   `DateSaved` alongside the pre-existing `Hot`, `Trending`, `Rising`,
-   `Controversial`, `HiddenGems`, `Shuffle`, `New`, `TopWindow`, `MostViewed`,
-   `MostLiked`, `MostFollowed`, `CreatorEngagementRate`, and `Ann`. Window
-   parameterization is what lets ~20 variants cover the 27 sort modes in
-   [USE_CASES.md](../../../USE_CASES.md) Appendix B.
-2. **Metadata-driven ordering.** Alphabetical sorts read the `title` metadata
-   field case-insensitively with missing titles last; `Shortest`/`Longest` read
-   `duration` in seconds with missing durations last. `DateSaved` reads the
-   querying user's save timestamp and therefore requires `for_user`.
-3. **Live content.** A `viewer_count` signal (short half-life, no windows, no
-   velocity — it represents a current concurrent count, not an accumulation) plus
-   a `live` built-in profile sorting on `LiveViewerCount`
-   (`tidal/src/ranking/builtins.rs`).
-4. **Engagement and geographic filters.** `FilterExpr::MinSignal` /
-   `MaxSignal` evaluate an AllTime windowed count against a threshold;
-   `FilterExpr::NearLocation { lat, lng, radius_km }` is a Haversine post-filter
-   over item `latitude`/`longitude` metadata — deliberately not index-backed.
-
-## Evidence
-
-- `tidal/tests/m6p3_sorts.rs` (9), `m6p3_filters.rs` (3),
-  `m6p3_edge_cases.rs` (3) — integration coverage.
-- `tidal/src/ranking/executor/tests/sort_tests.rs` — per-variant ordering unit
-  tests, the "at least one test per Sort variant" criterion.
-- `tidal/benches/sort.rs` — sort benchmarks added by the same commit.
-
-## Note on the "< 50 ms at 500 items" criterion
-
-Marked `[x]`; `tidal/benches/sort.rs` exists and targets it, but no run is
-recorded in `docs/`. Measurable on demand (`cargo bench -p tidaldb --bench
-sort`); not currently evidenced.

docs/planning/milestone-6/phase-4.md

@@ -1,44 +0,0 @@
-# m6p4 — User Collections + Watch History + Saved Searches (✅ COMPLETE 2026-02-23)
-
-Phase spec and acceptance criteria: [ROADMAP · Milestone 6 · Phase 4](../ROADMAP.md).
-Milestone index: [README.md](README.md). Backfilled record.
-
-## What shipped
-
-1. **Collections** (`tidal/src/entities/collection.rs`,
-   `tidal/src/db/collections.rs`). `create_collection(owner, name, visibility)`
-   with `Visibility::{Private, Shared, Public}`, plus `add_to_collection`
-   (idempotent), `remove_from_collection`, and `list_collections`. Membership is
-   a `RoaringBitmap` per collection for O(1) checks, persisted to fjall.
-2. **`FilterExpr::InCollection`**
-   (`tidal/src/storage/indexes/filter/expr.rs`, evaluated in
-   `filter/evaluator.rs`) constrains candidates to a collection's bitmap.
-3. **Watch history / in-progress.** The `in_progress` user-state filter selects
-   items with a partial-completion signal, closing the "continue watching"
-   surface.
-4. **Saved searches** (`tidal/src/session/saved_search.rs`). `save_search`,
-   `list_saved_searches`, `retrieve_saved_search` (re-executes with a
-   `created_after` bound so a saved search behaves as a persistent feed), and
-   `delete_saved_search`.
-5. **Cross-session preference aggregation — closes an M4 deferral.** On
-   `close_session`, every entity that received a session signal has its stored
-   content embedding blended into the user's global preference vector
-   (`tidal/src/db/sessions.rs:532-574`). Entities without stored embeddings are
-   skipped, and a dimension mismatch is **logged at WARN rather than silently
-   dropped**, because a silent drop hides a schema/embedding mismatch that
-   quietly degrades personalization. This is the M4 "Deferred to Later
-   Milestones" item "session signal influence on global user preference vector",
-   delivered here rather than deferred further.
-
-## Evidence
-
-- `tidal/tests/m6p4_collections.rs` — 10 `#[test]` functions, incl. persistence
-  across restart.
-- `tidal/tests/m6_crash_surfaces.rs`, `tidal/tests/m7_crash_m6.rs` — collections
-  and saved searches under crash injection.
-- `tidal/tests/session_durability.rs` — the session half of the aggregation path.
-
-## Note on the "< 10 ms `in_collection` at 100 items" criterion
-
-Marked `[x]` with no recorded measurement. Functional behaviour is proven by
-`m6p4_collections.rs`; the latency figure is not evidenced.

docs/planning/milestone-6/phase-5.md

@@ -1,45 +0,0 @@
-# m6p5 — Query Composition + SUGGEST Autocomplete (✅ COMPLETE 2026-02-23)
-
-Phase spec and acceptance criteria: [ROADMAP · Milestone 6 · Phase 5](../ROADMAP.md).
-Milestone index: [README.md](README.md). Backfilled record.
-
-## What shipped
-
-1. **`WithinScope` on SEARCH** (`tidal/src/query/search/types.rs:33-44`) — five
-   variants, exactly as specified:
-
-   | Variant | Candidate set |
-   |---------|---------------|
-   | `Trending { window_hours }` | items above the velocity threshold in the window, from the global ledger |
-   | `CohortTrending { cohort, window_hours }` | same, scoped to a defined cohort (m6p1) |
-   | `Following` | items from creators the querying user follows; requires `for_user` |
-   | `Category { name }` | items matching the category metadata value |
-   | `Collection { id }` | items in the collection's bitmap (m6p4) |
-
-   The scope resolves to a `RoaringBitmap` applied *before* candidate scoring, so
-   it reaches both retrieval arms: a post-filter for Tantivy BM25 and a predicate
-   callback for the USearch ANN search. Scoping before scoring is the point —
-   scoping after would rank the wrong universe and then discard the answer.
-2. **`db.suggest()`** (`tidal/src/query/suggest.rs`,
-   `tidal/src/db/query_ops.rs`). Prefix autocomplete over terms extracted from
-   item titles at write time, updated incrementally on
-   `write_item_with_metadata`; an empty prefix returns trending search terms by
-   recent query frequency, counted on each `db.search()` call.
-
-## Evidence
-
-- `tidal/tests/m6p5_scope.rs` — 13 `#[test]` functions: each `WithinScope`
-  variant independently, the composition case (search within cohort trending),
-  and SUGGEST with both a prefix and an empty prefix.
-- `tidal/src/query/search/types/tests.rs` — `WithinScope` unit coverage.
-
-## Dependencies this phase actually had
-
-`CohortTrending` needs m6p1's cohort ledger and `Collection` needs m6p4's
-collection bitmap, so despite the ROADMAP DAG showing phases 1–4 as broadly
-parallel, m6p5 was genuinely gated on both.
-
-## Note on the "< 50 ms SEARCH / < 20 ms SUGGEST" criteria
-
-Marked `[x]` with no recorded measurement in `docs/`. Functional behaviour is
-proven by `m6p5_scope.rs`; the two latency figures are not evidenced.

docs/planning/milestone-6/phase-6.md

@@ -1,64 +0,0 @@
-# m6p6 — Notification Capping + Adaptive Preferences + Creator Profile Modes + M6 UAT (✅ COMPLETE 2026-02-23)
-
-Phase spec and acceptance criteria: [ROADMAP · Milestone 6 · Phase 6](../ROADMAP.md).
-Milestone index: [README.md](README.md). Backfilled record.
-
-## What shipped
-
-1. **Notification capping** (`tidal/src/db/notification_tracker.rs`).
-   `NotificationCaps { max_per_creator_per_day, max_total_per_day }` attaches to
-   a query via `RetrieveBuilder::notification_caps(caps)` and is enforced as a
-   post-diversity pass, with per-`(user, creator, date)` delivery counts tracked
-   so the cap spans queries rather than only trimming one result page.
-2. **Adaptive preference learning rate** (`tidal/src/entities/preference.rs`).
-   The EMA alpha decays logarithmically with a user's update count:
-
-   ```
-   alpha = base_alpha / (1 + ln(update_count + 1))      // base_alpha default 0.1
-   ```
-
-   At `count = 0` this is exactly `base_alpha`. Early signals move a cold user's
-   vector hard; later signals refine it gently, which is what keeps a
-   well-established taste profile from being yanked by one outlier interaction.
-   Unit coverage for the decay curve lives beside the implementation
-   (`preference.rs` tests, incl. the explicit `count=0 ⇒ alpha=0.1` case).
-3. **Creator profile modes.** `RetrieveBuilder::for_creator(creator_id)` adds the
-   creator filter and restricts candidate generation to that creator's items, so
-   `for_creator(x) + for_you` ranks x's catalogue by the *querying user's*
-   preferences while `for_creator(x) + hot` ranks x's catalogue by heat.
-4. **The M6 UAT** (`tidal/tests/m6_uat.rs`) — 9 `#[test]` functions over a shared
-   fixture, exercising the full use-case surface.
-
-## Evidence
-
-- `tidal/tests/m6_uat.rs` (9) and `tidal/tests/m6p6_creator_profile.rs` (6).
-- Recorded at close in the ROADMAP status row: 1,082 total (835 lib + 247
-  integration), 9 `m6_uat` passing; the same row records that all prior milestone
-  UATs (m2, m3, m4, m5, m5p4) continued to pass — the phase's "no regression"
-  criterion.
-
-## Divergence from the plan
-
-**The "update counts are not persisted" limitation is stale — and it was closed
-by M12, not M7.** `tidal/src/entities/preference.rs:37-45` still carries a
-`# Known Limitation` block saying `update_counts` is in-memory only, that every
-user resets to `count = 0` on restart, and that persisting it is "deferred to M7
-(Production Hardening)". Both halves are now wrong:
-
-- The count **is** persisted. The multi-vector preference checkpoint format
-  serializes it — `[update_count:8 LE][importance_at_anchor:4 LE][anchor_ts:8 LE]`
-  (`tidal/src/entities/multi_preference.rs:127,149`) — with
-  `MultiPreferenceVectors::checkpoint` / `restore`
-  (`multi_preference.rs:642,703`) writing and reading it, and
-  `PreferenceVectors::insert_restored` (`preference.rs:239`) loading a legacy
-  single-vector row back as a cold-start user with its count intact.
-  Round-trip and legacy-row tests exist
-  (`checkpoint_restore_roundtrip_multi_cluster`,
-  `restore_reads_legacy_single_vector_rows_as_cold_start`).
-- The milestone that closed it was **M12** (multi-vector user preference
-  modelling), not M7. See
-  [milestone-12/multi-vector-preference.md](../milestone-12/multi-vector-preference.md).
-
-The stale doc comment is in Rust source and is out of scope for this backfill;
-recorded here so the next reader of that comment does not re-plan work that is
-already done.

docs/runbooks/README.md

@@ -14,7 +14,6 @@ and history live in the `orchard9-k3sf` repo (`cluster-state.yaml`,
 | Back up / restore / DR — object-store export, restore, byte-verify, query-proof, R2, rebuild | [`disaster-recovery.md`](disaster-recovery.md) |
 | Recover a single node / standalone engine — corrupt keyspace, WAL, lock, schema | [`../ops/recovery.md`](../ops/recovery.md) |
 | Size a deployment — single-node tables + the measured Ref-A cluster envelope | [`../ops/capacity-planning.md`](../ops/capacity-planning.md) |
-| Soak before a release — 1000 rps load with p99 / error-rate gates (moved out of CI on 2026-08-30; the shared runner cannot meet a 250ms p99) | [`nightly-soak.md`](nightly-soak.md) |
 | Read the metrics / wire dashboards & alerts | [`../ops/observability.md`](../ops/observability.md), [`../ops/grafana-tidaldb.json`](../ops/grafana-tidaldb.json), [`../ops/prometheus-alerts.yaml`](../ops/prometheus-alerts.yaml) |
 | Understand the live perf/topology findings (dev handoff) | [`../profiling/m12-cluster-deploy-findings.md`](../profiling/m12-cluster-deploy-findings.md) |
 

docs/runbooks/cluster.md

@@ -891,27 +891,14 @@ The `/sharded/*` routes hash-partition entities across regions
 and reads never disagree). Writes route to the single owning region; reads fan out
 to **all** regions and K-way merge by score.
 
-**Writes are SINGLE-COPY and require an explicit opt-in.** A `/sharded/*` write is
-applied to the owning region's LOCAL store with **no WAL append**, so it does not
-ride the leader relay and has **redundancy 1 regardless of the replication
-factor** — RF3 with `ack: quorum` does *not* replicate it. That is the design
-(parallel write throughput across shard owners: §11 measured 3,669 signals/s vs
-~90/s replicated), but the endpoint used to answer `201`/`204` with nothing saying
-so. It now requires `x-tidal-ack: local` and returns **400** without it, naming
-the header and the replicating alternative in the body.
-
 Write routes (each routes to the owning region; a non-owner gateway forwards):
 
 ```bash
-ACK='x-tidal-ack: local'   # the single-copy opt-in; without it every line below is 400
+curl -X POST "$BASE/sharded/items"      -d '{ "entity_id": 7, "metadata": { "title": "..." } }'  # 201
-curl -X POST "$BASE/sharded/items"      -H "$ACK" -d '{ "entity_id": 7, "metadata": { "title": "..." } }'  # 201
+curl -X POST "$BASE/sharded/embeddings" -d '{ "entity_id": 7, "values": [0.1,0.2,0.3,0.4] }'      # 204
-curl -X POST "$BASE/sharded/embeddings" -H "$ACK" -d '{ "entity_id": 7, "values": [0.1,0.2,0.3,0.4] }'      # 204
+curl -X POST "$BASE/sharded/signals"    -d '{ "entity_id": 7, "signal": "view", "weight": 1.0 }'  # 204
-curl -X POST "$BASE/sharded/signals"    -H "$ACK" -d '{ "entity_id": 7, "signal": "view", "weight": 1.0 }'  # 204
 ```
 
-For a **replicated** write use `POST /items` / `/embeddings` / `/signals` (leader
-WAL relay, `x-tidal-ack: leader|quorum`). `local` is rejected on those routes.
-
 Read routes (scatter-gather across all regions):
 
 ```bash
@@ -1634,27 +1621,8 @@ apply per follower. Read it per (pod, kind), not as a fleet-wide subtraction.
 ### 16.3 The decisive test (only after 16.1 and 16.2 are captured)
 
 Confirm `kubectl -n tidaldb-cluster get pdb tidaldb -o jsonpath='{.status.disruptionsAllowed}'`
-is `1`, run a quorum-write probe (`POST /items` with `x-tidal-ack: quorum` → expect
+is `1`, run a quorum-write probe (`POST /sharded/items` → expect 201), then restart
-201), then restart **one follower** that lacks the entity — never the leader.
+**one follower** that lacks the entity — never the leader.
-
-> **The probe MUST go through the replicating surface, and `/sharded/*` can never
-> serve as one.** This step used to say `POST /sharded/items` → expect 201. A
-> `/sharded/*` write is applied to the owning region's local store **with no WAL
-> append**, so it never enters the replication stream and no quorum is ever
-> consulted — its `201` means "one region accepted a single-copy write", which is
-> true whether quorum is intact, degraded, or gone. A green probe therefore
-> carried **no** information about the property it was run to check.
->
-> Do not reintroduce it on the reasoning that it answers faster: speed is exactly
-> what it buys by skipping the WAL append, and skipping the WAL append is precisely
-> what makes it blind. Only a write that appends to the leader WAL and waits for a
-> quorum ack can verify quorum, which is `POST /items|/embeddings|/signals` with
-> `x-tidal-ack: quorum`. (Since the opt-in landed, `/sharded/*` also returns 400
-> without `x-tidal-ack: local` — see §7.)
->
-> This probe was used as the between-step safety check of a staged rolling deploy
-> on 2026-08-30; that run's quorum verification must be treated as never having
-> happened. See `k3s-fleet/cluster-state.yaml`.
 
 - entity becomes retrievable after the rebuild ⇒ its **durable store had it**, only the
   live index was missing it. Data was safe.

docs/runbooks/deploy-verification.md

@@ -6,11 +6,9 @@ Walk this top to bottom. Every command here was executed against the live
 Each check states **what it proves**, the command, and what you should see. If a
 check fails, its **If it fails** line says where to look.
 
-Sections 1–8 verify what is deployed **now**. Section 9 covers operator authority
+Sections 1–8 verify what is deployed **now**. Section 9 covers operator
-plus the one feature still inert on the running **pods** — §9.3, structured logs,
+authority plus the two features that are committed but inert on the running
-which is now requested by the manifest and waits only on a restart. §9.1 and §9.2
+image — kept separate so their absence is not mistaken for a regression.
-were in that list and are now live; the header stays because a section that
-quietly drops its closed items teaches you nothing about how they closed.
 
 ## Run it automatically first
 
@@ -381,18 +379,12 @@ kubectl -n tidaldb-cluster logs tidaldb-0 --tail=20
 
 **Pass:** readable lines, no raw `\x1b[` escape fragments.
 
-> **Today the pods still emit plain text**, so the collector cannot parse the
+> Today's deployed image emits **plain text**, so `level:error` filtering in
-> line, keeps the text, and stamps every tidalDB entry `level=info` — *including
+> VictoriaLogs does **not** work yet — the collector cannot parse the line, keeps
-> its WARNs*. Measured 2026-08-30: `unit:tidaldb-cluster` faceted over 30m
+> the text, and stamps every entry `level=info`. Structured logging is built and
-> returned exactly one row, `{"service":"tidaldb","level":"info","n":"457"}`,
+> tested but requires the image roll and `JSON_LOGS=1`; see §9.3.
-> while `level:error` matched fine for services that do emit JSON (`relay`,
-> `external-secrets`). The collector is not broken. tidalDB was not speaking to
-> it.
->
-> The fix is a deployment change, not a code change: `JSON_LOGS=1` is now on the
-> StatefulSet and takes effect on the next pod restart. See §9.3.
 
-Until that roll lands, filter at the source:
+Until then, filter at the source:
 
 ```bash
 kubectl -n tidaldb-cluster logs tidaldb-0 --since=15m | grep -iE 'ERROR|WARN'
@@ -486,57 +478,25 @@ metadata:
 The cluster runs
 `registry.threesix.ai/tidal/server:m12-vsc-20260830@sha256:5c18d2b1…`
 (rolled 2026-08-30, built from `8aa1fbb`). That image carries the operator/data
-credential split (§9.2), the HTTP request metrics (§9.1), the blob-replication
+credential split (§9.2), the HTTP request metrics (§9.1), and the
-ledger, and the structured-log emitter (§9.3).
+blob-replication ledger.
 
-**§9.1 is LIVE.** Both `09-operator-authority.spec.ts` and the `CAP-015` capture
+**§9.1 went LIVE with this roll.** It was inert for the previous two images, and
-asserted `tidaldb_http_* = 0`; that became false and they failed, which is how
+this document said so — until the harness contradicted it. Both
-the drift surfaced within minutes instead of rotting here. Both assertions are
+`09-operator-authority.spec.ts` and the `CAP-015` capture asserted
-now inverted, so a rollback that removes the HTTP metrics fails them again.
+`tidaldb_http_* = 0`; the roll made that false and they failed, which is how the
-
+drift surfaced within minutes instead of rotting here. Both assertions are now
-One correction to the earlier wording, forced by the revision table below: the
+inverted, so a rollback that removes the HTTP metrics fails them again.
-HTTP metrics and the structured-log emitter came from the **same commit**,
-`4766f56`, so they reached production together at **rev 30**, not at rev 31. The
-`0` readings recorded in this section were taken while rev 29 was running; rev 30
-held the cluster for only 2h21m and was never measured. "Went live with the
-`m12-vsc` roll" is where it was *noticed*, not where it started.
 
 Live reading after the roll: `tidaldb_http_* = 185` against a baseline of
 `tidaldb_* = 552` — the baseline is what proves the scrape worked, so "present"
 is distinguishable from "unscraped".
 
-**§9.3's label was stale, and it is corrected here — with the dates, because the
+**§9.3 remains inert.** The StatefulSet still carries no `JSON_LOGS`, so logs are
-earlier wording was right when written.** "Requires the image roll and
+unstructured plain text and VictoriaLogs `level:error` cannot match them; filter
-`JSON_LOGS=1`" was true up to 2026-08-30T17:38Z. It stopped being true at that
+at the source. Container logs no longer carry ANSI escapes (BUG-006 resolved on
-moment and nobody updated it. The StatefulSet's own revision history is the
+this image), and `06-logs.spec.ts` now pins that in the other direction — a
-record:
+regression to coloured output fails there.
-
-| rev | image | live from | carries the emitter? |
-|---|---|---|---|
-| 29 | `m12-admin-gate-20260823` | 2026-08-23T05:32Z | no — built before `4766f56` |
-| 30 | `m12-vector-grow-20260830` | 2026-08-30T17:38Z | **yes** — `7c1c80d`, which descends from `4766f56` |
-| 31 | `m12-vsc-20260830` | 2026-08-30T19:59Z | **yes** — `8aa1fbb` |
-
-```bash
-kubectl -n tidaldb-cluster get controllerrevision -l app.kubernetes.io/name=tidaldb \
-  -o custom-columns=REV:.revision,WHEN:.metadata.creationTimestamp,IMAGE:'.data.spec.template.spec.containers[0].image'
-git merge-base --is-ancestor 4766f56 8aa1fbb && echo "emitter is in the running image"
-```
-
-So the image half of the requirement has been satisfied through **two** rolls,
-and the env half was never satisfied at all — neither roll added `JSON_LOGS`, and
-this section kept naming the image as the blocker after the image stopped being
-one. The lesson is not "the doc lied"; it is that a gap described by its
-*prerequisite* rather than its *current state* survives the prerequisite being
-met.
-
-`JSON_LOGS=1` and `TIDAL_SERVICE_NAME=tidaldb` are now in
-`k8s/cluster/statefulset.yaml`. **An env change does nothing to already-running
-pods**, so §9.3 stays inert until the next roll restarts them — that is the one
-item here still genuinely pending, and it is pending on a restart, not on code.
-Container logs carry no ANSI escapes (BUG-006 resolved on this image) and
-`06-logs.spec.ts` pins that in the other direction — a regression to coloured
-output fails there.
 
 ### 9.2 Operator/data credential split — LIVE, verify it stays that way
 
@@ -571,90 +531,33 @@ split closes.
 > WARN in the boot log does **not** mean the gate is open — test the behavior,
 > which is why the two curls above are the actual check.
 
-### 9.1 HTTP request/error metrics — LIVE
+### 9.1 HTTP request/error metrics — still inert
-
-Live since **rev 30** (`m12-vector-grow-20260830`, 2026-08-30T17:38Z) — the same
-commit `4766f56` that carried the structured-log emitter. This section claimed
-"still inert" for two images after that stopped being true, the same failure mode
-as §9.3, in the same document. Verify it stays live:
 
 ```bash
-for p in tidaldb-0 tidaldb-1 tidaldb-2; do
+PIP=$(kubectl -n tidaldb-cluster get pod tidaldb-0 -o jsonpath='{.status.podIP}')
-  PIP=$(kubectl -n tidaldb-cluster get pod "$p" -o jsonpath='{.status.podIP}')
+kubectl -n observability exec deploy/vmagent -- \
-  HTTP=$(kubectl -n observability exec deploy/vmagent -- \
+  sh -c "wget -qO- --timeout=15 http://$PIP:9091/metrics | grep -c tidaldb_http_requests_total"
-    sh -c "wget -qO- --timeout=15 http://$PIP:9091/metrics | grep -c tidaldb_http_requests_total")
-  ALL=$(kubectl -n observability exec deploy/vmagent -- \
-    sh -c "wget -qO- --timeout=15 http://$PIP:9091/metrics | grep -c tidaldb_")
-  echo "$p http=$HTTP all=$ALL"
-done
 ```
 
-**Pass:** non-zero `http` on all three. Observed 2026-08-31T02:19Z:
+Currently `0` on all three pods. After the observability image is rolled:
-`http=19/24/18` against `all=821/878/812`. (The preamble's `185` counts every
+non-zero, and the dashboard's *Request rate by route*, *Requests by status*,
-`tidaldb_http_*` line, a wider grep — same conclusion, different denominator.)
+*5xx ratio*, *Auth rejections* and *HTTP p99* panels populate. Until then those
-Always read the baseline too — it is what
+five panels are legitimately empty.
-distinguishes "the metric is missing" from "the scrape returned nothing". The
-dashboard's *Request rate by route*, *Requests by status*, *5xx ratio*, *Auth
-rejections* and *HTTP p99* panels populate from these.
 
 > **Use `--timeout=15`, not 6.** A 6-second `wget` truncates this scrape on a
 > busy node and returns zero lines, which reads exactly like "the metric is
 > missing" — it briefly looked like one pod had stopped exporting entirely.
 
-### 9.3 Structured logs — requested in the manifest, live after the next roll
+### 9.3 Structured logs
 
-The capability is not missing and never was; see the §9 preamble. What was
+Add `JSON_LOGS=1` to the StatefulSet, then:
-missing is the request for it, which is now in source:
 
 ```bash
-kubectl -n tidaldb-cluster get statefulset tidaldb \
+kubectl -n tidaldb-cluster logs tidaldb-0 --tail=5
-  -o jsonpath='{range .spec.template.spec.containers[*].env[*]}{.name}={.value}{"\n"}{end}' \
-  | grep -E 'JSON_LOGS|TIDAL_SERVICE_NAME'
 ```
 
-**Pass:** `JSON_LOGS=1` and `TIDAL_SERVICE_NAME=tidaldb`. Until the roll this
+**Pass:** one JSON object per line carrying `ts`, `level`, `service`, `msg`, and
-prints nothing on the live object while `kubectl kustomize k8s/cluster/` shows
+`request_id` inside a request span. `level:error` then works in VictoriaLogs.
-both — that divergence *is* the pending state, not a defect.
-
-**After the roll — emitter side:**
-
-```bash
-kubectl -n tidaldb-cluster logs tidaldb-0 -c tidaldb --tail=5 | python3 -c "
-import json,sys
-for line in sys.stdin:
-    json.loads(line)
-print('all sampled lines are JSON')"
-```
-
-**Pass:** one JSON object per line carrying `ts`, `level`, `service`, `target`,
-`msg`, and `request_id` inside a request span.
-
-**After the roll — consumer side, which is the check that actually matters:**
-
-```bash
-kubectl -n observability exec deploy/vmagent -- sh -c \
-  'wget -qO- --timeout=25 \
-   --post-data="query=unit:tidaldb-cluster | stats by (service, level) count() n&start=15m" \
-   http://victoria-logs:9428/select/logsql/query'
-```
-
-**Pass:** `service:"tidaldb"` now appears under more than one `level`, and
-`level:error` / `level:warn` return tidalDB lines. The pre-roll reading was a
-single row — `{"service":"tidaldb","level":"info","n":"457"}` — with every WARN
-flattened onto info. **Do not accept the emitter check alone.** The emitter has
-looked correct for weeks while the store still could not filter; that asymmetry
-is precisely how this section stayed wrong.
-
-> **Why `TIDAL_SERVICE_NAME` is set alongside it.** Vector's `normalize`
-> transform (`cm/vector-cluster-config`, ns `observability`) substitutes the
-> container name only `when !exists(.service)`. Plain-text lines have no
-> `service`, so today's land as `tidaldb`; the JSON emitter sets its own, default
-> `tidal-server`. Verified by piping a wire-format line through the cluster's own
-> `vector vrl` running that exact program: with `JSON_LOGS` alone the field flips
-> `tidaldb` → `tidal-server`. `service` is one of four `_stream_fields`, so its
-> value is index identity — an unannounced rename would split tidalDB's log
-> history at the very moment its format changed. Pinning it moves one variable
-> instead of two.
 
 ---
 

docs/runbooks/nightly-soak.md

@@ -1,98 +0,0 @@
-# Soak: pre-release load and regression gate
-
-Run this before tagging a release. It was a `nightly-soak` step in
-`.woodpecker.yaml` until 2026-08-30 and moved here **unchanged** — the commands
-below are the step's commands verbatim, so the coverage survives in a runnable
-form rather than only in git history.
-
-## Why this is not a CI step
-
-The step never actually ran: the `nightly` cron it was gated on was never
-created, so for 216 days it produced zero signal while reading like standing
-coverage.
-
-When the cron was finally configured, this step was **deliberately excluded**,
-and the reason is a measurement rather than a preference. The soak drives
-**1000 rps for 600s** and fails the build if **p99 > 250ms** or **errors > 1%**.
-Measured free CPU on the k3s cluster, 2026-08-30:
-
-| node | allocatable CPU | free CPU (requests) | free memory |
-| --- | --- | --- | --- |
-| `k3s-agent-1` | 4000m | 1700m | 4193Mi |
-| `k3s-server-1` | 3000m | 435m | 2309Mi |
-| `k3s-server-2` | 3000m | 550m | 2257Mi |
-
-A p99 gate of 250ms cannot be met from 1700m of contended CPU shared with
-production tidalDB. The step would fail nightly on **starvation, not
-regression** — a false alarm every morning, which is the fake-coverage defect
-inverted rather than fixed. A gate that cannot distinguish its own failure mode
-from the thing it is watching for is not a gate.
-
-So it runs **here**, by hand, on hardware where its numbers mean something.
-
-## Where to run it
-
-Anywhere with **≥ 2 dedicated cores** and no co-tenant under load. A developer
-workstation qualifies; the shared k3s nodes do not. If you only have the k3s
-cluster, the honest options are to raise its thresholds to match the hardware
-(and say so in the output) or to skip it and record that you skipped it — not to
-run it and read the result as meaningful.
-
-## Run
-
-```bash
-export TIDAL_SOAK_RPS=1000
-export TIDAL_SOAK_SECS=600
-export TIDAL_SOAK_MAX_P99_MS=250
-export TIDAL_SOAK_MAX_ERROR_PCT=1
-# Soak target port — an UNCLAIMED slot in the project's reserved dev band
-# (59520-59529): 59520=site, 59521=iknowyou, so the soak uses 59526.
-export TIDAL_SOAK_PORT=59526
-
-cargo build -p tidal-server -p tidal-stress
-
-PORT="${TIDAL_SOAK_PORT}"
-TARGET="${TIDAL_SOAK_TARGET:-http://127.0.0.1:$PORT}"
-if [ -z "$TIDAL_SOAK_TARGET" ]; then
-  mkdir -p /tmp/soak-data
-  ./target/debug/tidal-server standalone --listen "127.0.0.1:$PORT" \
-    --schema tidal-server/config/default-schema.yaml --data-dir /tmp/soak-data &
-  SRV=$!
-  # Reap the background server + scratch dir on ANY exit (success, gate failure,
-  # or the boot-failure exit below) so a re-run is idempotent.
-  trap 'kill "$SRV" 2>/dev/null; rm -rf /tmp/soak-data' EXIT
-  up=0
-  for i in $(seq 1 100); do
-    curl -sf "$TARGET/health/startup" >/dev/null 2>&1 && { up=1; break; } || sleep 0.3
-  done
-  # Fail FAST and unambiguously on a boot failure (bad schema, port already
-  # bound) rather than soaking a dead target and mislabeling it as an error-rate
-  # regression on the trend line.
-  if [ "$up" != "1" ]; then echo "soak target failed to start at $TARGET"; exit 1; fi
-fi
-
-./target/debug/tidal-stress --target "$TARGET" \
-  --ramp "${TIDAL_SOAK_RPS}:${TIDAL_SOAK_SECS}" --corpus 5000 --mix peach \
-  --json-summary soak-summary.json \
-  --max-error-pct "${TIDAL_SOAK_MAX_ERROR_PCT}" --max-p99-ms "${TIDAL_SOAK_MAX_P99_MS}" --fail-on-knee
-
-cat soak-summary.json
-```
-
-> The `$${VAR}` escaping in the original CI step is **not** needed here.
-> Woodpecker preprocesses a bare `${VAR}` before the shell sees it, so the step
-> had to write `$${VAR}` to pass a literal through. In a plain shell the single
-> `${VAR}` form above is correct.
-
-## What to expect
-
-`soak-summary.json` is the artifact — keep it with the release notes so there is
-a trend line rather than a single opinion. A `--fail-on-knee` failure means
-throughput stopped scaling before the target rps, which is a different finding
-from a p99 breach and worth reporting separately.
-
-## Targeting the live cluster
-
-Setting `TIDAL_SOAK_TARGET` to the live Ref-A cluster and `TIDAL_SOAK_SECS=3600`
-gives the GA-bar 1-hour 100k-DAU soak. **This soaks production at 1000 rps.**
-Do not set it casually, and never as part of an unattended run.

docs/runbooks/on-call.md

@@ -12,49 +12,45 @@ is `orchard9-k3sf/deployments/history/tidaldb.md`; live state is
 
 ---
 
-> ## ✅ RESOLVED — the narrowed client-Service selector is NO LONGER in force
+> ## ⚠ ACTIVE OPERATOR INTERVENTION — read before touching the client Service
 >
-> **Between 2026-08-21 and 2026-08-21T20:10Z** the client Service `tidaldb`
+> **As of 2026-08-21 the client Service `tidaldb` carries a NARROWED selector.**
-> carried an extra `tidaldb.orchard9.ai/serving: "true"` selector label that held
+> `tidaldb-0` serves a shard-1 frontier that was cross-seeded from shard 2's
-> `tidaldb-0` out of the read path, because it served a shard-1 frontier
+> snapshot artifact (see `incident_2026_08_20_reseed_livelock` in
-> cross-seeded from shard 2's snapshot (`incident_2026_08_20_reseed_livelock` in
+> `orchard9-k3sf/cluster-state.yaml`), so its reads are not trustworthy even when
-> `orchard9-k3sf/cluster-state.yaml`). **That intervention was reverted and this
+> it reports `lag_events: 0`. It is held out of the read path by an extra
-> section is history, not instruction.** It is kept because the footgun below is
+> selector label rather than by readiness, because the marker-clear bug lets it
-> worth knowing if anyone ever re-narrows the selector.
+> report Ready while degraded.
->
-> Verified 2026-08-31, three ways:
 >
 > ```bash
+> # what is in force
 > kubectl -n tidaldb-cluster get svc tidaldb -o jsonpath='{.spec.selector}'
-> #   => {"app.kubernetes.io/component":"cluster-node","app.kubernetes.io/name":"tidaldb"}
+> #   => includes tidaldb.orchard9.ai/serving: "true"
-> #      the plain manifest selector; no serving label
+> # only tidaldb-1 and tidaldb-2 carry that label (applied to the PODS, not the template)
-> kubectl -n tidaldb-cluster get endpoints tidaldb \
->   -o jsonpath='{range .subsets[*].addresses[*]}{.targetRef.name}{"\n"}{end}'
-> #   => tidaldb-0, tidaldb-1, tidaldb-2 — all three serving
-> kubectl -n tidaldb-cluster get pods -l app.kubernetes.io/name=tidaldb \
->   -o custom-columns=NAME:.metadata.name,SERVING:'.metadata.labels.tidaldb\.orchard9\.ai/serving'
-> #   => <none> on all three
 > ```
 >
-> `cluster-state.yaml` records the same: *"RESOLVED 2026-08-21T20:10Z (verified,
+> **FOOTGUN — this label does NOT survive pod recreation.** It is on the pods, not
-> not inferred) … Client Service selector reverted to the manifest's original."*
+> the StatefulSet template (adding it to the template would trigger a rolling
->
+> update, and with only two healthy voters that loses quorum). If `tidaldb-1` or
-> **If you ever re-apply that narrowing, the footgun is that the label goes on the
+> `tidaldb-2` is recreated it silently drops out of the client Service. If BOTH
-> PODS, not the StatefulSet template** (templating it triggers a rolling update,
+> are recreated the Service has ZERO endpoints and all reads fail.
-> which with two healthy voters loses quorum). Pod labels do not survive pod
-> recreation, so a recreated pod silently drops out of the client Service, and if
-> every labelled pod is recreated the Service has ZERO endpoints and all reads
-> fail. Symptom and repair:
 >
 > ```bash
-> kubectl -n tidaldb-cluster get endpoints tidaldb          # empty subsets?
+> # SYMPTOM: reads fail / Service has no endpoints
-> kubectl -n tidaldb-cluster label pod tidaldb-1 tidaldb-2 \
+> kubectl -n tidaldb-cluster get endpoints tidaldb
->   tidaldb.orchard9.ai/serving=true --overwrite
+> # REPAIR: re-label whichever healthy pods lost it
+> kubectl -n tidaldb-cluster label pod tidaldb-1 tidaldb-2 tidaldb.orchard9.ai/serving=true --overwrite
 > ```
 >
-> Do not re-narrow on a frontier number alone — a matching `lag_events: 0` is
+> **REVERT once tidaldb-0 genuinely holds every shard** (per-shard `applied_events`
-> exactly what misled the original diagnosis. Require per-shard `applied_events`
+> matching each shard's leader AND a real query agreeing with a healthy peer — a
-> matching each shard's leader AND a real query agreeing with a healthy peer.
+> frontier number alone is what misled us here):
+>
+> ```bash
+> kubectl -n tidaldb-cluster patch svc tidaldb --type=merge \
+>   -p '{"spec":{"selector":{"app.kubernetes.io/name":"tidaldb","app.kubernetes.io/component":"cluster-node"}}}'
+> kubectl -n tidaldb-cluster label pod tidaldb-1 tidaldb-2 tidaldb.orchard9.ai/serving-
+> ```
 
 ---
 
@@ -106,185 +102,6 @@ Golden signals (Grafana "tidalDB Overview" → Cluster Replication row, or `/met
 | **One pod's PVC lost/corrupt** | pod won't open its data dir | Delete that pod's PVC + pod → it reseeds fresh from the live quorum (snapshot install), converges lag=0. Do NOT object-store-restore for a single-pod loss. [`ops/recovery.md`] |
 | **Total cluster loss** | — | Rebuild fresh + `tidalctl restore` each shard before boot. [`disaster-recovery.md`] |
 | **Soak night failed** | `kubectl get jobs -n tidaldb-cluster -l app.kubernetes.io/name=tidal-soak` | A FAIL resets the 30-night streak. Read the night's log for the SLO breach (p99 > 150 ms or error > 1%) or an under-load restart. [`cluster.md` §15] |
-| **401 rate spike / "is someone attacking us?"** | `tidaldb_http_requests_total{status="401"}` for the RATE; the `rejected request` WARN for the REASON | Most 401s here are unauthenticated scanning of a public ingress and are working as designed. Do not page on volume — page on `reason=invalid_token`. Full triage: §2.1 below. |
-
-### 2.1 Auth rejections (401) — triage
-
-**Do not page on 401 volume.** As measured 2026-08-31T02:28Z, **87.5%** of all
-traffic reaching the public ingress is a 401 (281,093 of 321,342 requests over
-~14 days), and it has been that way continuously. Volume is the background rate
-of an internet-facing endpoint being scanned. What is worth waking someone for is
-a *change in kind*, and only the server-side WARN can tell you which kind.
-
-#### Two counters, two different questions — never compare them
-
-| Counter | What it measures | Use it for |
-|---|---|---|
-| `traefik_service_requests_total{service="tidaldb-cluster-tidaldb-9500@kubernetes",code="401"}` | Cumulative since **each Traefik pod** started (both started 2026-08-16T22:26Z) | **History.** Is this normal for this endpoint? |
-| `tidaldb_http_requests_total{status="401"}` | Cumulative since **each tidalDB pod** started (current pods: 2026-08-30T20:00–20:03Z) | **Live rate.** Is it changing right now? |
-
-**The trap:** these two have different epochs and different denominators. A
-change that appears to "cut 401s from 281,000 to 3,600" cut nothing — it
-restarted the pods. Any claim that a restriction worked must compare a *rate*
-before and after, from the same counter, across a pod lifetime that did not
-reset in between.
-
-**Two more traps in the history number itself.** There are **two** Traefik pods
-and each keeps its own counters, so a single-pod scrape undercounts — reading
-only `cxz7f` gives 101,901 where the fleet total is 281,093, a 64% undercount.
-And the metric is `traefik_service_requests_total`, **not**
-`traefik_router_requests_total`: router labels are disabled on this build and
-that metric does not exist here.
-
-```bash
-export KUBECONFIG=~/.kube/orchard9-k3sf.yaml
-
-# HISTORY — sum across BOTH traefik pods
-for p in $(kubectl -n kube-system get pods -l app.kubernetes.io/name=traefik -o name); do
-  IP=$(kubectl -n kube-system get "$p" -o jsonpath='{.status.podIP}')
-  echo "== $p"
-  kubectl -n observability exec deploy/vmagent -- sh -c \
-    "wget -qO- --timeout=20 http://$IP:9100/metrics | grep '^traefik_service_requests_total' | grep tidaldb"
-done
-
-# LIVE RATE — sample the same counter twice and difference it
-sample() {
-  for p in tidaldb-0 tidaldb-1 tidaldb-2; do
-    IP=$(kubectl -n tidaldb-cluster get pod "$p" -o jsonpath='{.status.podIP}')
-    N=$(kubectl -n observability exec deploy/vmagent -- sh -c \
-      "wget -qO- --timeout=20 http://$IP:9091/metrics | awk '/^tidaldb_http_requests_total.*status=\"401\"/{s+=\$2} END{print s+0}'")
-    echo -n "$p=$N "
-  done; echo
-}
-sample; sleep 90; sample
-```
-
-Reference reading, 2026-08-31T02:31–02:32Z (94s): `3640→3657`, `3664→3675`,
-`3144→3161` — **+45 fleet-wide, 28.7/min, ~9.6/min per pod.** That is the
-baseline. Triage a departure from it, not the number itself.
-
-#### The reason field is the whole diagnosis
-
-`unauthorized_response()` in `tidal-server/src/router.rs` is the single funnel for
-every 401 on every surface, and it emits one WARN per rejection carrying `reason`
-and `client` (the token itself is never logged, in either form). Correlate to the
-request via `request_id` on the surrounding span.
-
-| `reason` | `client` | What it means | Page? |
-|---|---|---|---|
-| `invalid_token` | one consistent value, steady rate | **A credential that STOPPED working.** A real client just lost access and is failing right now — almost always a key rotation that missed a consumer. | **YES.** Check the last rotation of `tidaldb-credentials` and find the consumer still holding the old key. |
-| `invalid_token` | many/varied values, bursty | A scanner spraying guessed bearers. Read the caveat below before paging. | No — but confirm it is varied, not one client. |
-| `missing_token` | an external address | Unauthenticated scanning of a public ingress. Expected, correctly rejected, nothing leaked. This is the 87.5% baseline. | **NO.** Never page on this. |
-| `missing_token` | `-` (absent) | **Most likely one of our workloads deployed without its key** — an absent XFF should mean the request never traversed Traefik, i.e. an in-cluster caller. Confirm that reading with the calibration below before relying on it. Not urgent, but the workload is broken until fixed. | No — file it, find the workload, give it `TIDAL_API_KEY`. |
-
-**Read `invalid_token` precisely.** The implementation
-(`unauthorized_response()`, `tidal-server/src/router.rs`) sets it whenever an
-`Authorization` header is **present and did not validate** — it does not, and
-cannot, know whether the token was ever valid. So a scanner sending
-`Authorization: Bearer admin` lands in the page-worthy bucket alongside a genuine
-rotation miss. The discriminator is `client` and the shape of the rate: a
-rotation miss is a *steady* rate from *one* client that starts at a rotation; a
-scanner is bursty and varied. Check both before waking anyone.
-
-**`client="-"` should be a signal, not a gap — confirm it in the same calibration
-run.** The field is `x-forwarded-for` with a `-` default. Traefik sets XFF on
-what it proxies, so an absent value should mean the request **did not come
-through the ingress** — it reached the pod or the ClusterIP Service directly,
-which on this cluster means an in-cluster caller. That is the cleanest
-internal/external split available and, unlike the address itself, it does not
-depend on SNAT fidelity. The three-request test below proves it in passing: if
-those known-external requests come back with a non-`-` `client`, XFF-on-proxied
-holds and `-` can be read as "in-cluster". This has not been observed yet — the
-WARN is not deployed — so do not lean on it until it has.
-
-A sudden move from a steady `missing_token` baseline to `invalid_token` is the
-shape that matters. Volume alone is noise; `reason` is signal.
-
-```bash
-# now: plain text on stdout
-kubectl -n tidaldb-cluster logs tidaldb-0 -c tidaldb --since=15m | grep 'rejected request'
-
-# after the JSON_LOGS roll (deploy-verification.md §9.3), ask the store instead:
-kubectl -n observability exec deploy/vmagent -- sh -c \
-  'wget -qO- --timeout=25 \
-   --post-data="query=_time:15m AND unit:tidaldb-cluster AND _msg:rejected AND _msg:request | stats by (reason, client) count() n" \
-   http://victoria-logs:9428/select/logsql/query'
-```
-
-> **Two things that will bite you here, both already paid for.**
->
-> The message field in LogsQL is **`_msg`**, not `msg` — Vector's sink declares
-> `_msg_field: msg`, so VictoriaLogs renames it on ingest. `msg:rejected` returns
-> nothing forever and looks exactly like "no rejections". Verified 2026-08-31: the
-> identical query shape with a phrase that *does* exist today
-> (`_msg:compaction AND _msg:complete`) returns `{"n":"226"}`, while `msg:` in the
-> same position returns empty.
->
-> Word filters, not a quoted phrase. `--post-data="query=…"` is already inside a
-> double-quoted shell string; a `"rejected request"` inside it terminates the
-> quoting and `wget` fails with `bad address 'request | stats…'`. Two `_msg:`
-> word filters ANDed need no quotes and cannot break.
-
-> **Not yet runnable.** The WARN ships with the next image, so the two commands
-> above are the only ones in §2.1 not yet executed against real data. The LogsQL
-> one runs clean today and returns **nothing** — `stats by (...)` emits no rows
-> when nothing matches, unlike `stats count() n` which returns `{"n":"0"}`. Its
-> *form* is verified; only the content is pending. Everything else in §2.1 was
-> run against the live cluster.
-
-#### Before trusting `client`, calibrate it
-
-`client` carries `x-forwarded-for`, because behind Traefik the socket peer is the
-ingress, not the caller. Its *presence* is the part expected to be reliable (see
-above); its **value**
-is not yet verified on this cluster, and the "an external address" row above
-depends on it. Two measured facts say to check rather than assume: the Traefik
-LoadBalancer Service is `externalTrafficPolicy: Cluster` (so the external source
-IP is SNATed before Traefik ever sees it), and Traefik access logging is off
-(no `--accesslog` argument), so there is no second opinion to compare against.
-
-Calibrate with a request whose origin you know — this exact procedure was run on
-2026-08-31 and the counter deltas below are its real output:
-
-```bash
-# 3 unauthenticated GETs from outside the cluster
-for i in 1 2 3; do
-  curl -s -o /dev/null -w "http=%{http_code} remote=%{remote_ip}\n" \
-    --max-time 15 "https://tidaldb.threesix.ai/search?query=probe"
-done
-#   -> http=401 remote=208.122.204.173
-#      http=401 remote=208.122.204.174
-#      http=401 remote=208.122.204.173
-# /search 401 counters moved 532/526/451 -> 533/527/453: +1 +1 +2 == the 3 requests
-```
-
-Then read the `client` value those three requests produced. If it is the address
-you sent from, the table above works as written. If it is a node IP
-(`208.122.204.172/173/174`) or an svclb pod IP (`10.42.0.13`, `10.42.1.182`,
-`10.42.2.58`), then XFF is carrying the SNAT hop: external callers are no longer
-distinguishable *from each other*, and a scanner will look like it came from your
-own infrastructure. The `client="-"` split still holds in that case — an absent
-XFF still means "never traversed Traefik" — but you lose the ability to
-attribute an external caller. Fix it (Traefik trusted-IP config, or
-`externalTrafficPolicy: Local`) before using the address itself to route
-anything.
-
-#### Corrected: the "all 401s land on tidaldb-0" claim
-
-Earlier notes recorded 3,535 401s "all on `tidaldb-0`" and flagged it as an
-unexplained asymmetry possibly caused by Traefik pinning an endpoint. **Measured
-2026-08-31, that asymmetry does not exist** — the earlier figure was one pod's
-counter read in isolation and generalized. All three pods take the load, at
-within-noise-equal rates:
-
-- Steady state: `tidaldb-0=3640 tidaldb-1=3664 tidaldb-2=3144`, all three climbing
-  (+17 / +11 / +17 over 94s).
-- Controlled test: 3 external requests produced deltas of **+1 / +1 / +2** across
-  the three pods.
-- The Service has all three as endpoints and no pod carries a restricting label
-  (see the RESOLVED banner at the top of this file).
-
-There is nothing to investigate here. Do not spend an incident chasing it.
 
 ---
 

k8s/cluster/statefulset.yaml

@@ -100,7 +100,7 @@ spec:
               mountPath: /data
       containers:
         - name: tidaldb
-          image: registry.threesix.ai/tidal/server:m12-harden-20260831@sha256:accdbad48814919fe9cece14738a904df755f3e7937e1f1d0df07cc9cd5cb9d2
+          image: registry.threesix.ai/tidal/server:m12-vsc-20260830@sha256:5c18d2b10f71d7ed63f776e45a7d4dba2889a52087b2eb11a6509afe0df0f1cf
           imagePullPolicy: IfNotPresent
           # The image ENTRYPOINT is the bare binary. We override the command with
           # a tiny /bin/sh wrapper (the bookworm-slim runtime HAS a shell) so we
@@ -194,43 +194,6 @@ spec:
               value: /etc/tidaldb/admin-key/admin-key
             - name: TIDAL_SERVER_LOG
               value: info
-            # Structured (one-JSON-object-per-line) container logs.
-            #
-            # NOTHING NEW SHIPS FOR THIS. The emitter has existed since 4766f56
-            # (2026-08-23) at tidal-server/src/logging.rs:44-85, and that commit
-            # is an ancestor of 8aa1fbb — the commit the deployed
-            # m12-vsc-20260830 image is built from. The capability was in every
-            # image running here; the DEPLOYMENT simply never asked for it, so
-            # the runbook recorded it for weeks as "inert pending an image roll"
-            # when the only missing piece was this one line. Verify from the
-            # consumer (VictoriaLogs `level:error` matching), not the emitter —
-            # that mislabel is exactly what emitter-side reasoning produces.
-            #
-            # Emitted keys: ts, level (lowercase error|warn|info|debug), service
-            # (TIDAL_SERVICE_NAME, default "tidal-server"), target, msg, plus
-            # event and span fields (request_id inside a request span). `env` is
-            # omitted rather than guessed when TIDAL_ENV is unset.
-            #
-            # An env change does NOT affect already-running pods: this takes
-            # effect on the next pod restart, i.e. the staged roll.
-            - name: JSON_LOGS
-              value: "1"
-            # Pin the `service` stream field across the format change.
-            #
-            # MEASURED, not assumed: the fleet's Vector normalize transform
-            # (cm/vector-cluster-config in ns observability) only falls back to
-            # the container name `when !exists(.service)`. Plain text has no
-            # `service`, so today's lines land as service="tidaldb" — 457 of
-            # them in the last 30m. The JSON emitter DOES set `service`, so
-            # JSON_LOGS alone silently renames the stream to "tidal-server"
-            # (confirmed by running the live VRL program through the cluster's
-            # own `vector vrl` against a wire-format line). `service` is one of
-            # four `_stream_fields` — its value IS the index identity, so an
-            # unannounced rename splits tidalDB's log history at the exact
-            # moment its format changes, i.e. mid-roll. Pinning the existing
-            # value keeps one variable moving instead of two.
-            - name: TIDAL_SERVICE_NAME
-              value: tidaldb
             - name: TIDAL_ALLOW_EXPERIMENTAL_CLUSTER
               value: "1"
             # m12p6: shorten the post-SIGTERM in-flight drain so the (long) HNSW

scripts/restore-fleet.sh

@@ -445,21 +445,9 @@ if [ "$RESTORE_CLUSTER" -eq 1 ]; then
   kubectl -n tidaldb-cluster wait --for=condition=Ready \
     pod -l app.kubernetes.io/name=tidaldb --timeout=300s
 
-  # The soak monitor is NOT started here. It evaluates the nightly endurance
+  printf '==> restoring soak monitor (nightly job stays suspended)\n'
-  # gate, and `--cluster` deliberately leaves that gate suspended — so starting
+  kubectl -n tidaldb-cluster scale deployment/tidal-soak-monitor --replicas=1
-  # the watcher without its producer creates an orphan that reports
+  kubectl -n tidaldb-cluster rollout status deployment/tidal-soak-monitor --timeout=300s
-  # "0/30 consecutive green nights" forever and pages about an intended state.
-  #
-  # That is not hypothetical: this block previously scaled the monitor to 1 while
-  # printing "nightly job stays suspended" in the same breath. The 2026-08-13
-  # reclaim correctly parked BOTH (see the rollback path below, which still
-  # does). The 2026-08-18 `--cluster` restore then brought back only the watcher,
-  # and it ran 13 days alerting continuously about a gate nobody intended to run.
-  #
-  # The pair moves together: the monitor starts in the --soak block, which
-  # already gates on --accept-failed-gate and on the soak-eval capability proof.
-  printf '    soak monitor NOT started: the gate it evaluates is suspended\n'
-  printf '    (use --soak to resume the gate and its monitor together)\n'
 fi
 
 if [ "$RESTORE_STANDALONE" -eq 1 ]; then
@@ -521,11 +509,6 @@ if [ "$RESTORE_SOAK" -eq 1 ]; then
   printf '==> unsuspending nightly soak after evaluator proof\n'
   kubectl -n tidaldb-cluster patch cronjob/tidal-soak-nightly \
     --type=merge -p '{"spec":{"suspend":false}}'
-
-  # Start the watcher only now that its producer is actually running.
-  printf '==> restoring soak monitor (its gate is now unsuspended)\n'
-  kubectl -n tidaldb-cluster scale deployment/tidal-soak-monitor --replicas=1
-  kubectl -n tidaldb-cluster rollout status deployment/tidal-soak-monitor --timeout=300s
 fi
 
 printf '==> final state\n'

tests/e2e/features/06-logs.spec.ts

@@ -243,7 +243,7 @@ test.describe('section 6 — logs', () => {
     }
   });
 
-  test('the deployed image emits structured uncoloured JSON — level filtering now works in the log store', async ({}, testInfo) => {
+  test('the deployed image emits uncoloured plain text — level filtering still belongs at the source', async ({}, testInfo) => {
     const result = await observed(testInfo, 'log format sample', () =>
       kubectl(['-n', NAMESPACE, 'logs', 'tidaldb-0', '--tail=5'], { timeoutMs: 45_000 }),
     );
@@ -266,10 +266,11 @@ test.describe('section 6 — logs', () => {
       jsonLines: jsonLines.length,
       ansiLines: ansiLines.length,
       conclusion:
-        'Structured uncoloured JSON. BUG-006 (ANSI escapes) stays RESOLVED, and JSON_LOGS ' +
+        'Uncoloured plain text. BUG-006 (ANSI escapes in container logs) is RESOLVED on ' +
-        'went live on m12-harden-20260831 — so VictoriaLogs `level:error` finally matches ' +
+        'the deployed image. Logs are still unstructured, so VictoriaLogs `level:error` ' +
-        'and filtering no longer has to happen at the source (runbook 9.3).',
+        'cannot match and filtering stays at the source (runbook 9.3).',
     });
+
     // BUG-006 resolved 2026-08-30. The previous version of this test asserted
     // `ansiLines > 0` — correct for the image running when it was written, and it
     // failed the moment a newer image was rolled. That failure is the test doing
@@ -281,17 +282,11 @@ test.describe('section 6 — logs', () => {
         'breaks log-store level matching and makes every downstream filter guess.',
     ).toBe(0);
 
-    // FLIPPED 2026-08-31, on the roll of m12-harden-20260831. This asserted
+    // Still-open tripwire, unchanged in direction: logs are NOT yet structured.
-    // `jsonLines === 0` and instructed its own inversion once structured logging
-    // shipped — the feature was implemented all along (logging.rs:85); only the
-    // StatefulSet never set JSON_LOGS. Now pinned in the other direction so a
-    // REGRESSION to unstructured output fails here, because the whole log-store
-    // level taxonomy depends on it.
     expect(
       jsonLines.length,
-      'logs stopped being structured JSON — JSON_LOGS has regressed off the StatefulSet. ' +
+      'logs became structured JSON — roll runbook section 9.3 from pending to live and ' +
-        'VictoriaLogs `level:` selectors silently match nothing when this breaks, so a ' +
+        'invert this assertion',
-        '"no errors" dashboard becomes indistinguishable from a healthy cluster.',
+    ).toBe(0);
-    ).toBe(rawLines.length);
   });
 });

tests/e2e/features/07-tidalctl.spec.ts

@@ -33,29 +33,23 @@ test.describe('section 7 — tidalctl live interrogation', () => {
           { timeoutMs: 45_000 },
         ),
       );
-      // Exit 2, not 0, on a cluster whose `shards[]` rows are all converged.
+      // Exit 2, not 0, on a FULLY CONVERGED cluster. This is not a tidalctl
+      // bug in isolation — the aggregated /cluster/status endpoint reports two
+      // of three healthy peers as `region: null, applied_events: 0,
+      // lag_events: 13322235, reachable: false`, while each of those peers'
+      // own /cluster/status/local reports lag=0 and all three agree on the
+      // leader (proven in 01-cluster-convergence.spec.ts). tidalctl correctly
+      // labels the gap `NO REPORT` but still folds it into its degraded
+      // verdict, so the exit code is 2.
       //
-      // This is no longer a fabrication. `/cluster/status` used to report two of
+      // Consequence: `tidalctl cluster-status && deploy` can NEVER pass on this
-      // three healthy peers as `applied_events: 0, lag_events: 13322235` — a
+      // deployment. The runbook claimed it was a safe gate; that claim was
-      // 500ms probe timeout rendered as the leader's entire history as a deficit —
+      // written from an exit code masked by a shell pipeline. See BUG-005.
-      // and tidalctl folded that invented lag into its verdict. Both halves are
-      // fixed: the server reports an unknown frontier as `null`, and tidalctl no
-      // longer treats an unknown as a deficit.
-      //
-      // What remains is honest and is the reason the code is still 2: this node
-      // cannot REACH those peers (`reachable: false`, and therefore
-      // `partitioned: true` — `apply_leader_partition_view` unions the unreachable
-      // set into the ship-skip set). A node that cannot see its peers does not
-      // know they are converged, and a deploy gate must not pass on that. The
-      // remaining defect is the peer HTTP probe failing between healthy pods,
-      // which is upstream of tidalctl entirely.
       expect(
         result.code,
-        'expected exit 2 — this node cannot reach its peers, so it cannot vouch for ' +
+        'expected exit 2 — the aggregated-status gap makes a converged cluster report ' +
-          'them. If this is now 0, the peer HTTP probe between healthy pods was ' +
+          'degraded. If this is now 0, the engine-side peer reporting was fixed: ' +
-          'fixed: update runbook section 7 and mark BUG-005 verified. If the regions ' +
+          'update runbook section 7 and mark BUG-005 verified.',
-          'table shows a NUMERIC lag instead of `?`, that is a real deficit and a ' +
-          'different finding.',
       ).toBe(2);
 
       // The output must still be correct and complete even though the verdict
@@ -66,7 +60,7 @@ test.describe('section 7 — tidalctl live interrogation', () => {
     });
   });
 
-  test('an unknown peer frontier is reported as NO REPORT, never as fabricated lag', async ({}, testInfo) => {
+  test('the aggregated-status gap is reported as NO REPORT, never as fabricated lag', async ({}, testInfo) => {
     await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => {
       const result = await observed(testInfo, 'tidalctl cluster-status regions', () =>
         tidalctl(
@@ -81,17 +75,14 @@ test.describe('section 7 — tidalctl live interrogation', () => {
           { timeoutMs: 45_000 },
         ),
       );
-      // Degraded verdict for the reason pinned in the previous test (this node
+      // Degraded verdict for the reason pinned in the previous test (BUG-005);
-      // cannot reach its peers); what matters here is HOW the gap is reported.
+      // what matters here is HOW the gap is reported, not the exit code.
       expect(result.code, result.stderr).toBe(2);
 
-      // `/cluster/status` reports a peer it holds no frontier report for as
+      // The aggregated endpoint reports a peer it holds no frontier report for
-      // `null`, which tidalctl renders as `?`. Every unknown MUST be labelled
+      // as applied=0 and derives lag against that zero, so a converged peer can
-      // NO REPORT, and — the other direction, which is what stops the marker
+      // read as the leader's entire history behind. tidalctl must name that
-      // being re-inferred — NO REPORT must appear ONLY where the wire said
+      // condition rather than repeat it as lag.
-      // unknown. A numeric `applied=0` is now a MEASURED zero (the PVC-wipe
-      // shape), and labelling that NO REPORT would hide a genuinely empty
-      // replica: the exact inversion of the truth.
       const regionLines = result.stdout
         .split('\n')
         .filter((line) => /applied=/.test(line));
@@ -99,18 +90,11 @@ test.describe('section 7 — tidalctl live interrogation', () => {
 
       expect(regionLines.length, 'expected a line per region').toBeGreaterThan(0);
       for (const line of regionLines) {
-        const unknown = /applied=\?|lag=\?/.test(line);
+        if (/applied=0\b/.test(line)) {
-        if (unknown) {
           expect(
             line,
-            'a peer whose frontier the wire reported as null must be labelled NO REPORT',
+            'a peer with no frontier report must be labelled NO REPORT, not shown as real lag',
           ).toContain('NO REPORT');
-        } else {
-          expect(
-            line,
-            'NO REPORT must be driven off the wire `null`, never inferred from a ' +
-              'measured number — a replica at a real applied=0 is BEHIND, not unreported',
-          ).not.toContain('NO REPORT');
         }
       }
     });

tests/e2e/features/08-backups.spec.ts

@@ -59,29 +59,7 @@ test.describe('section 8 — backups', () => {
         `schedule was renamed or it has never run, and the freshness alert watches this label`,
     ).toBeGreaterThan(0);
 
-    // Pick the newest backup that has actually FINISHED. Selecting `names.at(-1)`
+    const newest = names[names.length - 1];
-    // unconditionally made this test unpassable during the daily backup window:
-    // the schedule fires at 03:30 and measured runs take 9.1–24.8 min, so for
-    // ~25 minutes every day the newest object is legitimately `InProgress` and
-    // this asserted `phase === 'Completed'` against it. That is a false failure by
-    // construction — it says "backups are broken" when a backup is working.
-    // The in-flight case has its own bounded assertion below (see the stuck test).
-    const finished: string[] = [];
-    for (const name of names) {
-      const phaseProbe = await kubectl(
-        ['-n', BACKUP_NAMESPACE, 'get', 'backup.velero.io', name, '-o', 'jsonpath={.status.phase}'],
-        { timeoutMs: 45_000 },
-      );
-      if (phaseProbe.code === 0 && phaseProbe.stdout.trim() !== 'InProgress') {
-        finished.push(name);
-      }
-    }
-    expect(
-      finished.length,
-      'every fleet-schedule backup is still in flight — nothing has ever finished, which is ' +
-        'a real failure rather than a timing artifact',
-    ).toBeGreaterThan(0);
-    const newest = finished[finished.length - 1];
 
     // Guard the selector itself. A canary backup slipping through means the
     // filter regressed and this whole test would be verifying the wrong object.
@@ -166,7 +144,7 @@ test.describe('section 8 — backups', () => {
     ).toEqual(['Completed']);
   });
 
-  test('no Velero backup is in flight beyond the measured completion envelope', async ({}, testInfo) => {
+  test('no Velero backup in the namespace is stuck in progress', async ({}, testInfo) => {
     const result = await observed(testInfo, 'all backup phases', () =>
       kubectl(
         [
@@ -175,52 +153,30 @@ test.describe('section 8 — backups', () => {
           'get',
           'backup.velero.io',
           '-o',
-          'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.phase}{"\\t"}{.status.startTimestamp}{"\\n"}{end}',
+          'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.phase}{"\\n"}{end}',
         ],
         { timeoutMs: 45_000 },
       ),
     );
     expect(result.code, result.stderr).toBe(0);
 
-    // CALIBRATED 2026-08-31 against every fleet-daily run in the namespace:
+    const stuck = result.stdout
-    // successful backups complete in 9.1–24.8 min (n=15), and the observed
-    // FAILURE mode is a hard 240.0 min timeout that lands PartiallyFailed
-    // (2026-08-17/19/25). So "in flight" is normal and "in flight for an hour" is
-    // not. 60 min is ~2.4x the slowest success and a quarter of the timeout.
-    //
-    // The previous assertion was `InProgress → fail`, full stop. The daily fires
-    // at 03:30, so for ~25 minutes every day this test reported the fleet
-    // unprotected while it was actively being protected. A gate that cries wolf on
-    // a schedule gets muted, and then it is not a gate.
-    const STUCK_AFTER_MIN = 60;
-
-    const inFlight = result.stdout
       .trim()
       .split('\n')
       .filter((line) => line.trim() !== '')
       .map((line) => {
-        const [name, phase, startTimestamp] = line.split('\t');
+        const [name, phase] = line.split('\t');
-        const ageMin = startTimestamp
+        return { name, phase };
-          ? (Date.now() - new Date(startTimestamp).getTime()) / 60_000
-          : Number.POSITIVE_INFINITY;
-        return { name, phase, startTimestamp, ageMin: Number(ageMin.toFixed(1)) };
       })
       .filter((backup) => backup.phase === 'InProgress' || backup.phase === 'Deleting');
 
-    const stuck = inFlight.filter((backup) => backup.ageMin > STUCK_AFTER_MIN);
+    await recordJson(testInfo, 'in-flight-backups', stuck);
-
-    await recordJson(testInfo, 'in-flight-backups', {
-      stuckAfterMin: STUCK_AFTER_MIN,
-      inFlight,
-      stuck,
-    });
 
     // A backup wedged InProgress blocks the next scheduled run and silently
     // stops the whole fleet from being protected.
     expect(
       stuck,
-      `backups in flight past ${STUCK_AFTER_MIN} min: ` +
+      `backups stuck in flight: ${stuck.map((b) => `${b.name}=${b.phase}`).join(', ')}`,
-        `${stuck.map((b) => `${b.name}=${b.phase} (${b.ageMin} min)`).join(', ')}`,
     ).toEqual([]);
   });
 });

tests/e2e/features/09-operator-authority.spec.ts

@@ -205,7 +205,7 @@ test.describe('section 9 — operator authority', () => {
     }
   });
 
-  test('structured logging is enabled on the StatefulSet, with the service name pinned', async ({}, testInfo) => {
+  test('structured logging is not yet enabled on the StatefulSet', async ({}, testInfo) => {
     const result = await observed(testInfo, 'statefulset env', () =>
       kubectl(
         [
@@ -228,29 +228,11 @@ test.describe('section 9 — operator authority', () => {
       .filter((line) => line.trim() !== '');
     await recordJson(testInfo, 'statefulset-env', env);
 
-    // FLIPPED 2026-08-31 on the roll of m12-harden-20260831, exactly as the previous
-    // assertion instructed. The feature was never missing — `logging.rs:85` has
-    // implemented it all along; the StatefulSet simply never asked for it, which is
-    // why runbook 9.3 read as a gap for months.
     const jsonLogs = env.find((line) => line.startsWith('JSON_LOGS='));
     expect(
       jsonLogs,
-      'JSON_LOGS has been removed from the StatefulSet. Structured logs are what make ' +
+      'JSON_LOGS is now set — roll runbook section 9.3 from pending to live, verify one JSON ' +
-        'VictoriaLogs `level:` selectors match at all; without it every log-based alert ' +
+        'object per line, and invert this assertion',
-        'and dashboard silently returns nothing.',
+    ).toBeUndefined();
-    ).toBe('JSON_LOGS=1');
-
-    // TIDAL_SERVICE_NAME is load-bearing, not cosmetic: enabling JSON_LOGS makes the
-    // app's own `service` field win in the fleet's Vector normalize transform
-    // (victoria-logs.yaml:230), which would silently rename the log stream from
-    // `tidaldb` to `tidal-server` and blind every query keyed on it. The fleet's
-    // `_stream_fields` contract pins field NAMES but no legal VALUES, so nothing in
-    // k3s-fleet would have caught the flip — `verify-live` now asserts it too.
-    const serviceName = env.find((line) => line.startsWith('TIDAL_SERVICE_NAME='));
-    expect(
-      serviceName,
-      'TIDAL_SERVICE_NAME is unset while JSON_LOGS is on — the log stream will silently ' +
-        'rename itself to tidal-server and every `service:tidaldb` query will return zero.',
-    ).toBe('TIDAL_SERVICE_NAME=tidaldb');
   });
 });

tests/e2e/features/10-ranking-semantics.spec.ts

@@ -398,15 +398,12 @@ test.describe('section 10 — ranking semantics', () => {
       ).toBeLessThan(0.01);
 
       // Rank must be a dense 1..n sequence. This is the assertion that caught the
-      // duplicate-rank defect on the deployed cluster: standalone numbers ranks at
+      // duplicate-rank defect on the deployed cluster, where `scatter_merge`
-      // `tidal/src/query/executor/pipeline.rs:611` and was always dense, while the
+      // returns a merged slice without re-stamping rank
-      // cluster's `scatter_merge` returned a merged slice without re-stamping — the
+      // (`tidal-server/src/cluster/node.rs:7542`). Standalone numbers ranks at
-      // pair of results is what localised the defect to the merge. `scatter_merge`
+      // `tidal/src/query/executor/pipeline.rs:611` and is unaffected — the pair of
-      // now stamps too. This assertion guards the ENGINE half; the merge halves are
+      // results is what localises the defect to the merge.
-      // `mp_ranked_reads_stamp_dense_rank_on_both_merge_shapes`
+      // See 11-ranking-integrity.spec.ts for the cluster half.
-      // (`tidal-server/tests/cluster_sharding.rs`, both the multi-group merge path
-      // and the single-group fast path) and `11-ranking-integrity.spec.ts` (the
-      // deployed cluster).
       expect(
         ranks,
         'rank must be a dense 1..n sequence; duplicates mean a merge did not re-stamp it',

tests/e2e/features/11-ranking-integrity.spec.ts

@@ -1,32 +1,25 @@
 /**
  * Section 11 — ranking integrity on the deployed cluster.
  *
- * `rank` must be a dense ascending `1..n` over the returned page, on every
+ * A tripwire pair, in the honest direction. `rank` is currently WRONG on every
- * corpus-wide ranked response the cluster serves. This is the deployed-artifact
+ * corpus-wide ranked response from the cluster, and these checks pin that with
- * half of the dense-rank contract, which is why it lives here rather than only in
+ * its root cause so the fix announces itself instead of the defect persisting
- * the hermetic suites: the defect it guards existed ONLY under full placement, the
+ * silently. Same shape as the inert-feature checks in
- * production shape.
+ * `09-operator-authority.spec.ts`.
  *
- * ## What this file used to be
+ * Root cause, localised: `scatter_merge`
+ * (`tidal-server/src/cluster/node.rs:7471`) concatenates each shard group's
+ * locally-ranked slice, sorts by score, truncates, and returns at `:7542`
+ * WITHOUT re-stamping rank. Its sibling `merge_cross_shard` (`:7583`) does
+ * re-stamp, at `:7609`, with a comment naming this exact hazard — but `:7621`
+ * documents that full placement short-circuits past it, and this cluster is
+ * full-placement RF3, so the guarded path never runs.
  *
- * A tripwire pair, asserting that `rank` was WRONG and instructing its own
+ * The evidence that it is the MERGE and not the engine is two-sided:
- * deletion once fixed. `scatter_merge` (`tidal-server/src/cluster/node.rs`) sorted
+ *  - standalone numbers ranks densely (`10-ranking-semantics.spec.ts`, measured
- * and truncated the concatenated per-group slices but never re-stamped `rank`,
+ *    1..60 with no gaps) via `tidal/src/query/executor/pipeline.rs:611`;
- * while its sibling `merge_cross_shard` did — and full placement returns before
+ *  - the cluster returns per-group ranks concatenated, while SCORES remain
- * reaching the sibling. Each hosted group ranked its own slice `1..k` locally, so
+ *    correctly ordered — so the merge's sort is fine and only the stamp is absent.
- * live `/search?query=verification` answered `1, 1, 2`.
- *
- * `scatter_merge` now takes the same `set_rank` closure and stamps after
- * `truncate`. The tripwires are gone; this positive assertion replaces them, so
- * the deployed cluster keeps a dense-rank guard instead of the fix silently
- * regressing on the one shape only a real deployment exercises.
- *
- * Coverage split, all three layers:
- *  - engine ranking — `10-ranking-semantics.spec.ts` (hermetic standalone, 1..60);
- *  - BOTH merge shapes — `mp_ranked_reads_stamp_dense_rank_on_both_merge_shapes`
- *    in `tidal-server/tests/cluster_sharding.rs` (real 3-process clusters at
- *    3 groups AND 1 group, so the `[only]` fast path is asserted, not assumed);
- *  - the deployed cluster — this file.
  *
  * Read-only. Nothing here writes to the deployed cluster.
  */
@@ -39,8 +32,8 @@ type RankedItem = { entity_id: number; score: number; rank: number };
 
 /**
  * Enough rows that at least two shard groups must both contribute. With three
- * groups a limit of 1 or 2 can be answered from one group, and a per-group
+ * groups a limit of 1 or 2 can be answered from one group and would show no
- * counter that was never re-stamped would show no duplicate at all.
+ * duplicate at all.
  */
 const LIMIT = 12;
 
@@ -69,56 +62,83 @@ async function ranked(
   };
 }
 
-/**
+/** The order is right even though the stamp is wrong — the precise localisation. */
- * Scores must stay descending. The rank stamp is a renumbering, never a re-sort,
- * so an ordering change here is a DIFFERENT and worse defect than the one this
- * file used to pin: it would mean the merge's sort itself broke.
- */
 function expectScoresOrdered(scores: number[], surface: string): void {
   for (let index = 1; index < scores.length; index += 1) {
     expect(
       scores[index]!,
       `${surface} returned scores out of order at position ${index} ` +
-        `(${scores[index - 1]} then ${scores[index]}). The rank stamp must not re-sort.`,
+        `(${scores[index - 1]} then ${scores[index]}). That is a DIFFERENT and worse ` +
+        `defect than the rank stamp: it would mean the merge's sort is broken too.`,
     ).toBeLessThanOrEqual(scores[index - 1]!);
   }
 }
 
-const REGRESSION_HINT =
+const FIX_INSTRUCTION =
-  'Duplicate or zero ranks are the signature of per-group slices merged without a ' +
+  'Good news: scatter_merge appears to be fixed. Delete this tripwire, keep the ' +
-  're-stamp. Check that scatter_merge still stamps after truncate (and that the ' +
+  'dense-rank assertion in 10-ranking-semantics.spec.ts, and close the defect.';
-  'single-group [only] fast path still returns the engine order).';
 
-/**
+test.describe('section 11 — ranking integrity (cluster tripwires)', () => {
- * The two corpus-wide ranked surfaces. Both return through the same gateway
+  test('cluster /feed rank is duplicated — pinned defect in scatter_merge', async ({
- * merge, so asserting both is what shows the stamp lives in the merge rather than
+    request,
- * in one query pipeline.
+  }, testInfo) => {
- *
+    const { items, ranks, scores } = await ranked(
- * `/search` uses a term known to be indexed on this corpus: the production corpus
+      request,
- * has no titles for its 33k items, so an arbitrary word returns zero candidates
+      `/feed?profile=for_you&limit=${LIMIT}`,
- * and would make the check vacuous.
+    );
- */
+    const duplicates = ranks.length - new Set(ranks).size;
-const RANKED_SURFACES: Record<string, string> = {
-  '/feed': `/feed?profile=for_you&limit=${LIMIT}`,
-  '/search': `/search?query=verification&limit=${LIMIT}`,
-};
 
-test.describe('section 11 — ranking integrity (deployed cluster)', () => {
+    await recordJson(testInfo, 'cluster-feed-ranks', {
-  for (const [surface, path] of Object.entries(RANKED_SURFACES)) {
+      surface: `/feed?profile=for_you&limit=${LIMIT}`,
-    test(`cluster ${surface} returns dense ascending ranks`, async ({ request }, testInfo) => {
+      ranks,
-      const { items, ranks, scores } = await ranked(request, path);
+      expectedIfFixed: denseRanks(ranks.length),
-
+      duplicateCount: duplicates,
-      await recordJson(testInfo, `cluster-ranks${surface.replace('/', '-')}`, {
+      scores,
-        surface: path,
+      entityIds: items.map((item) => item.entity_id),
-        ranks,
+      rootCause: 'tidal-server/src/cluster/node.rs:7542 (scatter_merge returns without set_rank)',
-        expected: denseRanks(ranks.length),
-        duplicateCount: ranks.length - new Set(ranks).size,
-        scores,
-        entityIds: items.map((item) => item.entity_id),
-      });
-
-      expect(ranks, `${surface}: ${REGRESSION_HINT}`).toEqual(denseRanks(ranks.length));
-      expectScoresOrdered(scores, surface);
     });
-  }
+
+    expect(ranks, FIX_INSTRUCTION).not.toEqual(denseRanks(ranks.length));
+    expect(
+      duplicates,
+      'expected duplicate ranks — the signature of per-group slices merged without ' +
+        `a re-stamp. ${FIX_INSTRUCTION}`,
+    ).toBeGreaterThan(0);
+
+    // Ordering is correct; only the stamp is missing. If this ever fails, the
+    // defect has become materially worse.
+    expectScoresOrdered(scores, '/feed');
+  });
+
+  test('cluster /search rank is duplicated the same way, from the same merge', async ({
+    request,
+  }, testInfo) => {
+    // A term known to be indexed on this corpus. The production corpus has no
+    // titles for its 33k items, so an arbitrary word returns zero candidates and
+    // would make this check vacuous.
+    const { items, ranks, scores } = await ranked(
+      request,
+      `/search?query=verification&limit=${LIMIT}`,
+    );
+    const duplicates = ranks.length - new Set(ranks).size;
+
+    await recordJson(testInfo, 'cluster-search-ranks', {
+      surface: `/search?query=verification&limit=${LIMIT}`,
+      ranks,
+      expectedIfFixed: denseRanks(ranks.length),
+      duplicateCount: duplicates,
+      scores,
+      entityIds: items.map((item) => item.entity_id),
+    });
+
+    // `/search` shares the same gateway merge, so the same defect surfaces here.
+    // Asserting it on both surfaces is what shows the fault is in the merge
+    // rather than in one query pipeline.
+    expect(ranks, FIX_INSTRUCTION).not.toEqual(denseRanks(ranks.length));
+    expect(
+      duplicates,
+      `expected duplicate ranks on /search too. ${FIX_INSTRUCTION}`,
+    ).toBeGreaterThan(0);
+    expectScoresOrdered(scores, '/search');
+  });
 });

tidal-server/src/cluster/node.rs

@@ -5116,7 +5116,7 @@ async fn cluster_auth_middleware(
     next: Next,
 ) -> Response {
     if !creds.authenticated(req.headers()) {
-        return crate::router::unauthorized_response(req.headers());
+        return crate::router::unauthorized_response();
     }
     let marked =
         super::forward::is_internal(req.headers()) || super::forward::is_relayed(req.headers());
@@ -5452,37 +5452,21 @@ pub struct AggregatedStatusResponse {
 }
 
 /// One region's aggregated status within an [`AggregatedStatusResponse`].
-///
-/// `applied_events` / `lag_events` are `Option<u64>` because the aggregating node
-/// frequently CANNOT know a peer's frontier: its status probe can fail, and only
-/// the leader holds an ack record to fall back on. Before this was
-/// representable the `None` arm of [`aggregate_region_row`] returned
-/// `applied_events: 0, lag_events: leader_last_seq` — and since `lag` is
-/// `leader_hwm − applied`, "the probe timed out" was rendered as a 13.3-million
-/// event deficit on a cluster whose own `shards[]` rows showed every replica
-/// converged at an identical frontier. `null` is the honest answer; `0` is the
-/// most alarming possible lie.
 #[derive(Serialize, ToSchema)]
 pub struct AggregatedRegionStatus {
     /// Region name.
     name: String,
-    /// Replication events applied on this region (from its own local status; the
+    /// Replication events applied on this region (from its local status; the
-    /// leader row mirrors today's semantics — its replication-from-others count),
+    /// leader row mirrors today's semantics — its replication-from-others count).
-    /// or **`null` when this node has no report for it**. `null` means UNKNOWN —
+    applied_events: u64,
-    /// never read it as 0.
+    /// Events this region lags the leader by: `leader_last_seq − applied`. For an
-    applied_events: Option<u64>,
+    /// unreachable region this is the worst-case `leader_last_seq`.
-    /// Events this region lags the leader by (`leader_last_seq − applied_events`),
+    lag_events: u64,
-    /// or **`null` when either side of that subtraction is unknown** (no frontier
-    /// report for the region, or the leader itself unreachable so there is no
-    /// high-water-mark to subtract from). `null` is NOT `0`: it means this node
-    /// cannot say, and an operator must query the region directly.
-    lag_events: Option<u64>,
     /// Whether this region is currently partitioned from the leader.
     partitioned: bool,
     /// Whether this node could reach the region's `/cluster/status/local` within
-    /// the per-peer budget. An unreachable region reports `applied_events: null`,
+    /// the per-peer budget. An unreachable region reports `applied 0`, `lag =
-    /// `lag_events: null`, `partitioned: true` — the honest unknown, never a
+    /// leader_last_seq`, `partitioned: true`.
-    /// fabricated worst case.
     reachable: bool,
     /// The region's reported build version (m11p8): `/cluster/status` is the
     /// single pane an operator reads to confirm the whole cluster is within the
@@ -5497,13 +5481,9 @@ pub struct AggregatedRegionStatus {
 /// Queries every region's `/cluster/status/local` (own region in-process, peers
 /// over HTTP) concurrently with a 500ms per-peer budget, then assembles the
 /// cluster-wide view: `relay_log_len` is the leader's `last_seq`, and each
-/// region's `lag_events = leader_last_seq − applied` when BOTH are known.
+/// region's `lag_events = leader_last_seq.saturating_sub(applied)`. An
-///
+/// unreachable peer is honestly reported as `reachable: false`, `partitioned:
-/// An unreachable peer is reported as `reachable: false`, `partitioned: true`,
+/// true`, `applied 0`, `lag = leader_last_seq` (worst-case).
-/// `applied_events: null`, `lag_events: null` — an explicit UNKNOWN. It used to
-/// report `applied 0` / `lag = leader_last_seq`, which turned a 500ms probe
-/// timeout into a fabricated multi-million-event deficit; nothing here
-/// reconstructs a guess when the value is not known.
 ///
 /// m11p6: the flat fields describe the DEFAULT (lowest-id hosted) group only —
 /// exact for `S=1`. `shards` carries this node's per-group leadership so a
@@ -5573,14 +5553,13 @@ pub async fn cluster_status(
 
     // The leader's last_seq is the high-water-mark every region converges to.
     // Source it from the leader's local status row (or this node's own when it
-    // leads). `None` when the leader itself was unreachable — there is then NO
+    // leads). Fall back to 0 if the leader was unreachable.
-    // high-water-mark, so every row's `lag_events` is `null` rather than a
+    let leader_last_seq = results
-    // deficit measured against a zero nobody reported.
-    let leader_last_seq: Option<u64> = results
         .iter()
         .find(|(_, name, _)| name == &leader_name)
         .and_then(|(_, _, json)| json.as_ref())
-        .and_then(|j| j.get("last_seq").and_then(serde_json::Value::as_u64));
+        .and_then(|j| j.get("last_seq").and_then(serde_json::Value::as_u64))
+        .unwrap_or(0);
 
     // Leader-side gRPC liveness, for the false-partition fix: under a sustained
     // quorum write-burst (large 1536-dim HNSW inserts), a follower's HTTP
@@ -5615,10 +5594,7 @@ pub async fn cluster_status(
                 && state
                     .transport
                     .peer_grpc_fresh(shard_of_region(rid), forward::GRPC_LIVENESS_WINDOW);
-            // `None` when the leader holds no mark for this peer yet (and always
+            let leader_mark = peer_marks.get(&shard_of_region(rid)).copied().unwrap_or(0);
-            // off the leader, where the map is empty): an absent mark is an
-            // unknown frontier, never `applied 0`.
-            let leader_mark = peer_marks.get(&shard_of_region(rid)).copied();
             aggregate_region_row(
                 name,
                 json.as_ref(),
@@ -5637,11 +5613,7 @@ pub async fn cluster_status(
 
     Ok(Json(AggregatedStatusResponse {
         leader: leader_name,
-        // `0` here means the leader's own status probe failed, so its
+        relay_log_len: leader_last_seq,
-        // high-water-mark is unknown. Nothing DERIVES from that zero: every
-        // region's `lag_events` is `null` in that case (see
-        // `aggregate_region_row`), which is the fabrication that mattered.
-        relay_log_len: leader_last_seq.unwrap_or(0),
         regions: region_rows,
         // Per-hosted-group leadership for THIS node — the multi-shard view the
         // flat `leader` cannot express (S=1 ⇒ one row mirroring the flat fields).
@@ -5653,28 +5625,18 @@ pub async fn cluster_status(
 /// reply (`json`), or from the leader's own view when that probe failed.
 ///
 /// Three cases:
-/// 1. **`json` present** — the peer answered ABOUT ITSELF, which is an honest
+/// 1. **`json` present** — the peer answered: copy its applied/version, derive lag
-///    source whichever node is aggregating: copy its applied/version, derive lag
+///    from the leader HWM, and mirror its self-reported partition flag.
-///    from the leader HWM, and mirror its self-reported partition flag. A reply
-///    that omits `applied_events` yields `None`, not `0`.
 /// 2. **`json` absent but `grpc_fresh`** — the HTTP probe timed out yet the leader
 ///    has had a recent gRPC round-trip (accepted ship OR backpressure) with the
 ///    peer: it is SLOW-but-alive (its HTTP control-plane is starved under an apply
-///    burst), so report `reachable: true`, `partitioned: false`, with an HONEST
+///    burst), so report `reachable: true`, `partitioned: false`, with an HONEST lag
-///    frontier from the leader's own ack mark (`leader_mark`) — and `None` when the
+///    from the leader's own ack mark (`leader_mark`) — never worst-case. This is the
-///    leader holds no mark for it. Only the leader ever has marks, so a FOLLOWER
+///    apply-burst false-partition fix.
-///    never reaches this arm (`grpc_fresh` is gated on `leader_view`): a follower
-///    has no ack record and therefore never guesses from one.
 /// 3. **`json` absent and not `grpc_fresh`** — no reply and no recent gRPC contact:
-///    a genuine partition/dead peer, or simply a peer this node cannot see. Report
+///    a genuine partition/dead peer. Report `reachable: false`, `partitioned: true`,
-///    `reachable: false`, `partitioned: true`, and `applied_events`/`lag_events`
+///    `lag = leader_last_seq` (worst-case). A real TCP severance kills BOTH the HTTP
-///    **`null`** — the honest unknown. This arm used to return
+///    probe and the gRPC ship, so it lands here — preserving the chaos-suite contract.
-///    `applied_events: 0, lag_events: leader_last_seq`, which rendered a 500ms
-///    probe timeout as a multi-million-event deficit on a converged cluster.
-///
-/// `lag_events` is `Some` only when BOTH sides of the subtraction are known: an
-/// unknown frontier or an unreachable leader (no HWM) both yield `null`, never a
-/// number derived from a value nobody reported.
 ///
 /// `apply_leader_partition_view` runs after this and re-stamps `partitioned` from
 /// the authoritative ship-skip set, so an operator `/cluster/partition` always wins
@@ -5683,27 +5645,21 @@ fn aggregate_region_row(
     name: String,
     json: Option<&serde_json::Value>,
     leader_name: &str,
-    leader_last_seq: Option<u64>,
+    leader_last_seq: u64,
     grpc_fresh: bool,
-    leader_mark: Option<u64>,
+    leader_mark: u64,
 ) -> AggregatedRegionStatus {
-    /// `leader_hwm − frontier`, or `None` when either side is unknown.
-    fn lag_from(leader_hwm: Option<u64>, frontier: Option<u64>) -> Option<u64> {
-        frontier
-            .zip(leader_hwm)
-            .map(|(applied, hwm)| hwm.saturating_sub(applied))
-    }
-
     match json {
         Some(j) => {
-            // The peer's report about its OWN frontier. Absent ⇒ unknown.
+            let applied = j
-            let applied = j.get("applied_events").and_then(serde_json::Value::as_u64);
+                .get("applied_events")
+                .and_then(serde_json::Value::as_u64)
+                .unwrap_or(0);
             let is_leader = name == leader_name;
             let lag = if is_leader {
-                // The leader is by definition at its own high-water-mark.
+                0
-                Some(0)
             } else {
-                lag_from(leader_last_seq, applied)
+                leader_last_seq.saturating_sub(applied)
             };
             let partitioned = j
                 .get("partitioned")
@@ -5732,17 +5688,16 @@ fn aggregate_region_row(
         None if grpc_fresh => AggregatedRegionStatus {
             name,
             applied_events: leader_mark,
-            lag_events: lag_from(leader_last_seq, leader_mark),
+            lag_events: leader_last_seq.saturating_sub(leader_mark),
             partitioned: false,
             reachable: true,
             version: String::new(),
         },
-        // No reply, no recent gRPC contact: this node cannot see the peer. It says
+        // No reply, no recent gRPC contact: genuinely unreachable.
-        // so, and says NOTHING about the peer's frontier.
         None => AggregatedRegionStatus {
             name,
-            applied_events: None,
+            applied_events: 0,
-            lag_events: None,
+            lag_events: leader_last_seq,
             partitioned: true,
             reachable: false,
             version: String::new(),
@@ -7513,17 +7468,10 @@ async fn broadcast_to_peers<B: serde::Serialize + Sync>(
 /// pre-m11p6 single read. The cross-group merge keeps each group's own diversity
 /// pass but does NOT re-diversify across groups (a cross-shard re-rank is the L4
 /// follow-up; disjoint groups make the score-merge sound for cardinality).
-///
-/// `set_rank` re-stamps the 1-based page rank over the merged order (see the
-/// stamp site below). It mirrors [`merge_cross_shard`]'s parameter of the same
-/// name so there is exactly ONE rank-stamping mechanism in the file, and it is a
-/// no-op for result types with no rank field. The `S=1` fast path deliberately
-/// bypasses it: the engine's own result is already densely ranked `1..k`.
 async fn scatter_merge<T, F>(
     dbs: Vec<Arc<TidalDb>>,
     limit: usize,
     score: impl Fn(&T) -> f64 + Send,
-    set_rank: impl Fn(&mut T, usize) + Send,
     per_db: F,
 ) -> std::result::Result<(Vec<T>, usize), ServerError>
 where
@@ -7591,16 +7539,6 @@ where
             .unwrap_or(std::cmp::Ordering::Equal)
     });
     merged.truncate(limit);
-    // Re-stamp the 1-based page rank over the MERGED order. Each hosted group
-    // ranks its own slice locally, so without this the wire `rank` is per-group
-    // counters concatenated (observed live: 1,1,2 for a 3-item /search).
-    // `merge_cross_shard` does the same thing for the PARTIAL-placement path;
-    // full placement returns from HERE, which is why it was missed. `set_rank`
-    // is a no-op for result types that carry no rank field (e.g. vector
-    // matches), exactly as in `merge_cross_shard`.
-    for (i, item) in merged.iter_mut().enumerate() {
-        set_rank(item, i + 1);
-    }
     Ok((merged, total))
 }
 
@@ -7938,7 +7876,6 @@ pub async fn feed(
         dbs,
         limit,
         |it: &tidaldb::query::RetrieveResult| it.score,
-        |it: &mut tidaldb::query::RetrieveResult, rank| it.rank = rank,
         move |db: Arc<TidalDb>| {
             let r = db.retrieve(&retrieve).map_err(ServerError::Tidal)?;
             Ok((r.items, r.total_candidates))
@@ -8134,7 +8071,6 @@ pub async fn search(
         dbs,
         limit as usize,
         |it: &tidaldb::query::SearchResultItem| it.score,
-        |it: &mut tidaldb::query::SearchResultItem, rank| it.rank = rank,
         move |db: Arc<TidalDb>| {
             db.reload_text_index().map_err(ServerError::Tidal)?;
             let r = db.search(&search_query).map_err(ServerError::Tidal)?;
@@ -8279,9 +8215,6 @@ pub async fn vector_search(
         // Distance is "lower = better"; scatter_merge ranks by "higher =
         // better", so the merge key is the negated distance.
         |r: &tidaldb::storage::vector::VectorSearchResult| -f64::from(r.distance),
-        // Vector matches carry no rank field (ordered by distance on the wire),
-        // so the stamp is a genuine no-op — never a panic.
-        |_r: &mut tidaldb::storage::vector::VectorSearchResult, _rank| {},
         move |db: Arc<TidalDb>| {
             let r = db
                 .vector_search_items(&vector, k, ef_search)
@@ -8770,70 +8703,9 @@ fn http_shard_context(
     )))
 }
 
-/// The `x-tidal-ack` value a `/sharded/*` WRITE must carry to opt in to
-/// single-copy durability.
-///
-/// Deliberately NOT an [`AckMode`] variant: `local` describes a surface that
-/// never appends to the WAL, so it is meaningless on the replicating routes and
-/// `AckMode::parse` must keep rejecting it there ("must be leader or quorum").
-/// Same header, one name, no second durability knob.
-pub(super) const ACK_LOCAL: &str = "local";
-
-/// Gate a `/sharded/*` WRITE on an EXPLICIT single-copy opt-in.
-///
-/// The surface hash-partitions and applies to the owning region's LOCAL store
-/// with no WAL append (see [`sharded_write_route`]), so its data has redundancy
-/// 1 no matter what the replication factor is. It answered `201`/`204` with
-/// nothing at the call site, in the response, or in the `OpenAPI` saying so — and a
-/// caller on a cluster configured `ack: quorum` with RF3 reasonably assumes their
-/// write replicated. An operator probing with `/sharded/embeddings` found each
-/// one on exactly one of three nodes and filed a durability incident that had to
-/// be retracted.
-///
-/// So: reject (400) unless the caller said `x-tidal-ack: local`. Never silently
-/// accept, and never silently reroute to the replicating path — a reroute would
-/// change the write's performance characteristics under the caller's feet, which
-/// is its own bandaid.
-///
-/// The error names the header AND the replicating alternative, derived from the
-/// route path so the two cannot drift.
-///
-/// `pub(super)` so `cluster::routes` (the single-process cluster's copy of these
-/// three routes) enforces the SAME gate from the SAME definition. One URL, one
-/// contract: a client must not have to know the server's process topology to know
-/// whether its write replicated.
-pub(super) fn require_local_ack(headers: &HeaderMap, path: &str) -> Result<()> {
-    let header = forward::ACK_HEADER;
-    let value = headers.get(header).map(|v| v.to_str());
-    let got = match value {
-        Some(Ok(ACK_LOCAL)) => return Ok(()),
-        None => "the header was absent".to_owned(),
-        Some(Ok(v)) => format!("got {v:?}"),
-        Some(Err(_)) => "got a non-ASCII value".to_owned(),
-    };
-    let replicating = path.strip_prefix("/sharded").unwrap_or(path);
-    Err(ServerError::BadRequest(format!(
-        "{path} applies the write to the owning region's LOCAL store with no WAL \
-         append, so it is SINGLE-COPY regardless of the replication factor. Send \
-         \"{header}: {ACK_LOCAL}\" to opt in to that, or POST {replicating} \
-         instead for a replicated write (leader WAL relay; \
-         \"{header}: leader|quorum\"). Rejected: {got}."
-    )))
-}
-
 /// Forward a sharded WRITE to the owning region (marker set) when this node is
 /// not the owner; else apply locally. Returns the relayed response, or the local
 /// status on a local apply.
-///
-/// The ONE funnel every `/sharded/*` write passes through, and therefore where
-/// the single-copy opt-in is enforced ([`require_local_ack`]) — once, for all
-/// three routes, before anything is applied or forwarded.
-///
-/// An INTERNAL request is exempt: the marker means a verified cluster sibling
-/// already forwarded this write, and `cluster_auth_middleware` rejects the marker
-/// from anyone without a valid node token, so an external caller cannot use it to
-/// slip past the gate. Gating the forwarded leg too would reject the owner's own
-/// hop, since the forward carries the auth + node token, not the caller's headers.
 async fn sharded_write_route<B: serde::Serialize + Sync>(
     state: &Arc<ShardReplica>,
     headers: &HeaderMap,
@@ -8843,14 +8715,10 @@ async fn sharded_write_route<B: serde::Serialize + Sync>(
     local_apply: impl FnOnce() -> Result<()> + Send + 'static,
     success: StatusCode,
 ) -> std::result::Result<Response, ClusterAppError> {
-    let internal = is_internal(headers);
-    if !internal {
-        require_local_ack(headers, path).map_err(ClusterAppError)?;
-    }
     let shards = sharded_region_ids(state);
     let owner = entity_shard(EntityId::new(entity_id), &shards);
     // Owner is self, or this is an internal (already-forwarded) write → apply local.
-    if owner == state.region || internal {
+    if owner == state.region || is_internal(headers) {
         offload_region_read(local_apply).await?;
         return Ok(success.into_response());
     }
@@ -8885,21 +8753,14 @@ async fn sharded_write_route<B: serde::Serialize + Sync>(
 }
 
 /// `POST /sharded/items` — route to the owning region (engine `ShardRouter` hash).
-///
-/// **SINGLE-COPY.** The write is applied to the owning region's LOCAL store with
-/// NO WAL append, so it does not ride the leader relay and is not replicated —
-/// redundancy 1 regardless of the replication factor. That is by design (parallel
-/// write throughput across shard owners), which is why the surface requires
-/// `x-tidal-ack: local` as an explicit acknowledgement of the tradeoff. For a
-/// replicated write use `POST /items`.
 #[utoipa::path(
     post,
     path = "/sharded/items",
     tag = "sharded",
     request_body = ItemRequest,
     responses(
-        (status = 201, description = "Item written SINGLE-COPY to its owning region's local store (no WAL append, not replicated)"),
+        (status = 201, description = "Item written to its owning region"),
-        (status = 400, description = "Invalid request, or the required `x-tidal-ack: local` single-copy opt-in is missing (use POST /items for a replicated write)"),
+        (status = 400, description = "Invalid request"),
         (status = 401, description = "Missing or invalid API key"),
         (status = 503, description = "Owning region unreachable"),
     ),
@@ -8928,20 +8789,14 @@ pub async fn sharded_create_item(
 }
 
 /// `POST /sharded/embeddings` — route to the owning region.
-///
-/// **SINGLE-COPY.** Applied to the owning region's LOCAL store via
-/// `ShardReplica::apply_embedding_local`, which performs no WAL append and so
-/// ships nothing to peers — redundancy 1 regardless of the replication factor.
-/// Requires `x-tidal-ack: local` as an explicit acknowledgement of the tradeoff.
-/// For a replicated write use `POST /embeddings`.
 #[utoipa::path(
     post,
     path = "/sharded/embeddings",
     tag = "sharded",
     request_body = EmbeddingRequest,
     responses(
-        (status = 204, description = "Embedding written SINGLE-COPY to its owning region's local store (no WAL append, not replicated)"),
+        (status = 204, description = "Embedding written to its owning region"),
-        (status = 400, description = "Invalid request, or the required `x-tidal-ack: local` single-copy opt-in is missing (use POST /embeddings for a replicated write)"),
+        (status = 400, description = "Invalid request"),
         (status = 401, description = "Missing or invalid API key"),
         (status = 503, description = "Owning region unreachable"),
     ),
@@ -8969,21 +8824,17 @@ pub async fn sharded_write_embedding(
     .await
 }
 
-/// `POST /sharded/signals` — route to the owning region.
+/// `POST /sharded/signals` — route to the owning region. The signal is applied
-///
+/// to the owner's LOCAL store (the `/sharded/*` surface hash-partitions; it does
-/// **SINGLE-COPY.** The signal is applied to the owner's LOCAL store (the
+/// NOT ride the leader WAL relay — that is the non-sharded `/signals` surface).
-/// `/sharded/*` surface hash-partitions; it does NOT ride the leader WAL relay —
-/// that is the non-sharded `/signals` surface), so it has redundancy 1 regardless
-/// of the replication factor. Requires `x-tidal-ack: local` as an explicit
-/// acknowledgement of the tradeoff. For a replicated write use `POST /signals`.
 #[utoipa::path(
     post,
     path = "/sharded/signals",
     tag = "sharded",
     request_body = SignalRequest,
     responses(
-        (status = 204, description = "Signal written SINGLE-COPY to its owning region's local store (no WAL append, not replicated)"),
+        (status = 204, description = "Signal written to its owning region"),
-        (status = 400, description = "Invalid request, or the required `x-tidal-ack: local` single-copy opt-in is missing (use POST /signals for a replicated write)"),
+        (status = 400, description = "Invalid request"),
         (status = 401, description = "Missing or invalid API key"),
         (status = 503, description = "Owning region unreachable"),
     ),
@@ -9122,87 +8973,16 @@ where
     offload_read(f).await.map_err(ClusterAppError)
 }
 
-#[cfg(test)]
-#[allow(clippy::unwrap_used)]
-mod sharded_optin_tests {
-    //! The `/sharded/*` single-copy opt-in gate. A caller must not be able to
-    //! write redundancy-1 data by accident: the surface answered `201`/`204` with
-    //! nothing saying so, and a live probe through it was filed as a durability
-    //! incident that had to be retracted.
-    use super::{ACK_LOCAL, require_local_ack};
-    use crate::cluster::forward::ACK_HEADER;
-    use reqwest::header::{HeaderMap, HeaderValue};
-
-    fn ack(value: &str) -> HeaderMap {
-        let mut h = HeaderMap::new();
-        h.insert(ACK_HEADER, HeaderValue::from_str(value).unwrap());
-        h
-    }
-
-    #[test]
-    fn the_explicit_optin_is_accepted() {
-        assert!(require_local_ack(&ack(ACK_LOCAL), "/sharded/items").is_ok());
-    }
-
-    #[test]
-    fn a_missing_header_is_rejected_naming_the_header_and_the_alternative() {
-        let err = require_local_ack(&HeaderMap::new(), "/sharded/embeddings")
-            .expect_err("a write with no opt-in must be rejected");
-        let msg = err.to_string();
-        assert!(msg.contains(ACK_HEADER), "must name the header: {msg}");
-        assert!(msg.contains(ACK_LOCAL), "must name the value: {msg}");
-        assert!(
-            msg.contains("/embeddings"),
-            "must name the REPLICATING alternative: {msg}"
-        );
-        assert!(
-            msg.contains("SINGLE-COPY"),
-            "must say what the caller was about to get: {msg}"
-        );
-    }
-
-    /// The replicating ack modes are not an opt-in to single-copy. A caller who
-    /// asked for `quorum` most emphatically did not ask for redundancy 1.
-    #[test]
-    fn a_replicating_ack_mode_is_not_an_optin() {
-        for mode in ["leader", "quorum", "", "LOCAL"] {
-            let err = require_local_ack(&ack(mode), "/sharded/signals")
-                .expect_err("only the exact value `local` opts in");
-            assert!(err.to_string().contains("/signals"), "mode {mode:?}");
-        }
-    }
-
-    /// Every write route derives its replicating alternative from its own path,
-    /// so the message cannot drift from the routing table.
-    #[test]
-    fn the_alternative_is_derived_from_the_route() {
-        for (sharded, replicating) in [
-            ("/sharded/items", "/items"),
-            ("/sharded/embeddings", "/embeddings"),
-            ("/sharded/signals", "/signals"),
-        ] {
-            let msg = require_local_ack(&HeaderMap::new(), sharded)
-                .expect_err("no opt-in")
-                .to_string();
-            assert!(
-                msg.contains(&format!("POST {replicating} ")),
-                "{sharded} must point at {replicating}: {msg}"
-            );
-        }
-    }
-}
-
 #[cfg(test)]
 #[allow(clippy::unwrap_used)]
 mod aggregate_region_row_tests {
     //! The apply-burst false-partition fix, at the row-assembly seam: a
     //! slow-but-alive peer (HTTP probe failed, gRPC contact fresh) stays
     //! `reachable: true`; a genuinely silent peer (no reply, no gRPC contact)
-    //! is still flagged `reachable: false, partitioned: true` — and now reports
+    //! is still flagged `reachable: false, partitioned: true`.
-    //! its frontier as UNKNOWN rather than a fabricated worst case.
     use super::aggregate_region_row;
 
-    const HWM: Option<u64> = Some(1_000);
+    const HWM: u64 = 1_000;
 
     #[test]
     fn peer_that_answered_is_reachable_with_real_lag() {
@@ -9212,11 +8992,11 @@ mod aggregate_region_row_tests {
             "partitioned": [],
         });
         // grpc_fresh / leader_mark are irrelevant when the peer answered.
-        let row = aggregate_region_row("eu-west".into(), Some(&json), "us-east", HWM, false, None);
+        let row = aggregate_region_row("eu-west".into(), Some(&json), "us-east", HWM, false, 0);
         assert!(row.reachable);
         assert!(!row.partitioned);
-        assert_eq!(row.applied_events, Some(940));
+        assert_eq!(row.applied_events, 940);
-        assert_eq!(row.lag_events, Some(60), "lag = HWM - applied");
+        assert_eq!(row.lag_events, 60, "lag = HWM - applied");
         assert_eq!(row.version, "0.1.0+dev");
     }
 
@@ -9226,7 +9006,7 @@ mod aggregate_region_row_tests {
         // gRPC contact, so the peer is alive — its HTTP control-plane is just
         // starved under the apply burst. It must NOT be flagged partitioned, and
         // its lag is the HONEST gap from the leader's ack mark, not worst-case.
-        let row = aggregate_region_row("eu-west".into(), None, "us-east", HWM, true, Some(980));
+        let row = aggregate_region_row("eu-west".into(), None, "us-east", HWM, true, 980);
         assert!(
             row.reachable,
             "a peer with fresh gRPC contact must stay reachable despite an HTTP-probe timeout"
@@ -9235,42 +9015,19 @@ mod aggregate_region_row_tests {
             !row.partitioned,
             "a slow-but-alive peer must NOT be marked partitioned"
         );
+        assert_eq!(row.applied_events, 980, "reports the leader's ack mark");
         assert_eq!(
-            row.applied_events,
+            row.lag_events, 20,
-            Some(980),
-            "reports the leader's ack mark"
-        );
-        assert_eq!(
-            row.lag_events,
-            Some(20),
             "honest lag = HWM - ack mark, NOT worst-case leader_last_seq"
         );
     }
 
-    /// A leader with fresh gRPC contact but NO ack mark for the peer knows the
-    /// peer is alive and nothing about its frontier. Reporting `applied 0` there
-    /// was the same fabrication one arm down.
     #[test]
-    fn alive_peer_without_an_ack_mark_reports_an_unknown_frontier() {
+    fn genuinely_silent_peer_is_flagged_unreachable() {
-        let row = aggregate_region_row("eu-west".into(), None, "us-east", HWM, true, None);
-        assert!(row.reachable, "gRPC contact proves the peer is alive");
-        assert!(!row.partitioned);
-        assert_eq!(
-            row.applied_events, None,
-            "no ack mark ⇒ unknown frontier, never 0"
-        );
-        assert_eq!(
-            row.lag_events, None,
-            "a lag cannot be derived from an unknown frontier"
-        );
-    }
-
-    #[test]
-    fn genuinely_silent_peer_is_flagged_unreachable_with_an_unknown_frontier() {
         // No HTTP reply AND no recent gRPC contact (a real partition / dead peer):
         // the honest unreachable verdict stands — the chaos-suite contract. A real
         // TCP severance kills both the HTTP probe and the gRPC ship, landing here.
-        let row = aggregate_region_row("ap-south".into(), None, "us-east", HWM, false, None);
+        let row = aggregate_region_row("ap-south".into(), None, "us-east", HWM, false, 0);
         assert!(
             !row.reachable,
             "a peer with no reply and no gRPC contact is genuinely unreachable"
@@ -9279,46 +9036,10 @@ mod aggregate_region_row_tests {
             row.partitioned,
             "a genuinely unreachable peer is partitioned"
         );
-        // THE 04a FIX: this used to be `Some(HWM)` — a 500ms probe timeout
-        // rendered as the leader's entire history as a deficit. Live, that was
-        // 13.3M events against a cluster whose `shards[]` rows all read lag 0.
         assert_eq!(
-            row.applied_events, None,
+            row.lag_events, HWM,
-            "an unreachable peer's frontier is UNKNOWN, not 0"
+            "an unreachable peer reports worst-case lag (= leader HWM)"
         );
-        assert_eq!(
-            row.lag_events, None,
-            "no worst-case lag may be manufactured from an unknown frontier"
-        );
-    }
-
-    /// When the LEADER's own probe failed there is no high-water-mark, so no lag
-    /// can be computed even for a peer that answered about itself. Reporting the
-    /// old `hwm.unwrap_or(0) - applied = 0` would have said "converged" about a
-    /// cluster nobody had measured — the dangerous direction of the same bug.
-    #[test]
-    fn unknown_leader_hwm_yields_an_unknown_lag_not_zero() {
-        let json = serde_json::json!({ "applied_events": 940u64, "partitioned": [] });
-        let row = aggregate_region_row("eu-west".into(), Some(&json), "us-east", None, false, None);
-        assert!(row.reachable);
-        assert_eq!(
-            row.applied_events,
-            Some(940),
-            "the peer's own report is still honest"
-        );
-        assert_eq!(
-            row.lag_events, None,
-            "lag needs BOTH sides; an unknown leader frontier ⇒ unknown lag"
-        );
-    }
-
-    /// The leader row is at its own high-water-mark by definition, so its lag is
-    /// a known zero rather than an unknown.
-    #[test]
-    fn leader_row_reports_a_known_zero_lag() {
-        let json = serde_json::json!({ "applied_events": 1_000u64, "partitioned": [] });
-        let row = aggregate_region_row("us-east".into(), Some(&json), "us-east", HWM, false, None);
-        assert_eq!(row.lag_events, Some(0));
     }
 }
 

tidal-server/src/cluster/routes.rs

@@ -8,7 +8,7 @@ use std::{sync::Arc, time::Duration};
 use axum::{
     Json, Router,
     extract::{Query, Request, State},
-    http::{HeaderMap, StatusCode},
+    http::StatusCode,
     middleware::{self, Next},
     response::{IntoResponse, Response},
     routing::{get, post},
@@ -22,7 +22,7 @@ use tower::{ServiceBuilder, limit::ConcurrencyLimitLayer};
 use tower_http::timeout::TimeoutLayer;
 use utoipa::ToSchema;
 
-use super::{node::require_local_ack, state::ClusterState};
+use super::state::ClusterState;
 use crate::{
     dto::{
         EmbeddingRequest, FeedItem, FeedQuery, FeedResponse, ItemRequest, MAX_LIMIT, SearchItem,
@@ -114,7 +114,7 @@ pub fn build_cluster_router(
         let creds = Arc::clone(&creds);
         async move {
             if !creds.authenticated(req.headers()) {
-                return crate::router::unauthorized_response(req.headers());
+                return crate::router::unauthorized_response();
             }
             let principal = creds.principal(req.headers());
             if let Err((retry_after_ms, limit)) = creds.check_rate(&principal) {
@@ -557,34 +557,22 @@ pub async fn search(
 
 // ── Sharded (scatter-gather) routes ─────────────────────────────────────────
 
-/// `POST /sharded/items` — write to the entity's owning shard.
-///
-/// **SINGLE-COPY.** The write lands on the owning shard's store only; it does not
-/// ride the leader relay the non-sharded `/items` surface uses, so it has
-/// redundancy 1 regardless of the replication factor. That is by design (parallel
-/// write throughput across shard owners), which is why the surface requires
-/// `x-tidal-ack: local` as an explicit acknowledgement of the tradeoff. For a
-/// replicated write use `POST /items`.
 #[utoipa::path(
     post,
     path = "/sharded/items",
     tag = "sharded",
     request_body = ItemRequest,
     responses(
-        (status = 201, description = "Item written SINGLE-COPY to its owning shard (not replicated)"),
+        (status = 201, description = "Item written to its owning shard"),
-        (status = 400, description = "Invalid request, or the required `x-tidal-ack: local` single-copy opt-in is missing (use POST /items for a replicated write)"),
+        (status = 400, description = "Invalid request"),
         (status = 401, description = "Missing or invalid API key"),
     ),
     security(("bearerAuth" = [])),
 )]
 pub async fn sharded_create_item(
     State(state): State<Arc<ClusterState>>,
-    headers: HeaderMap,
     Json(req): Json<ItemRequest>,
 ) -> std::result::Result<StatusCode, ClusterAppError> {
-    // The same gate, from the same definition, as the multi-process region router
-    // (`cluster::node`). One URL, one durability contract.
-    require_local_ack(&headers, "/sharded/items").map_err(ClusterAppError)?;
     let shards = state.shard_ids().map_err(ClusterAppError)?;
     // The single-shard write does a blocking storage + WAL-fsync `TidalDb` call;
     // running it inline would pin this reactor worker for the whole write (the
@@ -601,28 +589,22 @@ pub async fn sharded_create_item(
     Ok(StatusCode::CREATED)
 }
 
-/// `POST /sharded/embeddings` — write to the entity's owning shard.
-///
-/// **SINGLE-COPY** — see [`sharded_create_item`]. Requires `x-tidal-ack: local`;
-/// for a replicated write use `POST /embeddings`.
 #[utoipa::path(
     post,
     path = "/sharded/embeddings",
     tag = "sharded",
     request_body = EmbeddingRequest,
     responses(
-        (status = 204, description = "Embedding written SINGLE-COPY to its owning shard (not replicated)"),
+        (status = 204, description = "Embedding written to its owning shard"),
-        (status = 400, description = "Invalid request, or the required `x-tidal-ack: local` single-copy opt-in is missing (use POST /embeddings for a replicated write)"),
+        (status = 400, description = "Invalid request"),
         (status = 401, description = "Missing or invalid API key"),
     ),
     security(("bearerAuth" = [])),
 )]
 pub async fn sharded_write_embedding(
     State(state): State<Arc<ClusterState>>,
-    headers: HeaderMap,
     Json(req): Json<EmbeddingRequest>,
 ) -> std::result::Result<StatusCode, ClusterAppError> {
-    require_local_ack(&headers, "/sharded/embeddings").map_err(ClusterAppError)?;
     let shards = state.shard_ids().map_err(ClusterAppError)?;
     // Offload the blocking single-shard embedding write off the reactor (see
     // [`sharded_create_item`]).
@@ -636,28 +618,22 @@ pub async fn sharded_write_embedding(
     Ok(StatusCode::NO_CONTENT)
 }
 
-/// `POST /sharded/signals` — write to the entity's owning shard.
-///
-/// **SINGLE-COPY** — see [`sharded_create_item`]. Requires `x-tidal-ack: local`;
-/// for a replicated write use `POST /signals`.
 #[utoipa::path(
     post,
     path = "/sharded/signals",
     tag = "sharded",
     request_body = SignalRequest,
     responses(
-        (status = 204, description = "Signal written SINGLE-COPY to its owning shard (not replicated)"),
+        (status = 204, description = "Signal written to its owning shard"),
-        (status = 400, description = "Invalid request, or the required `x-tidal-ack: local` single-copy opt-in is missing (use POST /signals for a replicated write)"),
+        (status = 400, description = "Invalid request"),
         (status = 401, description = "Missing or invalid API key"),
     ),
     security(("bearerAuth" = [])),
 )]
 pub async fn sharded_write_signal(
     State(state): State<Arc<ClusterState>>,
-    headers: HeaderMap,
     Json(req): Json<SignalRequest>,
 ) -> std::result::Result<StatusCode, ClusterAppError> {
-    require_local_ack(&headers, "/sharded/signals").map_err(ClusterAppError)?;
     let shards = state.shard_ids().map_err(ClusterAppError)?;
     // Offload the blocking single-shard signal write off the reactor (see
     // [`sharded_create_item`]).

tidal-server/src/router.rs

@@ -154,7 +154,7 @@ pub fn build_router(
             if let Some(key) = creds.bearer()
                 && !bearer_token_ok(req.headers(), &key)
             {
-                return unauthorized_response(req.headers());
+                return unauthorized_response();
             }
             // m11p7 per-principal rate limit (standalone principals are always
             // external — there is no inter-node plane here).
@@ -239,7 +239,7 @@ pub async fn bearer_auth(request: Request, next: Next, expected_key: &str) -> Re
     if bearer_token_ok(request.headers(), expected_key) {
         next.run(request).await
     } else {
-        unauthorized_response(request.headers())
+        unauthorized_response()
     }
 }
 
@@ -270,47 +270,8 @@ pub(crate) fn bearer_token_ok(headers: &axum::http::HeaderMap, expected_key: &st
 }
 
 /// The 401 returned when the bearer token is missing or invalid.
-///
-/// Emits a `WARN` naming the rejection reason and the forwarded client address.
-///
-/// WHY THIS LOGS: on 2026-08-30 the live cluster had served **3,535** 401s across
-/// the seven publicly-ingressed data routes, climbing at ~9/min, and there was no
-/// record anywhere of who was calling or why. `tidaldb_http_requests_total` gave
-/// the aggregate — it is what surfaced the flood at all — but a counter cannot
-/// name a caller, so the only way to tell a scanner from a misconfigured client
-/// from a rotated-key outage was to guess. Rejecting a request correctly and then
-/// discarding every fact about it is a swallowed error: the security control
-/// worked and reported nothing, which is indistinguishable from silence.
-///
-/// The enclosing `request` span (see [`with_request_id_tracing`]) already carries
-/// `method`, `uri` and `request_id`, and it wraps OUTSIDE both auth layers, so
-/// those fields come free on this event. Only what the span cannot know is added
-/// here: which of the two failure modes occurred, and the forwarded client
-/// address — behind Traefik the socket peer is the ingress, so `x-forwarded-for`
-/// is the only thing that identifies the real caller.
-///
-/// The token itself is NEVER logged, in either form. Presence is the signal;
-/// content would put a credential in the log store.
 #[must_use]
-pub(crate) fn unauthorized_response(headers: &axum::http::HeaderMap) -> Response {
+pub(crate) fn unauthorized_response() -> Response {
-    // Distinguishing these two is the whole diagnostic value: "missing" is an
-    // unauthenticated prober or a client that never got configured, "invalid" is
-    // a real credential that stopped working — a rotation that missed a consumer.
-    // Collapsing them into one line would leave the ambiguity this exists to end.
-    let reason = if headers.contains_key(AUTHORIZATION) {
-        "invalid_token"
-    } else {
-        "missing_token"
-    };
-    let client = headers
-        .get("x-forwarded-for")
-        .and_then(|v| v.to_str().ok())
-        .unwrap_or("-");
-    tracing::warn!(
-        reason,
-        client,
-        "rejected request: missing or invalid api key"
-    );
     (
         StatusCode::UNAUTHORIZED,
         [("www-authenticate", "Bearer")],

tidal-server/tests/cluster_routes.rs

@@ -227,25 +227,6 @@ fn post_json(
     client.post(url).json(body).send().unwrap()
 }
 
-/// POST a `/sharded/*` WRITE with the single-copy opt-in header the surface
-/// requires.
-///
-/// `/sharded/{items,embeddings,signals}` apply to the owning region's local store
-/// with no WAL append, so they are single-copy regardless of the replication
-/// factor and reject (400) a caller who has not said `x-tidal-ack: local`.
-fn post_sharded(
-    client: &reqwest::blocking::Client,
-    url: &str,
-    body: &serde_json::Value,
-) -> reqwest::blocking::Response {
-    client
-        .post(url)
-        .header("x-tidal-ack", "local")
-        .json(body)
-        .send()
-        .unwrap()
-}
-
 // ── Tests ─────────────────────────────────────────────────────────────────────
 
 /// A signal POSTed to a FOLLOWER is forwarded to the leader (204), durably
@@ -520,12 +501,12 @@ fn sharded_feed_degrades_honestly() {
     // owner is the DOWN node will 503, which we tolerate — we only need the live
     // shards populated for the read fan-out.)
     for i in 1..=12u64 {
-        let _ = post_sharded(
+        let _ = post_json(
             client,
             &format!("{}/sharded/items", s.base(0)),
             &serde_json::json!({ "entity_id": i, "metadata": { "title": format!("item {i}") } }),
         );
-        let _ = post_sharded(
+        let _ = post_json(
             client,
             &format!("{}/sharded/signals", s.base(0)),
             &serde_json::json!({ "entity_id": i, "signal": "view", "weight": i as f64 }),
@@ -575,7 +556,7 @@ fn sharded_write_routes_to_owner() {
     // The engine hash is deterministic; iterate ids until one owns to region 1 or 2.
     let mut routed = false;
     for entity_id in 1..=200u64 {
-        let resp = post_sharded(
+        let resp = post_json(
             client,
             &format!("{}/sharded/items", s.base(0)),
             &serde_json::json!({ "entity_id": entity_id, "metadata": { "title": format!("e{entity_id}") } }),
@@ -585,7 +566,7 @@ fn sharded_write_routes_to_owner() {
             continue;
         }
         // Add a signal too so the item is rankable on its owner.
-        let _ = post_sharded(
+        let _ = post_json(
             client,
             &format!("{}/sharded/signals", s.base(0)),
             &serde_json::json!({ "entity_id": entity_id, "signal": "view", "weight": 5.0 }),

tidal-server/tests/cluster_runbook.rs

@@ -93,16 +93,11 @@ struct AggregatedStatus {
 }
 
 /// One region row inside [`AggregatedStatus`] (runbook §6).
-///
-/// `applied_events` / `lag_events` are `Option<u64>`: `/cluster/status` reports a
-/// peer it has no frontier report for as JSON `null` rather than a fabricated `0`
-/// (`node.rs` `aggregate_region_row`), so a `u64` here would fail to deserialize
-/// exactly when a probe misses — e.g. the transient window right after a promote.
 #[derive(Debug, Deserialize)]
 struct AggregatedRegion {
     name: String,
-    applied_events: Option<u64>,
+    applied_events: u64,
-    lag_events: Option<u64>,
+    lag_events: u64,
     partitioned: bool,
     reachable: bool,
 }
@@ -595,12 +590,9 @@ fn runbook_s7_sharded() {
     let cluster = MultiProcCluster::start(3);
 
     // Sharded writes route to the owning region (engine ShardRouter hash) and
-    // return the documented status codes. They carry `x-tidal-ack: local` (via
+    // return the documented status codes.
-    // `post_sharded`): the `/sharded/*` write surface applies to the owning
-    // region's local store with no WAL append, so it is single-copy regardless of
-    // the replication factor and refuses a caller who has not said so.
     for entity_id in 1..=12u64 {
-        let resp = cluster.post_sharded(
+        let resp = cluster.post(
             LEADER,
             "/sharded/items",
             &serde_json::json!({
@@ -611,14 +603,14 @@ fn runbook_s7_sharded() {
         assert_eq!(resp.status().as_u16(), 201, "/sharded/items → 201");
 
         let v = entity_id as f32;
-        let resp = cluster.post_sharded(
+        let resp = cluster.post(
             LEADER,
             "/sharded/embeddings",
             &serde_json::json!({ "entity_id": entity_id, "values": [v, v + 1.0, v + 2.0, v + 3.0] }),
         );
         assert_eq!(resp.status().as_u16(), 204, "/sharded/embeddings → 204");
 
-        let resp = cluster.post_sharded(
+        let resp = cluster.post(
             LEADER,
             "/sharded/signals",
             &serde_json::json!({ "entity_id": entity_id, "signal": "view", "weight": 1.0 }),
@@ -626,19 +618,6 @@ fn runbook_s7_sharded() {
         assert_eq!(resp.status().as_u16(), 204, "/sharded/signals → 204");
     }
 
-    // The opt-in is REQUIRED: the same write with no `x-tidal-ack: local` is a 400
-    // naming the header and the replicating alternative (runbook §7).
-    let refused = cluster.post(
-        LEADER,
-        "/sharded/items",
-        &serde_json::json!({ "entity_id": 99u64, "metadata": {} }),
-    );
-    assert_eq!(
-        refused.status().as_u16(),
-        400,
-        "/sharded/items without the single-copy opt-in → 400"
-    );
-
     // /sharded/feed: 200 + the documented scatter_gather block keys, with a tight
     // deadline honored. All shards healthy ⇒ not degraded, no unavailable shards.
     let resp = cluster.get(
@@ -848,25 +827,20 @@ fn runbook_s10_partition_drill() {
     );
     println!("[s10.4] stale-follower read on ap-south OK (pre-partition view)");
 
-    // Aggregated status reports ap-south unreachable with an UNKNOWN frontier.
+    // Aggregated status reports ap-south unreachable with worst-case lag.
-    // It used to assert a worst-case `lag >= 1`; that number was manufactured
-    // from `applied_events: 0` (the leader's whole history rendered as a
-    // deficit). The honest report for a peer this node cannot reach is `null`.
     poll_until(
         Duration::from_secs(10),
-        "[s10] aggregated status must show ap-south reachable:false with a null frontier",
+        "[s10] aggregated status must show ap-south reachable:false",
         || {
             cluster
                 .get(LEADER, "/cluster/status")
                 .json::<AggregatedStatus>()
                 .ok()
                 .and_then(|s| s.regions.into_iter().find(|r| r.name == "ap-south"))
-                .is_some_and(|r| {
+                .is_some_and(|r| !r.reachable && r.lag_events >= 1)
-                    !r.reachable && r.applied_events.is_none() && r.lag_events.is_none()
-                })
         },
     );
-    println!("[s10] aggregated status: ap-south reachable:false with an unknown (null) frontier");
+    println!("[s10] aggregated status: ap-south reachable:false with worst-case lag");
 
     // 7. Scatter-gather degradation: /sharded/feed degrades honestly (degraded:true,
     //    ap-south in unavailable_shards) yet still returns live-shard items.
@@ -903,9 +877,8 @@ fn runbook_s10_partition_drill() {
     let agg: AggregatedStatus = cluster.get(LEADER, "/cluster/status").json().unwrap();
     let ap = agg.regions.iter().find(|r| r.name == "ap-south").unwrap();
     assert_eq!(
-        ap.lag_events,
+        ap.lag_events, 0,
-        Some(0),
+        "[s10.6] ap-south lag back to 0 after heal: {ap:?}"
-        "[s10.6] ap-south lag back to a KNOWN 0 after heal: {ap:?}"
     );
     assert!(
         ap.reachable,

tidal-server/tests/cluster_sharding.rs

@@ -524,7 +524,7 @@ fn mp_embedding_is_searchable_on_every_replica_without_restart() {
         assert_eq!(
             resp.status().as_u16(),
             201,
-            "entity {entity} via node {via}: /items must 201"
+            "entity {entity} via node {via}: /sharded/items must 201"
         );
         let resp = cluster.post(
             via,
@@ -534,7 +534,7 @@ fn mp_embedding_is_searchable_on_every_replica_without_restart() {
         assert_eq!(
             resp.status().as_u16(),
             204,
-            "entity {entity} via node {via}: /embeddings must 204"
+            "entity {entity} via node {via}: /sharded/embeddings must 204"
         );
     }
 
@@ -637,8 +637,7 @@ fn vector_search_finds(cluster: &MultiProcCluster, node: usize, entity: u64) ->
     })
 }
 
-/// The `/sharded/*` write surface is single-copy BY DESIGN, and now says so — pin
+/// The `/sharded/*` write surface is single-copy BY DESIGN — pin it.
-/// both halves.
 ///
 /// `node.rs:8828-8829` states it: the `/sharded/*` surface hash-partitions and applies
 /// to the owning region's LOCAL store, and does NOT ride the leader WAL relay (that is
@@ -656,12 +655,6 @@ fn vector_search_finds(cluster: &MultiProcCluster, node: usize, entity: u64) ->
 /// If a future change makes `/sharded/*` replicate, this test SHOULD fail: that is a
 /// deliberate contract change, and the failure is the prompt to update the docs, the
 /// runbook, and any durability claim that depends on it.
-///
-/// Since the opt-in landed the surface REQUIRES `x-tidal-ack: local`, so this test
-/// pins two things: the single-copy semantics WHEN opted in, and the 400 (naming
-/// the header and the replicating alternative) when not. Making the tradeoff
-/// explicit is what stops a caller taking it by accident; it does not change what
-/// the tradeoff is.
 #[test]
 fn mp_sharded_surface_writes_are_local_to_the_owner() {
     let _heavy = heavy_test_guard();
@@ -669,17 +662,12 @@ fn mp_sharded_surface_writes_are_local_to_the_owner() {
     let _leaders = cluster.wait_shard_leaders_agreed(convergence_budget());
 
     // Six ids spread across the hash space, each written through a different node.
-    //
-    // The writes carry `x-tidal-ack: local` (via `post_sharded`) because the
-    // surface now REQUIRES that opt-in. The point of this test is unchanged: the
-    // surface is still single-copy WHEN opted in. The opt-in makes the tradeoff a
-    // decision; it does not soften it.
     const ENTITIES: [u64; 6] = [922_001, 922_002, 922_003, 922_004, 922_005, 922_006];
     for (n, entity) in ENTITIES.iter().enumerate() {
         let via = n % NODES;
         assert_eq!(
             cluster
-                .post_sharded(
+                .post(
                     via,
                     "/sharded/items",
                     &serde_json::json!({ "entity_id": entity, "metadata": {} })
@@ -690,7 +678,7 @@ fn mp_sharded_surface_writes_are_local_to_the_owner() {
         );
         assert_eq!(
             cluster
-                .post_sharded(
+                .post(
                     via,
                     "/sharded/embeddings",
                     &serde_json::json!({ "entity_id": entity, "values": embedding_for(*entity) })
@@ -700,27 +688,6 @@ fn mp_sharded_surface_writes_are_local_to_the_owner() {
             204
         );
     }
-
-    // And WITHOUT the opt-in the same write is refused, with a body that names the
-    // header and the replicating alternative. This is the half that makes the
-    // single-copy tradeoff impossible to take by accident.
-    let refused = cluster.post(
-        0,
-        "/sharded/items",
-        &serde_json::json!({ "entity_id": 922_099_u64, "metadata": {} }),
-    );
-    assert_eq!(
-        refused.status().as_u16(),
-        400,
-        "a /sharded/* write with no x-tidal-ack: local opt-in must be refused"
-    );
-    let body = refused.text().unwrap_or_default();
-    for expected in ["x-tidal-ack", "local", "SINGLE-COPY", "/items"] {
-        assert!(
-            body.contains(expected),
-            "the rejection must name {expected:?}: {body}"
-        );
-    }
     cluster.wait_converged_all(convergence_budget());
     // Generous settle: the claim is "never replicates", so give replication every
     // chance to happen before asserting that it did not.
@@ -752,130 +719,3 @@ fn mp_sharded_surface_writes_are_local_to_the_owner() {
         "every /sharded/* write must still be durable on its owner: {spread:?}"
     );
 }
-
-/// The shared title token every seeded item carries, so ONE `/search?query=` hit
-/// returns the whole seeded page (`item_token` is per-entity unique and would
-/// return exactly one row, which cannot show a rank sequence).
-const RANK_TOKEN: &str = "rnkdense";
-
-/// Entity ids spread across the hash space so all three groups contribute rows
-/// on the multi-group shape. 12 ⇒ several rows per group, so a per-group counter
-/// that was never re-stamped shows up as duplicates rather than by luck.
-const RANK_ENTITIES: [u64; 12] = [
-    933_001, 933_002, 933_003, 933_004, 933_005, 933_006, 933_007, 933_008, 933_009, 933_010,
-    933_011, 933_012,
-];
-
-/// One ranked page from `node`, as `(rank, score)` in wire order.
-fn ranked_page(cluster: &MultiProcCluster, node: usize, path: &str) -> Vec<(u64, f64)> {
-    let body = cluster.get_json(node, path);
-    body["items"]
-        .as_array()
-        .unwrap_or(&Vec::new())
-        .iter()
-        .map(|it| {
-            (
-                it["rank"].as_u64().unwrap_or_default(),
-                it["score"].as_f64().unwrap_or_default(),
-            )
-        })
-        .collect()
-}
-
-/// Seed `RANK_ENTITIES` on a fresh `shards`-group cluster and assert every node's
-/// `/feed` and `/search` page carries a DENSE ascending `rank` with descending
-/// scores.
-fn assert_dense_ranks(shards: usize) {
-    let cluster = MultiProcCluster::start_sharded(NODES, shards, Some(FAST_ELECTION_YAML));
-    let _leaders = cluster.wait_shard_leaders_agreed(convergence_budget());
-
-    for (n, entity) in RANK_ENTITIES.iter().enumerate() {
-        let via = n % NODES;
-        let resp = cluster.post(
-            via,
-            "/items",
-            &serde_json::json!({
-                "entity_id": entity,
-                "metadata": { "title": format!("{RANK_TOKEN} {}", item_token(*entity)) }
-            }),
-        );
-        assert_eq!(
-            resp.status().as_u16(),
-            201,
-            "entity {entity} via node {via}: /items must 201"
-        );
-    }
-    cluster.wait_converged_all(convergence_budget());
-
-    let limit = RANK_ENTITIES.len();
-    let feed_path = format!("/feed?profile=for_you&limit={limit}");
-    let search_path = format!("/search?query={RANK_TOKEN}&limit={limit}");
-
-    // The text index auto-commits on a ~2s cadence, so poll until the whole
-    // seeded page is visible on every node. A SHORT page would make the density
-    // assertion pass vacuously (a 1-row page is trivially dense).
-    for path in [&feed_path, &search_path] {
-        let full = wait_until(convergence_budget(), || {
-            (0..NODES).all(|n| ranked_page(&cluster, n, path).len() == limit)
-        });
-        assert!(
-            full,
-            "shards={shards}: every node must serve all {limit} seeded rows on {path} before \
-             rank density can be judged; got {:?}",
-            (0..NODES)
-                .map(|n| ranked_page(&cluster, n, path).len())
-                .collect::<Vec<_>>()
-        );
-    }
-
-    for node in 0..NODES {
-        for path in [&feed_path, &search_path] {
-            let page = ranked_page(&cluster, node, path);
-            let ranks: Vec<u64> = page.iter().map(|&(r, _)| r).collect();
-            let expected: Vec<u64> = (1..=page.len() as u64).collect();
-            assert_eq!(
-                ranks, expected,
-                "shards={shards} node {node} {path}: rank must be a dense ascending 1..n \
-                 sequence. Duplicates are the signature of per-group slices merged without a \
-                 re-stamp (scatter_merge); zeros mean a remote slice was never stamped."
-            );
-            for w in page.windows(2) {
-                assert!(
-                    w[1].1 <= w[0].1 + 1e-9,
-                    "shards={shards} node {node} {path}: scores must stay descending — the \
-                     rank stamp is not a re-sort. Got {:?} then {:?}",
-                    w[0],
-                    w[1]
-                );
-            }
-        }
-    }
-    println!("[rank] shards={shards}: dense 1..{limit} on /feed and /search, all {NODES} nodes");
-}
-
-/// `rank` on a cluster's ranked reads must be DENSE and ascending — on BOTH of
-/// the two structurally different paths the same handler reaches the wire by.
-///
-/// Under full placement (production: every group on every node) `missing_groups()`
-/// is empty and `/feed` / `/search` return straight out of `scatter_merge`
-/// (`node.rs`), skipping `merge_cross_shard` — the sibling that DOES re-stamp.
-/// Live `/search?query=verification` returned ranks `1,1,2`: each hosted group
-/// ranks its own slice `1..k` locally, the slices get concatenated and
-/// score-sorted, and nothing renumbered them. `scatter_merge` now takes the same
-/// `set_rank` closure and stamps after `truncate`.
-///
-/// Two shapes, because there are two paths:
-///  - **multi-group** (`shards = SHARDS`): the fan-out + merge path, where the
-///    stamp lives. This is the production shape and the one that was broken.
-///  - **single-group** (`shards = 1`): the `[only]` fast path returns the engine's
-///    own result and deliberately BYPASSES the stamp. That the engine already
-///    ranks densely is an assumption until asserted, so assert it.
-///
-/// Scores are checked to stay descending on both: the fix is a renumbering, never
-/// a re-sort, so an ordering change here would be a different and worse defect.
-#[test]
-fn mp_ranked_reads_stamp_dense_rank_on_both_merge_shapes() {
-    let _heavy = heavy_test_guard();
-    assert_dense_ranks(SHARDS);
-    assert_dense_ranks(1);
-}

tidal-server/tests/support/multiproc.rs

@@ -758,30 +758,6 @@ impl MultiProcCluster {
             .unwrap_or_else(|e| panic!("POST {url} failed: {e}"))
     }
 
-    /// `POST {node(idx)}{path}` carrying the `/sharded/*` single-copy opt-in
-    /// header (`x-tidal-ack: local`).
-    ///
-    /// The `/sharded/*` WRITE routes apply to the owning region's local store with
-    /// no WAL append, so they are single-copy regardless of the replication factor
-    /// and now REJECT (400) a caller that has not said so explicitly. Every
-    /// `/sharded/{items,embeddings,signals}` write in the suites goes through here
-    /// so the opt-in is stated once.
-    #[must_use]
-    pub fn post_sharded(
-        &self,
-        idx: usize,
-        path: &str,
-        body: &serde_json::Value,
-    ) -> reqwest::blocking::Response {
-        let url = format!("{}{path}", self.node(idx));
-        self.client
-            .post(&url)
-            .header("x-tidal-ack", "local")
-            .json(body)
-            .send()
-            .unwrap_or_else(|e| panic!("POST {url} failed: {e}"))
-    }
-
     // ── Crash / restart ──────────────────────────────────────────────────────
 
     /// SIGKILL the node at `idx` (a real crash — no graceful drain). After this,

tidal/src/db/items.rs

@@ -67,18 +67,7 @@ impl TidalDb {
         // "entered the outbound stream", so a staging or fsync failure must not
         // inflate it and make a peer look like it lost a record that never shipped.
         #[cfg(feature = "metrics")]
-        {
+        self.metrics.cluster.observe_blobs_originated(kind, 1);
-            self.metrics.cluster.observe_blobs_originated(kind, 1);
-            // Same durable-append fact, one series narrower: an embedding that
-            // reached here is a vector every replica of this group MUST also
-            // hold, which is exactly what the divergence alert asserts. The
-            // follower side of the same vector is counted in
-            // `apply_replicated_blobs`; counting only here would make the series
-            // per-node-origin and the replicas could never agree.
-            if kind == crate::wal::format::batch::BlobKind::Embedding {
-                self.metrics.observe_replicated_vectors(1);
-            }
-        }
         Ok(Some(seq))
     }
     /// Write (or overwrite) item metadata and update in-memory indexes.
@@ -213,16 +202,6 @@ impl TidalDb {
                 }
                 if outcome.is_ok() {
                     cluster.observe_blobs_applied(kind, n);
-                    // The follower side of `wal_blob_first`'s vector count: this
-                    // node now holds vectors the originating node counted when
-                    // its append went durable, so both ends of one replicated
-                    // write land on the same series and replicas of a group can
-                    // actually converge. Success-only, and this is the LIVE apply
-                    // path — boot replay (`replay_recovered_blobs`) is excluded
-                    // here for the same reason `applied` excludes it.
-                    if kind == crate::wal::format::batch::BlobKind::Embedding {
-                        self.metrics.observe_replicated_vectors(n);
-                    }
                 } else {
                     // One failure per kind present in the halted round. The round is
                     // all-or-nothing from the receiver's perspective, and the error
@@ -1039,5 +1018,309 @@ impl TidalDb {
 
 #[cfg(test)]
 #[allow(clippy::unwrap_used)]
-#[path = "items_tests.rs"]
+mod tests {
-mod tests;
+    use std::collections::HashMap;
+
+    use crate::{TidalDb, schema::EntityId};
+
+    /// Minimal valid schema so ephemeral mode wires in-memory storage (needed by
+    /// any test that actually persists an item).
+    fn minimal_schema() -> crate::schema::Schema {
+        use crate::schema::{DecaySpec, EntityKind, SchemaBuilder, Window};
+        let mut b = SchemaBuilder::new();
+        let _ = b
+            .signal(
+                "view",
+                EntityKind::Item,
+                DecaySpec::Exponential {
+                    half_life: std::time::Duration::from_secs(3600),
+                },
+            )
+            .windows(&[Window::AllTime])
+            .velocity(false)
+            .add();
+        b.build().expect("schema must be valid")
+    }
+
+    /// The apply-side ledger counts both edges, and counting a failure must not
+    /// turn it into a success.
+    ///
+    /// `apply_failed` exists to make a lost blob visible; if incrementing it also
+    /// swallowed the `Err`, the receiver would advance past a record it never
+    /// applied and the counter would document a silent data loss instead of
+    /// preventing one (`CODING_GUIDELINES` :193-195).
+    #[test]
+    fn blob_apply_counts_both_edges_and_still_propagates_the_error() {
+        use crate::wal::format::batch::{BlobRecord, EmbeddingRecord, TermMarkerRecord};
+
+        let db = TidalDb::builder()
+            .ephemeral()
+            .with_schema(minimal_schema())
+            .open()
+            .unwrap();
+
+        // Success edge: a valid, finite, non-zero-norm embedding.
+        db.apply_replicated_blobs(vec![BlobRecord::Embedding(EmbeddingRecord {
+            entity_id: 1,
+            values: vec![1.0, 0.0, 0.0],
+        })])
+        .expect("a valid embedding blob must apply");
+
+        // Failure edge: term 0 is invalid by construction (no topology era ever
+        // journals one), so Phase 1 rejects it deterministically.
+        let err = db
+            .apply_replicated_blobs(vec![BlobRecord::TermMarker(TermMarkerRecord {
+                term: 0,
+                leader_region: 0,
+            })])
+            .expect_err("term 0 must be rejected, not counted-and-swallowed");
+        assert!(
+            format!("{err}").contains("term 0"),
+            "the original error must reach the caller unchanged, got: {err}"
+        );
+
+        let mut out = String::new();
+        db.metrics.cluster.render_into(&mut out, 0);
+        assert!(
+            out.contains(
+                r#"tidaldb_cluster_blobs_applied_total{kind="embedding",partition_id="0"} 1"#
+            ),
+            "successful apply must be counted:\n{out}"
+        );
+        assert!(
+            out.contains(
+                r#"tidaldb_cluster_blobs_apply_failed_total{kind="term_marker",partition_id="0"} 1"#
+            ),
+            "failed apply must be counted under its own kind:\n{out}"
+        );
+        assert!(
+            out.contains(
+                r#"tidaldb_cluster_blobs_applied_total{kind="term_marker",partition_id="0"} 0"#
+            ),
+            "a failed apply must NOT also count as applied:\n{out}"
+        );
+    }
+
+    #[test]
+    fn write_item_rejects_oversized_metadata_value() {
+        let db = TidalDb::builder().ephemeral().open().unwrap();
+        let mut meta = HashMap::new();
+        // Insert a value that exceeds the 8 KB per-value limit.
+        meta.insert("big_key".to_string(), "x".repeat(9 * 1024));
+        let err = db
+            .write_item_with_metadata(EntityId::new(1), &meta)
+            .unwrap_err();
+        assert!(
+            err.to_string().contains("metadata value too long"),
+            "expected value-too-long error, got: {err}"
+        );
+        db.close().unwrap();
+    }
+
+    #[test]
+    fn write_item_rejects_too_many_metadata_keys() {
+        let db = TidalDb::builder().ephemeral().open().unwrap();
+        let mut meta = HashMap::new();
+        for i in 0..65 {
+            meta.insert(format!("key_{i}"), "val".to_string());
+        }
+        let err = db
+            .write_item_with_metadata(EntityId::new(1), &meta)
+            .unwrap_err();
+        assert!(
+            err.to_string().contains("max key count"),
+            "expected key-count error, got: {err}"
+        );
+        db.close().unwrap();
+    }
+
+    #[test]
+    fn write_item_rejects_oversized_total_metadata() {
+        let db = TidalDb::builder().ephemeral().open().unwrap();
+        let mut meta = HashMap::new();
+        // 10 keys x 7 KB values = 70 KB > 64 KB limit.
+        for i in 0..10 {
+            meta.insert(format!("k{i}"), "x".repeat(7 * 1024));
+        }
+        let err = db
+            .write_item_with_metadata(EntityId::new(1), &meta)
+            .unwrap_err();
+        assert!(
+            err.to_string().contains("total size too large"),
+            "expected total-size error, got: {err}"
+        );
+        db.close().unwrap();
+    }
+
+    #[test]
+    fn write_item_rejects_id_above_u32_universe_limit() {
+        let db = TidalDb::builder()
+            .ephemeral()
+            .with_schema(minimal_schema())
+            .open()
+            .unwrap();
+        let meta = HashMap::new();
+        // u32::MAX + 1: the smallest id that would alias a lower id once
+        // narrowed to u32 for the candidate-generation bitmaps.
+        let oversized = EntityId::new(u64::from(u32::MAX) + 1);
+        let err = db.write_item_with_metadata(oversized, &meta).unwrap_err();
+        assert!(
+            err.to_string().contains("u32 item-universe limit"),
+            "expected u32 item-universe rejection, got: {err}"
+        );
+        // The rejected item must not have leaked into storage or the universe
+        // bitmap — a rejected write leaves no trace.
+        assert_eq!(db.item_count(), 0, "rejected item must not enter universe");
+        assert!(
+            db.get_item_metadata(oversized).unwrap().is_none(),
+            "rejected item must not be persisted"
+        );
+        db.close().unwrap();
+    }
+
+    #[test]
+    fn write_item_accepts_id_at_u32_universe_boundary() {
+        let db = TidalDb::builder()
+            .ephemeral()
+            .with_schema(minimal_schema())
+            .open()
+            .unwrap();
+        let meta = HashMap::new();
+        // u32::MAX is the largest in-bounds id: narrowing is lossless.
+        let boundary = EntityId::new(u64::from(u32::MAX));
+        db.write_item_with_metadata(boundary, &meta).unwrap();
+        assert_eq!(db.item_count(), 1, "boundary id must be accepted");
+        db.close().unwrap();
+    }
+
+    #[test]
+    fn overwrite_scrubs_stale_index_entries() {
+        // W22 regression: write_item_with_metadata is "write OR overwrite". Before
+        // the re-index scrub, the in-memory bitmap/range indexes only ever
+        // INSERTED, so overwriting an item with new values left it indexed under
+        // BOTH the old and new value — a phantom hit on metadata-filtered
+        // RETRIEVE/SEARCH and a corrupted recency order. The durable store is
+        // always correct; this exercises the LIVE index, which is what diverged.
+        use std::ops::Bound;
+
+        let make = |pairs: &[(&str, &str)]| -> HashMap<String, String> {
+            pairs
+                .iter()
+                .map(|(k, v)| ((*k).to_string(), (*v).to_string()))
+                .collect()
+        };
+
+        let db = TidalDb::builder()
+            .ephemeral()
+            .with_schema(minimal_schema())
+            .open()
+            .unwrap();
+        let id = EntityId::new(1);
+
+        // First write under the OLD values.
+        db.write_item_with_metadata(
+            id,
+            &make(&[
+                ("category", "news"),
+                ("creator_id", "10"),
+                ("tags", "alpha,beta"),
+                ("duration", "300"),
+            ]),
+        )
+        .unwrap();
+        assert!(
+            db.category_index.get("news").is_some_and(|b| b.contains(1)),
+            "item must be indexed under its initial category"
+        );
+        assert!(db.creator_items.get(10).is_some_and(|b| b.contains(1)));
+
+        // Overwrite with ALL-NEW values.
+        db.write_item_with_metadata(
+            id,
+            &make(&[
+                ("category", "sports"),
+                ("creator_id", "20"),
+                ("tags", "gamma"),
+                ("duration", "600"),
+            ]),
+        )
+        .unwrap();
+
+        // No stale entry survives under any OLD value (the phantom-hit bug).
+        assert!(
+            db.category_index.get("news").is_none_or(|b| !b.contains(1)),
+            "stale category 'news' must be scrubbed on overwrite"
+        );
+        assert!(
+            db.creator_index.get("10").is_none_or(|b| !b.contains(1)),
+            "stale creator '10' must be scrubbed"
+        );
+        assert!(
+            db.tag_index.get("alpha").is_none_or(|b| !b.contains(1)),
+            "stale tag 'alpha' must be scrubbed"
+        );
+        assert!(
+            db.creator_items.get(10).is_none_or(|b| !b.contains(1)),
+            "stale creator_items[10] association must be scrubbed"
+        );
+        assert!(
+            !db.duration_index
+                .range(Bound::Included(&300), Bound::Included(&300))
+                .contains(1),
+            "stale duration 300 must be scrubbed"
+        );
+
+        // The NEW values are all present.
+        assert!(
+            db.category_index
+                .get("sports")
+                .is_some_and(|b| b.contains(1))
+        );
+        assert!(db.creator_index.get("20").is_some_and(|b| b.contains(1)));
+        assert!(db.tag_index.get("gamma").is_some_and(|b| b.contains(1)));
+        assert!(db.creator_items.get(20).is_some_and(|b| b.contains(1)));
+        assert!(
+            db.duration_index
+                .range(Bound::Included(&600), Bound::Included(&600))
+                .contains(1),
+            "new duration 600 must be indexed"
+        );
+        db.close().unwrap();
+    }
+
+    #[test]
+    fn cluster_node_stamps_configured_partition_id() {
+        // W13 regression: the construction site must call `set_partition_id` with
+        // the configured shard, so each node's identity metric lines carry its own
+        // `partition_id` instead of the default `0` (otherwise cluster series
+        // collide when scraped into one Prometheus). Exercises the real build
+        // wiring end-to-end, not just the setter.
+        use crate::db::config::NodeConfig;
+        use crate::replication::ShardId;
+
+        let db = TidalDb::builder()
+            .ephemeral()
+            .with_schema(minimal_schema())
+            .with_cluster(NodeConfig {
+                shard_id: ShardId(3),
+                ..NodeConfig::default()
+            })
+            .open()
+            .unwrap();
+
+        let out = db.metrics().render_prometheus();
+        assert!(
+            out.contains("tidaldb_health_ok{partition_id=\"3\"}"),
+            "node must stamp its configured shard id (3) on identity lines: {out}"
+        );
+        assert!(
+            out.contains("tidaldb_uptime_seconds{partition_id=\"3\"}"),
+            "uptime line must carry the configured partition_id: {out}"
+        );
+        assert!(
+            !out.contains("partition_id=\"0\""),
+            "a shard-3 node must not emit the single-node default partition_id=0: {out}"
+        );
+        db.close().unwrap();
+    }
+}

tidal/src/db/items_tests.rs

@@ -1,475 +0,0 @@
-use std::collections::HashMap;
-
-use crate::{TidalDb, schema::EntityId};
-
-/// Minimal valid schema so ephemeral mode wires in-memory storage (needed by
-/// any test that actually persists an item).
-fn minimal_schema() -> crate::schema::Schema {
-    use crate::schema::{DecaySpec, EntityKind, SchemaBuilder, Window};
-    let mut b = SchemaBuilder::new();
-    let _ = b
-        .signal(
-            "view",
-            EntityKind::Item,
-            DecaySpec::Exponential {
-                half_life: std::time::Duration::from_secs(3600),
-            },
-        )
-        .windows(&[Window::AllTime])
-        .velocity(false)
-        .add();
-    b.build().expect("schema must be valid")
-}
-
-/// The apply-side ledger counts both edges, and counting a failure must not
-/// turn it into a success.
-///
-/// `apply_failed` exists to make a lost blob visible; if incrementing it also
-/// swallowed the `Err`, the receiver would advance past a record it never
-/// applied and the counter would document a silent data loss instead of
-/// preventing one (`CODING_GUIDELINES` :193-195).
-#[test]
-fn blob_apply_counts_both_edges_and_still_propagates_the_error() {
-    use crate::wal::format::batch::{BlobRecord, EmbeddingRecord, TermMarkerRecord};
-
-    let db = TidalDb::builder()
-        .ephemeral()
-        .with_schema(minimal_schema())
-        .open()
-        .unwrap();
-
-    // Success edge: a valid, finite, non-zero-norm embedding.
-    db.apply_replicated_blobs(vec![BlobRecord::Embedding(EmbeddingRecord {
-        entity_id: 1,
-        values: vec![1.0, 0.0, 0.0],
-    })])
-    .expect("a valid embedding blob must apply");
-
-    // Failure edge: term 0 is invalid by construction (no topology era ever
-    // journals one), so Phase 1 rejects it deterministically.
-    let err = db
-        .apply_replicated_blobs(vec![BlobRecord::TermMarker(TermMarkerRecord {
-            term: 0,
-            leader_region: 0,
-        })])
-        .expect_err("term 0 must be rejected, not counted-and-swallowed");
-    assert!(
-        format!("{err}").contains("term 0"),
-        "the original error must reach the caller unchanged, got: {err}"
-    );
-
-    let mut out = String::new();
-    db.metrics.cluster.render_into(&mut out, 0);
-    assert!(
-        out.contains(r#"tidaldb_cluster_blobs_applied_total{kind="embedding",partition_id="0"} 1"#),
-        "successful apply must be counted:\n{out}"
-    );
-    assert!(
-        out.contains(
-            r#"tidaldb_cluster_blobs_apply_failed_total{kind="term_marker",partition_id="0"} 1"#
-        ),
-        "failed apply must be counted under its own kind:\n{out}"
-    );
-    assert!(
-        out.contains(
-            r#"tidaldb_cluster_blobs_applied_total{kind="term_marker",partition_id="0"} 0"#
-        ),
-        "a failed apply must NOT also count as applied:\n{out}"
-    );
-}
-
-/// The replicated vector count must increment on the FOLLOWER apply path,
-/// which never goes through `wal_blob_first`.
-///
-/// This is the half that makes the series usable: the divergence alert
-/// compares replicas of a group and pages on a spread. If only the
-/// originating node counted, each vector would be counted on exactly one
-/// node, three replicas would NEVER agree, and `max - min` would be
-/// permanently nonzero — a worse signal than the raw count it replaces.
-#[test]
-fn replicated_vectors_total_increments_on_the_follower_apply_path() {
-    use std::sync::atomic::Ordering;
-
-    use crate::wal::format::batch::{
-        BlobRecord, EmbeddingRecord, ItemMetadataRecord, TermMarkerRecord,
-    };
-
-    // Ephemeral: no WAL, so `wal_blob_first` returns `Ok(None)` here and
-    // cannot be what increments the counter — only the apply path can.
-    let db = TidalDb::builder()
-        .ephemeral()
-        .with_schema(minimal_schema())
-        .open()
-        .unwrap();
-    assert_eq!(
-        db.metrics
-            .usearch_replicated_vectors_total
-            .load(Ordering::Relaxed),
-        0,
-        "a freshly opened node has counted no replicated vectors"
-    );
-
-    // Two embeddings and one metadata record in one round: only the vectors
-    // count.
-    db.apply_replicated_blobs(vec![
-        BlobRecord::Embedding(EmbeddingRecord {
-            entity_id: 1,
-            values: vec![1.0, 0.0, 0.0],
-        }),
-        BlobRecord::Embedding(EmbeddingRecord {
-            entity_id: 2,
-            values: vec![0.0, 1.0, 0.0],
-        }),
-        BlobRecord::ItemMetadata(ItemMetadataRecord {
-            entity_id: 3,
-            metadata_bytes: crate::db::metadata::serialize_metadata(&HashMap::new()),
-        }),
-    ])
-    .expect("valid replicated blobs must apply");
-    assert_eq!(
-        db.metrics
-            .usearch_replicated_vectors_total
-            .load(Ordering::Relaxed),
-        2,
-        "each applied embedding is one replicated vector; metadata is not a vector"
-    );
-
-    // A halted round applies nothing, so it must count nothing: term 0 is
-    // rejected in Phase 1, before the embedding beside it is ever applied.
-    let err = db
-        .apply_replicated_blobs(vec![
-            BlobRecord::Embedding(EmbeddingRecord {
-                entity_id: 4,
-                values: vec![0.0, 0.0, 1.0],
-            }),
-            BlobRecord::TermMarker(TermMarkerRecord {
-                term: 0,
-                leader_region: 0,
-            }),
-        ])
-        .expect_err("term 0 must halt the round");
-    assert!(
-        format!("{err}").contains("term 0"),
-        "unexpected error: {err}"
-    );
-    assert_eq!(
-        db.metrics
-            .usearch_replicated_vectors_total
-            .load(Ordering::Relaxed),
-        2,
-        "a vector that was never applied must not be counted as held"
-    );
-
-    // And it reaches the exposition, per-group shape included.
-    let out = db.metrics().render_prometheus();
-    assert!(
-        out.contains("\ntidaldb_usearch_replicated_vectors_total 2\n"),
-        "the follower's count must be exposed unlabeled for its own group:\n{out}"
-    );
-}
-
-/// The originating node counts the same vector, keyed on the one definition
-/// of "replicated": `wal_blob_first` returning `Ok(Some(seq))`.
-///
-/// Persistent + `peer_shards` non-empty is the only configuration where that
-/// happens; a standalone node's blobs never enter a replication stream, so
-/// they must not inflate a series whose whole point is that its replicas
-/// agree.
-#[test]
-fn replicated_vectors_total_increments_at_the_origin_only_in_cluster_mode() {
-    use std::sync::atomic::Ordering;
-
-    use crate::db::config::NodeConfig;
-    use crate::replication::ShardId;
-
-    // Standalone: `wal_blob_first` short-circuits to `Ok(None)`.
-    let solo_dir = tempfile::tempdir().unwrap();
-    let solo = TidalDb::builder()
-        .with_data_dir(solo_dir.path())
-        .with_schema(minimal_schema())
-        .open()
-        .unwrap();
-    assert_eq!(
-        solo.write_item_embedding(EntityId::new(1), &[1.0, 0.0, 0.0])
-            .unwrap(),
-        None,
-        "a standalone write never enters a replication stream"
-    );
-    assert_eq!(
-        solo.metrics
-            .usearch_replicated_vectors_total
-            .load(Ordering::Relaxed),
-        0,
-        "no WAL seqno means no replicated vector"
-    );
-    solo.close().unwrap();
-
-    // Cluster mode with a real WAL: the append is journaled and the counter
-    // follows the `Some(seq)`.
-    let dir = tempfile::tempdir().unwrap();
-    let db = TidalDb::builder()
-        .with_data_dir(dir.path())
-        .with_schema(minimal_schema())
-        .with_cluster(NodeConfig {
-            peer_shards: vec![ShardId(1)],
-            ..NodeConfig::default()
-        })
-        .open()
-        .unwrap();
-
-    let seq = db
-        .write_item_embedding(EntityId::new(1), &[1.0, 0.0, 0.0])
-        .unwrap();
-    assert!(
-        seq.is_some(),
-        "a cluster-mode embedding write must be journaled"
-    );
-    assert_eq!(
-        db.metrics
-            .usearch_replicated_vectors_total
-            .load(Ordering::Relaxed),
-        1,
-        "the journaled embedding is one replicated vector"
-    );
-
-    // Metadata rides the same WAL-first path but is not a vector.
-    let mut meta = HashMap::new();
-    meta.insert("category".to_string(), "doc".to_string());
-    assert!(
-        db.write_item_with_metadata(EntityId::new(2), &meta)
-            .unwrap()
-            .is_some(),
-        "a cluster-mode metadata write must also be journaled"
-    );
-    assert_eq!(
-        db.metrics
-            .usearch_replicated_vectors_total
-            .load(Ordering::Relaxed),
-        1,
-        "only embeddings are vectors; metadata must not inflate the count"
-    );
-    db.close().unwrap();
-}
-
-#[test]
-fn write_item_rejects_oversized_metadata_value() {
-    let db = TidalDb::builder().ephemeral().open().unwrap();
-    let mut meta = HashMap::new();
-    // Insert a value that exceeds the 8 KB per-value limit.
-    meta.insert("big_key".to_string(), "x".repeat(9 * 1024));
-    let err = db
-        .write_item_with_metadata(EntityId::new(1), &meta)
-        .unwrap_err();
-    assert!(
-        err.to_string().contains("metadata value too long"),
-        "expected value-too-long error, got: {err}"
-    );
-    db.close().unwrap();
-}
-
-#[test]
-fn write_item_rejects_too_many_metadata_keys() {
-    let db = TidalDb::builder().ephemeral().open().unwrap();
-    let mut meta = HashMap::new();
-    for i in 0..65 {
-        meta.insert(format!("key_{i}"), "val".to_string());
-    }
-    let err = db
-        .write_item_with_metadata(EntityId::new(1), &meta)
-        .unwrap_err();
-    assert!(
-        err.to_string().contains("max key count"),
-        "expected key-count error, got: {err}"
-    );
-    db.close().unwrap();
-}
-
-#[test]
-fn write_item_rejects_oversized_total_metadata() {
-    let db = TidalDb::builder().ephemeral().open().unwrap();
-    let mut meta = HashMap::new();
-    // 10 keys x 7 KB values = 70 KB > 64 KB limit.
-    for i in 0..10 {
-        meta.insert(format!("k{i}"), "x".repeat(7 * 1024));
-    }
-    let err = db
-        .write_item_with_metadata(EntityId::new(1), &meta)
-        .unwrap_err();
-    assert!(
-        err.to_string().contains("total size too large"),
-        "expected total-size error, got: {err}"
-    );
-    db.close().unwrap();
-}
-
-#[test]
-fn write_item_rejects_id_above_u32_universe_limit() {
-    let db = TidalDb::builder()
-        .ephemeral()
-        .with_schema(minimal_schema())
-        .open()
-        .unwrap();
-    let meta = HashMap::new();
-    // u32::MAX + 1: the smallest id that would alias a lower id once
-    // narrowed to u32 for the candidate-generation bitmaps.
-    let oversized = EntityId::new(u64::from(u32::MAX) + 1);
-    let err = db.write_item_with_metadata(oversized, &meta).unwrap_err();
-    assert!(
-        err.to_string().contains("u32 item-universe limit"),
-        "expected u32 item-universe rejection, got: {err}"
-    );
-    // The rejected item must not have leaked into storage or the universe
-    // bitmap — a rejected write leaves no trace.
-    assert_eq!(db.item_count(), 0, "rejected item must not enter universe");
-    assert!(
-        db.get_item_metadata(oversized).unwrap().is_none(),
-        "rejected item must not be persisted"
-    );
-    db.close().unwrap();
-}
-
-#[test]
-fn write_item_accepts_id_at_u32_universe_boundary() {
-    let db = TidalDb::builder()
-        .ephemeral()
-        .with_schema(minimal_schema())
-        .open()
-        .unwrap();
-    let meta = HashMap::new();
-    // u32::MAX is the largest in-bounds id: narrowing is lossless.
-    let boundary = EntityId::new(u64::from(u32::MAX));
-    db.write_item_with_metadata(boundary, &meta).unwrap();
-    assert_eq!(db.item_count(), 1, "boundary id must be accepted");
-    db.close().unwrap();
-}
-
-#[test]
-fn overwrite_scrubs_stale_index_entries() {
-    // W22 regression: write_item_with_metadata is "write OR overwrite". Before
-    // the re-index scrub, the in-memory bitmap/range indexes only ever
-    // INSERTED, so overwriting an item with new values left it indexed under
-    // BOTH the old and new value — a phantom hit on metadata-filtered
-    // RETRIEVE/SEARCH and a corrupted recency order. The durable store is
-    // always correct; this exercises the LIVE index, which is what diverged.
-    use std::ops::Bound;
-
-    let make = |pairs: &[(&str, &str)]| -> HashMap<String, String> {
-        pairs
-            .iter()
-            .map(|(k, v)| ((*k).to_string(), (*v).to_string()))
-            .collect()
-    };
-
-    let db = TidalDb::builder()
-        .ephemeral()
-        .with_schema(minimal_schema())
-        .open()
-        .unwrap();
-    let id = EntityId::new(1);
-
-    // First write under the OLD values.
-    db.write_item_with_metadata(
-        id,
-        &make(&[
-            ("category", "news"),
-            ("creator_id", "10"),
-            ("tags", "alpha,beta"),
-            ("duration", "300"),
-        ]),
-    )
-    .unwrap();
-    assert!(
-        db.category_index.get("news").is_some_and(|b| b.contains(1)),
-        "item must be indexed under its initial category"
-    );
-    assert!(db.creator_items.get(10).is_some_and(|b| b.contains(1)));
-
-    // Overwrite with ALL-NEW values.
-    db.write_item_with_metadata(
-        id,
-        &make(&[
-            ("category", "sports"),
-            ("creator_id", "20"),
-            ("tags", "gamma"),
-            ("duration", "600"),
-        ]),
-    )
-    .unwrap();
-
-    // No stale entry survives under any OLD value (the phantom-hit bug).
-    assert!(
-        db.category_index.get("news").is_none_or(|b| !b.contains(1)),
-        "stale category 'news' must be scrubbed on overwrite"
-    );
-    assert!(
-        db.creator_index.get("10").is_none_or(|b| !b.contains(1)),
-        "stale creator '10' must be scrubbed"
-    );
-    assert!(
-        db.tag_index.get("alpha").is_none_or(|b| !b.contains(1)),
-        "stale tag 'alpha' must be scrubbed"
-    );
-    assert!(
-        db.creator_items.get(10).is_none_or(|b| !b.contains(1)),
-        "stale creator_items[10] association must be scrubbed"
-    );
-    assert!(
-        !db.duration_index
-            .range(Bound::Included(&300), Bound::Included(&300))
-            .contains(1),
-        "stale duration 300 must be scrubbed"
-    );
-
-    // The NEW values are all present.
-    assert!(
-        db.category_index
-            .get("sports")
-            .is_some_and(|b| b.contains(1))
-    );
-    assert!(db.creator_index.get("20").is_some_and(|b| b.contains(1)));
-    assert!(db.tag_index.get("gamma").is_some_and(|b| b.contains(1)));
-    assert!(db.creator_items.get(20).is_some_and(|b| b.contains(1)));
-    assert!(
-        db.duration_index
-            .range(Bound::Included(&600), Bound::Included(&600))
-            .contains(1),
-        "new duration 600 must be indexed"
-    );
-    db.close().unwrap();
-}
-
-#[test]
-fn cluster_node_stamps_configured_partition_id() {
-    // W13 regression: the construction site must call `set_partition_id` with
-    // the configured shard, so each node's identity metric lines carry its own
-    // `partition_id` instead of the default `0` (otherwise cluster series
-    // collide when scraped into one Prometheus). Exercises the real build
-    // wiring end-to-end, not just the setter.
-    use crate::db::config::NodeConfig;
-    use crate::replication::ShardId;
-
-    let db = TidalDb::builder()
-        .ephemeral()
-        .with_schema(minimal_schema())
-        .with_cluster(NodeConfig {
-            shard_id: ShardId(3),
-            ..NodeConfig::default()
-        })
-        .open()
-        .unwrap();
-
-    let out = db.metrics().render_prometheus();
-    assert!(
-        out.contains("tidaldb_health_ok{partition_id=\"3\"}"),
-        "node must stamp its configured shard id (3) on identity lines: {out}"
-    );
-    assert!(
-        out.contains("tidaldb_uptime_seconds{partition_id=\"3\"}"),
-        "uptime line must carry the configured partition_id: {out}"
-    );
-    assert!(
-        !out.contains("partition_id=\"0\""),
-        "a shard-3 node must not emit the single-node default partition_id=0: {out}"
-    );
-    db.close().unwrap();
-}

tidal/src/db/metrics/mod.rs

@@ -221,34 +221,6 @@ pub struct MetricsState {
     /// Number of vectors stored in the `USearch` index.
     #[cfg(feature = "metrics")]
     pub(crate) usearch_vector_count: AtomicU64,
-    /// Number of vectors this group has seen enter the replicated stream since
-    /// process start — the replication-safe companion to
-    /// [`usearch_vector_count`](Self::usearch_vector_count).
-    ///
-    /// The raw vector count cannot distinguish a real divergence from a
-    /// `/sharded/*` single-copy write, which lands on exactly one owner by
-    /// design and never converges; that benign, permanent skew is why the
-    /// divergence alert had to be demoted to a warning. This series counts ONLY
-    /// vectors that entered the replicated log, so its replicas MUST agree and
-    /// a spread is always a real defect.
-    ///
-    /// **Process-lifetime, and it resets to 0 on restart.** Nothing rebuilds it
-    /// at open (that would mean retro-classifying stored vectors, which no
-    /// on-disk state supports). Replicas therefore share a starting point only
-    /// until the first pod restart, after which the restarted replica reads
-    /// lower than its peers FOREVER. Any alert on this series must be
-    /// restart-aware — compare `increase(...[w])` over a window, or gate the
-    /// raw comparison on every replica's process uptime exceeding the window —
-    /// because a raw cross-replica `max - min` pages on every rollout.
-    ///
-    /// Incremented on BOTH sides of one replicated write — the originating node
-    /// when its `wal_blob_first` append is durable, and every applying follower
-    /// in the live blob-apply path. Counting only the origin would make it a
-    /// per-node-origin counter (each vector counted on exactly one node), so
-    /// three replicas would never agree and the alert built on it would page
-    /// forever. See [`observe_replicated_vectors`](Self::observe_replicated_vectors).
-    #[cfg(feature = "metrics")]
-    pub(crate) usearch_replicated_vectors_total: AtomicU64,
     /// Total cardinality across all bitmap index entries (category + format + creator + tag).
     #[cfg(feature = "metrics")]
     pub(crate) bitmap_index_cardinality: AtomicU64,
@@ -375,8 +347,6 @@ impl MetricsState {
             #[cfg(feature = "metrics")]
             usearch_vector_count: AtomicU64::new(0),
             #[cfg(feature = "metrics")]
-            usearch_replicated_vectors_total: AtomicU64::new(0),
-            #[cfg(feature = "metrics")]
             bitmap_index_cardinality: AtomicU64::new(0),
             checkpoint_failures_total: AtomicU64::new(0),
             checkpoint_thread_died: AtomicBool::new(false),
@@ -438,9 +408,7 @@ impl MetricsState {
 
     /// Register a co-located shard group's NODE-level metrics so this node's one
     /// `/metrics` listener also exposes that group's per-group node series
-    /// (`tidaldb_usearch_vector_count` and
+    /// (currently `tidaldb_usearch_vector_count{shard="<shard>"}`).
-    /// `tidaldb_usearch_replicated_vectors_total`, each stamped
-    /// `shard="<shard>"`).
     ///
     /// Separate from [`register_cluster_sibling`](Self::register_cluster_sibling)
     /// because the two carry different state: that one shares
@@ -453,51 +421,6 @@ impl MetricsState {
             .push((shard, metrics));
     }
 
-    /// Count `n` vectors that entered this group's replicated stream.
-    ///
-    /// Called from exactly two places, and BOTH are required for the series to
-    /// mean anything:
-    /// 1. the **originating** node, once its `wal_blob_first` append is durable
-    ///    (`Ok(Some(seq))` — the single definition of "replicated"), and
-    /// 2. every **applying follower**, in the live blob-apply path, which never
-    ///    goes through `wal_blob_first`.
-    ///
-    /// With only (1) each vector is counted on exactly one node, the replicas of
-    /// a group never converge, and the divergence alert reading `max - min`
-    /// pages forever — strictly worse than the raw count it replaces. Boot-time
-    /// WAL replay is deliberately NOT counted, for the same reason
-    /// `tidaldb_cluster_blobs_applied_total` excludes it: replay would re-count
-    /// records this process already counted before it restarted.
-    #[cfg(feature = "metrics")]
-    pub(crate) fn observe_replicated_vectors(&self, n: u64) {
-        self.usearch_replicated_vectors_total
-            .fetch_add(n, Ordering::Relaxed);
-    }
-
-    /// Render one `<name>{shard="N"} <value>` line per co-located shard group.
-    ///
-    /// Every per-group NODE-level series goes through this one renderer so a
-    /// second series cannot drift into a different shape. The caller writes the
-    /// owner's line UNLABELED (wire compatibility) and the owner is
-    /// deterministically the same group on every node, so a cross-node
-    /// comparison grouped `by (shard)` puts each replica set in its own bucket
-    /// (owner -> the empty-label bucket) and never mixes unlike groups or
-    /// double-counts a node. Label cardinality stays closed: `shard` only ever
-    /// takes the `u16` ids handed to
-    /// [`register_node_sibling`](Self::register_node_sibling).
-    #[cfg(feature = "metrics")]
-    fn render_node_sibling_series(&self, out: &mut String, name: &str, load: fn(&Self) -> u64) {
-        use std::fmt::Write;
-        for (shard, sib) in self
-            .node_siblings
-            .read()
-            .unwrap_or_else(std::sync::PoisonError::into_inner)
-            .iter()
-        {
-            let _ = writeln!(out, "{name}{{shard=\"{shard}\"}} {}", load(sib));
-        }
-    }
-
     /// Maximum age (nanoseconds) a checkpoint may reach before health reports
     /// degraded. The periodic checkpoint thread runs every 30s; a checkpoint
     /// older than 5 minutes means the thread is stuck, dead, or failing — all
@@ -717,32 +640,27 @@ impl MetricsState {
                 "gauge",
                 self.usearch_vector_count.load(Ordering::Relaxed) as f64,
             );
-            // Co-located groups' vector counts, each stamped `shard="N"`; the
+            // Co-located groups' vector counts, each stamped `shard="N"`. The
-            // owner's line above stays UNLABELED (see
+            // owner's line above stays UNLABELED for wire compatibility, and the
-            // `render_node_sibling_series`).
+            // owner is deterministically the same group on every node, so a
-            self.render_node_sibling_series(&mut out, "tidaldb_usearch_vector_count", |m| {
+            // cross-node comparison grouped `by (shard)` puts each replica set in
-                m.usearch_vector_count.load(Ordering::Relaxed)
+            // its own bucket (owner -> the empty-label bucket) and never mixes
-            });
+            // unlike groups or double-counts a node.
-            // Replication-safe companion to the count above: only vectors that
+            {
-            // entered the replicated log, counted identically on the originating
+                use std::fmt::Write;
-            // node and on every applying follower. A `/sharded/*` single-copy
+                for (shard, sib) in self
-            // write never enters it, so unlike the raw count a spread between
+                    .node_siblings
-            // replicas of a group is ALWAYS a real divergence. Same
+                    .read()
-            // owner-unlabeled + `shard="N"` shape, so both series group
+                    .unwrap_or_else(std::sync::PoisonError::into_inner)
-            // `by (shard)` the same way.
+                    .iter()
-            write_metric_line(
+                {
-                &mut out,
+                    let _ = writeln!(
-                "tidaldb_usearch_replicated_vectors_total",
+                        &mut out,
-                "Vectors that entered the replicated WAL stream since process start",
+                        "tidaldb_usearch_vector_count{{shard=\"{shard}\"}} {}",
-                "counter",
+                        sib.usearch_vector_count.load(Ordering::Relaxed)
-                self.usearch_replicated_vectors_total
+                    );
-                    .load(Ordering::Relaxed) as f64,
+                }
-            );
+            }
-            self.render_node_sibling_series(
-                &mut out,
-                "tidaldb_usearch_replicated_vectors_total",
-                |m| m.usearch_replicated_vectors_total.load(Ordering::Relaxed),
-            );
             write_metric_line(
                 &mut out,
                 "tidaldb_bitmap_index_cardinality",
@@ -931,3 +849,64 @@ mod diagnostics;
 
 #[cfg(test)]
 mod tests;
+
+#[cfg(test)]
+mod partition_id_tests {
+    use super::*;
+
+    /// A fresh metrics state stamps the single-node default `partition_id="0"`
+    /// onto every node-identity line — preserving the pre-W13 behavior when the
+    /// shard id is never installed.
+    #[test]
+    fn default_partition_id_is_zero() {
+        let state = MetricsState::new();
+        let out = state.render_prometheus();
+        assert!(out.contains("tidaldb_uptime_seconds{partition_id=\"0\"}"));
+        assert!(out.contains("tidaldb_health_ok{partition_id=\"0\"}"));
+        assert!(out.contains("partition_id=\"0\"} 1"));
+    }
+
+    /// After the construction site installs a real shard id (cluster mode), the
+    /// node-identity label reflects it so per-node series do not collide when
+    /// many nodes are scraped into one Prometheus (W13). The old hardcoded `0`
+    /// must no longer appear on those lines.
+    #[test]
+    fn cluster_partition_id_is_stamped_on_node_lines() {
+        let state = MetricsState::new();
+        state.set_partition_id(7);
+        let out = state.render_prometheus();
+        assert!(out.contains("tidaldb_uptime_seconds{partition_id=\"7\"}"));
+        assert!(out.contains("tidaldb_health_ok{partition_id=\"7\"}"));
+        assert!(out.contains("partition_id=\"7\"} 1"));
+        assert!(
+            !out.contains("partition_id=\"0\""),
+            "cluster node must not emit the single-node default partition_id: {out}"
+        );
+    }
+
+    /// The unconditional `checkpoint_failures_total` counter must carry the
+    /// `partition_id` label like every other node-identity series, so per-node
+    /// failure counts stay distinct in a multi-node Prometheus scrape (W13).
+    #[test]
+    fn checkpoint_failures_total_carries_partition_id() {
+        let state = MetricsState::new();
+        let out = state.render_prometheus();
+        assert!(
+            out.contains("tidaldb_checkpoint_failures_total{partition_id=\"0\"}"),
+            "checkpoint_failures_total must carry the default partition_id label: {out}"
+        );
+        // Never emit the old unlabeled series.
+        assert!(
+            !out.contains("\ntidaldb_checkpoint_failures_total 0\n"),
+            "checkpoint_failures_total must not emit an unlabeled series: {out}"
+        );
+
+        let cluster = MetricsState::new();
+        cluster.set_partition_id(7);
+        let cout = cluster.render_prometheus();
+        assert!(
+            cout.contains("tidaldb_checkpoint_failures_total{partition_id=\"7\"}"),
+            "checkpoint_failures_total must reflect the cluster shard id: {cout}"
+        );
+    }
+}

tidal/src/db/metrics/tests.rs

@@ -451,116 +451,3 @@ fn every_colocated_group_exposes_its_own_vector_count() {
         "one vector-count series per hosted group, no duplicates:\n{out}"
     );
 }
-
-/// The replicated-only vector count must be rendered with the SAME per-group
-/// shape as the raw count: unlabeled for the metrics owner, `shard="N"` for each
-/// co-located sibling, one series per hosted group.
-///
-/// The divergence alert groups `by (shard)` and compares replicas of a group. A
-/// second, differently-shaped per-group series would either mix unlike groups
-/// into one bucket or leave a group with no series at all — the exact hole the
-/// unlabeled-owner convention was introduced to close on 2026-08-30.
-#[test]
-fn every_colocated_group_exposes_its_own_replicated_vectors_total() {
-    let owner = MetricsState::new();
-    owner
-        .usearch_replicated_vectors_total
-        .store(9001, Ordering::Relaxed);
-
-    for (shard, count) in [(1u16, 77u64), (2u16, 88u64)] {
-        let sib = std::sync::Arc::new(MetricsState::new());
-        sib.usearch_replicated_vectors_total
-            .store(count, Ordering::Relaxed);
-        owner.register_node_sibling(shard, sib);
-    }
-
-    let out = owner.render_prometheus();
-    assert!(
-        out.contains("\ntidaldb_usearch_replicated_vectors_total 9001\n"),
-        "owner series must stay unlabeled, like the raw count:\n{out}"
-    );
-    assert!(
-        out.contains("tidaldb_usearch_replicated_vectors_total{shard=\"1\"} 77"),
-        "group 1 replicated count missing:\n{out}"
-    );
-    assert!(
-        out.contains("tidaldb_usearch_replicated_vectors_total{shard=\"2\"} 88"),
-        "group 2 replicated count missing:\n{out}"
-    );
-    assert_eq!(
-        out.lines()
-            .filter(|l| l.starts_with("tidaldb_usearch_replicated_vectors_total"))
-            .count(),
-        3,
-        "one replicated-count series per hosted group, no duplicates:\n{out}"
-    );
-    // The raw count is a separate series and must not have been disturbed by
-    // sharing the sibling renderer.
-    assert_eq!(
-        out.lines()
-            .filter(|l| l.starts_with("tidaldb_usearch_vector_count"))
-            .count(),
-        3,
-        "the raw per-group count must still render one series per group:\n{out}"
-    );
-}
-
-// ── partition_id stamping (W13) ─────────────────────────────────────────────
-// Moved here from an inline `mod partition_id_tests` in `mod.rs` so the module
-// carries implementation only; `tests.rs` is this module's one test home.
-
-/// A fresh metrics state stamps the single-node default `partition_id="0"`
-/// onto every node-identity line — preserving the pre-W13 behavior when the
-/// shard id is never installed.
-#[test]
-fn default_partition_id_is_zero() {
-    let state = MetricsState::new();
-    let out = state.render_prometheus();
-    assert!(out.contains("tidaldb_uptime_seconds{partition_id=\"0\"}"));
-    assert!(out.contains("tidaldb_health_ok{partition_id=\"0\"}"));
-    assert!(out.contains("partition_id=\"0\"} 1"));
-}
-
-/// After the construction site installs a real shard id (cluster mode), the
-/// node-identity label reflects it so per-node series do not collide when
-/// many nodes are scraped into one Prometheus (W13). The old hardcoded `0`
-/// must no longer appear on those lines.
-#[test]
-fn cluster_partition_id_is_stamped_on_node_lines() {
-    let state = MetricsState::new();
-    state.set_partition_id(7);
-    let out = state.render_prometheus();
-    assert!(out.contains("tidaldb_uptime_seconds{partition_id=\"7\"}"));
-    assert!(out.contains("tidaldb_health_ok{partition_id=\"7\"}"));
-    assert!(out.contains("partition_id=\"7\"} 1"));
-    assert!(
-        !out.contains("partition_id=\"0\""),
-        "cluster node must not emit the single-node default partition_id: {out}"
-    );
-}
-
-/// The unconditional `checkpoint_failures_total` counter must carry the
-/// `partition_id` label like every other node-identity series, so per-node
-/// failure counts stay distinct in a multi-node Prometheus scrape (W13).
-#[test]
-fn checkpoint_failures_total_carries_partition_id() {
-    let state = MetricsState::new();
-    let out = state.render_prometheus();
-    assert!(
-        out.contains("tidaldb_checkpoint_failures_total{partition_id=\"0\"}"),
-        "checkpoint_failures_total must carry the default partition_id label: {out}"
-    );
-    // Never emit the old unlabeled series.
-    assert!(
-        !out.contains("\ntidaldb_checkpoint_failures_total 0\n"),
-        "checkpoint_failures_total must not emit an unlabeled series: {out}"
-    );
-
-    let cluster = MetricsState::new();
-    cluster.set_partition_id(7);
-    let cout = cluster.render_prometheus();
-    assert!(
-        cout.contains("tidaldb_checkpoint_failures_total{partition_id=\"7\"}"),
-        "checkpoint_failures_total must reflect the cluster shard id: {cout}"
-    );
-}

tidalctl/src/commands/live.rs

@@ -30,12 +30,6 @@
 //! Shares the crate contract: `0` ok, `1` usage error, [`EXIT_DEGRADED`] (2) when
 //! the server is unreachable or answered non-2xx. That keeps
 //! `tidalctl cluster-status --url … && deploy` honest.
-//!
-//! `cluster-status` / `watch` additionally exit [`EXIT_DEGRADED`] when the cluster
-//! is NOT CONVERGED — a KNOWN non-zero lag, an unreachable or partitioned region,
-//! a pending reseed, or a local shard this node lags on. A region whose frontier
-//! the server reports as `null` ("no report") is an UNKNOWN, not a deficit, and
-//! does not by itself make the command fail; see [`status_is_degraded`].
 
 use std::time::Duration;
 
@@ -253,16 +247,6 @@ pub(crate) fn run_feed(
     }
 }
 
-/// The process exit code a convergence verdict maps to.
-///
-/// The load-bearing half of `tidalctl cluster-status && deploy`: extracted from
-/// its two call sites so the mapping itself is unit-testable. A converged cluster
-/// returning anything but `0` silently disables the one place an operator was
-/// told to gate a deploy on cluster health.
-const fn convergence_exit(degraded: bool) -> i32 {
-    if degraded { EXIT_DEGRADED } else { 0 }
-}
-
 /// `tidalctl cluster-status` — the replication view of a live cluster.
 ///
 /// `--pretty` emits raw JSON; the default is a compact human summary, because
@@ -275,7 +259,7 @@ pub(crate) fn run_cluster_status(target: &Target, pretty: bool) -> Result<(Strin
                 return Ok((render(&body, true), 0));
             }
             let (text, degraded) = summarize_status(&body);
-            Ok((text, convergence_exit(degraded)))
+            Ok((text, if degraded { EXIT_DEGRADED } else { 0 }))
         }
         Err(e) => Ok(e.to_output()),
     }
@@ -300,7 +284,7 @@ pub(crate) fn run_watch(
             Ok(body) => {
                 let (line, degraded) = watch_line(&body);
                 println!("{line}");
-                last_exit = convergence_exit(degraded);
+                last_exit = if degraded { EXIT_DEGRADED } else { 0 };
             }
             Err(e) => {
                 let (text, code) = e.to_output();
@@ -327,45 +311,34 @@ fn render(body: &serde_json::Value, pretty: bool) -> String {
     }
 }
 
-/// A KNOWN non-zero region lag, an unreachable region, a partition, or a pending
+/// Non-zero lag, an unreachable region, a partition, or a pending reseed all
-/// reseed all mean "not converged" and drive the degraded exit code. A shard row
+/// mean "not converged" and drive the degraded exit code.
-/// this node lags on does too — `shards[]` is the node's own authoritative view
-/// of the groups it hosts, and it is the surface that stayed trustworthy while
-/// `regions[]` was fabricating.
-///
-/// An **unknown** (`null`) region frontier does NOT. Since the server stopped
-/// inventing `applied_events: 0, lag_events: <leader hwm>` for a peer it could not
-/// probe, `null` means "no report", and no report is not a deficit. That is what
-/// makes `tidalctl cluster-status && deploy` usable: a converged cluster whose
-/// aggregator merely lacks a frontier for a peer now exits 0.
-///
-/// `partitioned: true` is checked EXPLICITLY and is never softened by an unknown
-/// frontier. A partitioned peer almost always also has an unknown frontier, so a
-/// verdict that only asked "are the remaining gaps unknowns?" would green-light a
-/// deploy in the middle of a partition — strictly worse than the over-eager exit 2
-/// this replaces. `/cluster/status` re-stamps `partitioned` from the leader's
-/// ship-skip set (`node.rs` `apply_leader_partition_view`), so the field is the
-/// authority here.
 fn status_is_degraded(body: &serde_json::Value) -> bool {
     let regions = body.get("regions").and_then(|r| r.as_array());
     let region_bad = regions.is_some_and(|rs| {
         rs.iter().any(|r| {
-            // A KNOWN deficit. `None` (absent or null) is an unknown and is
+            r.get("lag_events")
-            // deliberately not a deficit.
+                .and_then(serde_json::Value::as_u64)
-            frontier(r, "lag_events").is_some_and(|lag| lag > 0)
+                .unwrap_or(0)
-                || !flag(r, "reachable", true)
+                > 0
-                || flag(r, "partitioned", false)
+                || !r
+                    .get("reachable")
+                    .and_then(serde_json::Value::as_bool)
+                    .unwrap_or(true)
+                || r.get("partitioned")
+                    .and_then(serde_json::Value::as_bool)
+                    .unwrap_or(false)
         })
     });
     let shards = body.get("shards").and_then(|s| s.as_array());
     let shard_bad = shards.is_some_and(|ss| {
         ss.iter().any(|s| {
-            flag(s, "reseed_required", false)
+            s.get("reseed_required")
-                || flag(s, "reseeding", false)
+                .and_then(serde_json::Value::as_bool)
-                // This node's own frontier for a group it hosts. Load-bearing now
+                .unwrap_or(false)
-                // that a null region frontier is tolerated: the local shard view
+                || s.get("reseeding")
-                // is the deficit signal that cannot go unknown.
+                    .and_then(serde_json::Value::as_bool)
-                || num(s, "lag_events") > 0
+                    .unwrap_or(false)
         })
     });
     region_bad || shard_bad
@@ -388,39 +361,37 @@ pub(crate) fn summarize_status(body: &serde_json::Value) -> (String, bool) {
                 .get("name")
                 .and_then(serde_json::Value::as_str)
                 .unwrap_or("?");
-            let applied = frontier(r, "applied_events");
+            let applied = num(r, "applied_events");
-            let lag = frontier(r, "lag_events");
+            let lag = num(r, "lag_events");
             let reachable = flag(r, "reachable", true);
             let partitioned = flag(r, "partitioned", false);
             let mut notes = Vec::new();
-            // NO REPORT is now read off the WIRE: `/cluster/status` reports a peer
+            // `applied == 0` together with non-zero lag is NOT a follower that is
-            // it has no frontier for as `null`. The CLI used to INFER the gap from
+            // behind: it is the aggregated view having received no frontier report
-            // `applied == 0 && lag > 0`, because the server fabricated
+            // for that peer, so `lag` was derived against an uninitialised zero and
-            // `applied 0, lag = <leader hwm>` for an unprobed peer (observed live:
+            // equals the leader's whole history. Observed on a cluster where every
-            // two healthy peers shown at 13.3M lag while every node's own
+            // node individually reported lag=0 and converged, while this surface
-            // `shards[]` read lag 0). The inference is gone with the fabrication —
+            // called two healthy peers UNREACHABLE/PARTITIONED with 13.3M lag.
-            // and dropping it also un-hides the case it was smothering: a genuinely
+            // Saying "BEHIND" here would repeat that lie.
-            // WIPED replica sitting at a measured `applied: 0` with a real deficit
+            let no_report = applied == 0 && lag > 0;
-            // now correctly reads BEHIND instead of NO REPORT.
+            if no_report {
-            if applied.is_none() || lag.is_none() {
                 notes.push("NO REPORT (aggregated view; query the node directly)");
-            }
+            } else {
-            if !reachable {
+                if !reachable {
-                notes.push("UNREACHABLE");
+                    notes.push("UNREACHABLE");
-            }
+                }
-            if partitioned {
+                if partitioned {
-                notes.push("PARTITIONED");
+                    notes.push("PARTITIONED");
-            }
+                }
-            if lag.is_some_and(|l| l > 0) {
+                if lag > 0 {
-                notes.push("BEHIND");
+                    notes.push("BEHIND");
+                }
             }
             let note = if notes.is_empty() {
                 String::new()
             } else {
                 format!("  <- {}", notes.join(" "))
             };
-            let applied = cell(applied);
-            let lag = cell(lag);
             let _ = writeln!(out, "  {name:<12} applied={applied:<12} lag={lag}{note}");
         }
     }
@@ -494,7 +465,7 @@ fn watch_line(body: &serde_json::Value) -> (String, bool) {
                 .get("name")
                 .and_then(serde_json::Value::as_str)
                 .unwrap_or("?");
-            let lag = cell(frontier(r, "lag_events"));
+            let lag = num(r, "lag_events");
             let mark = if flag(r, "reachable", true) { "" } else { "!" };
             let _ = write!(line, "  {name}={lag}{mark}");
         }
@@ -508,36 +479,6 @@ fn num(v: &serde_json::Value, key: &str) -> u64 {
     v.get(key).and_then(serde_json::Value::as_u64).unwrap_or(0)
 }
 
-/// A wire frontier field: `Some(n)` when the server reported a number, `None`
-/// when it reported JSON `null` or omitted the field.
-///
-/// Deliberately NOT [`num`]: collapsing `null` to `0` is the fabrication this
-/// surface exists to stop reporting. `applied_events` / `lag_events` on
-/// `regions[]` are `Option<u64>` on the wire.
-fn frontier(v: &serde_json::Value, key: &str) -> Option<u64> {
-    v.get(key).and_then(serde_json::Value::as_u64)
-}
-
-/// Render a possibly-unknown frontier: the number, or `?` for "not reported".
-const fn cell(v: Option<u64>) -> DisplayFrontier {
-    DisplayFrontier(v)
-}
-
-/// `Display` wrapper so an unknown frontier renders as `?` while still honoring
-/// the `{:<12}` column width the numeric field had.
-struct DisplayFrontier(Option<u64>);
-
-impl std::fmt::Display for DisplayFrontier {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        // `pad` (not `write!`) so width/fill/alignment apply exactly as they did
-        // when this column was a `u64`.
-        match self.0 {
-            Some(n) => f.pad(&n.to_string()),
-            None => f.pad("?"),
-        }
-    }
-}
-
 fn num_at(arr: &[serde_json::Value], idx: usize) -> u64 {
     arr.get(idx)
         .and_then(serde_json::Value::as_u64)
@@ -570,109 +511,43 @@ mod tests {
         })
     }
 
-    /// A peer whose frontier the aggregator could not learn: `null`, the shape
-    /// `/cluster/status` now emits. Not to be confused with a measured zero.
-    fn with_unknown_peer_frontier(v: &mut serde_json::Value) {
-        v["regions"][1]["applied_events"] = serde_json::Value::Null;
-        v["regions"][1]["lag_events"] = serde_json::Value::Null;
-    }
-
     #[test]
     fn converged_status_is_not_degraded() {
         let (text, degraded) = summarize_status(&converged());
         assert!(!degraded, "a converged cluster must exit 0: {text}");
         assert!(text.contains("leader: tidaldb-0"));
         assert!(!text.contains("NOT CONVERGED"));
-        // The exit code is the load-bearing half: the documented
-        // `tidalctl cluster-status && deploy` gate is only usable if a converged
-        // cluster actually returns 0, and that regression is what this pins.
-        assert_eq!(
-            convergence_exit(degraded),
-            0,
-            "a converged cluster must exit 0, not {EXIT_DEGRADED}"
-        );
     }
 
-    /// `/cluster/status` reports a peer it has no frontier report for as `null`.
+    /// The aggregated `/cluster/status` reports a peer it has no frontier report
-    /// The summary must name the reporting gap, never render it as replication
+    /// for as `applied=0`, and derives `lag` against that zero — so a converged
-    /// lag. Observed live before the server was fixed: two healthy nodes shown
+    /// peer appears to be the leader's entire history behind. Observed live: two
-    /// UNREACHABLE/PARTITIONED at 13.3M lag while every node individually
+    /// healthy nodes shown UNREACHABLE/PARTITIONED at 13.3M lag while every node
-    /// reported lag=0 — the server fabricated `applied 0` and derived the deficit
+    /// individually reported lag=0. The summary must name the reporting gap, not
-    /// from it. Now the gap arrives as `null` and the CLI reads it off the wire.
+    /// repeat it as replication lag.
     #[test]
     fn missing_peer_report_is_not_reported_as_lag() {
         let mut v = converged();
-        with_unknown_peer_frontier(&mut v);
+        v["regions"][1]["applied_events"] = serde_json::json!(0);
+        v["regions"][1]["lag_events"] = serde_json::json!(13_322_237_u64);
         v["regions"][1]["reachable"] = serde_json::json!(false);
         v["regions"][1]["partitioned"] = serde_json::json!(true);
 
         let (text, degraded) = summarize_status(&v);
-        assert!(
+        assert!(degraded, "a missing report is still not-converged");
-            degraded,
-            "an unreachable + partitioned peer is still not-converged: {text}"
-        );
         assert!(text.contains("NO REPORT"), "{text}");
         assert!(
             !text.contains("BEHIND"),
-            "an unknown frontier is not a measured deficit: {text}"
+            "must not claim replication lag from an uninitialised zero: {text}"
         );
         assert!(
             text.contains("query the node directly"),
             "must point at the surface that can actually answer: {text}"
         );
-        assert!(
-            text.contains("lag=?"),
-            "an unknown lag must render as unknown, not as a number: {text}"
-        );
     }
 
-    /// The headline 04b fix: a cluster that is converged everywhere it CAN be
+    /// A genuine lag report (non-zero applied) must still say BEHIND, so the
-    /// measured, with the only remaining gap being a peer frontier the aggregator
+    /// carve-out above cannot hide a real follower falling behind.
-    /// does not know, exits 0. Before this, `lag_events` was fabricated for that
-    /// peer and the command exited 2 on a healthy cluster, making the documented
-    /// `cluster-status && deploy` gate unusable.
-    #[test]
-    fn converged_with_an_unknown_frontier_exits_zero() {
-        let mut v = converged();
-        with_unknown_peer_frontier(&mut v);
-        let (text, degraded) = summarize_status(&v);
-        assert!(
-            !degraded,
-            "an unknown frontier is not a deficit; this must exit 0: {text}"
-        );
-        assert_eq!(convergence_exit(degraded), 0);
-        assert!(
-            text.contains("NO REPORT"),
-            "the gap must still be VISIBLE, just not fatal: {text}"
-        );
-        assert!(!text.contains("NOT CONVERGED"), "{text}");
-    }
-
-    /// The case that would otherwise green-light a deploy mid-partition: a
-    /// partitioned peer almost always ALSO has an unknown frontier, so a verdict
-    /// that only asked "are the remaining gaps unknowns?" would call this
-    /// converged. `partitioned: true` is checked explicitly and wins.
-    #[test]
-    fn partitioned_peer_with_an_unknown_frontier_is_not_converged() {
-        let mut v = converged();
-        with_unknown_peer_frontier(&mut v);
-        v["regions"][1]["partitioned"] = serde_json::json!(true);
-        // Reachable on purpose: isolates `partitioned` as the sole trigger, so
-        // this cannot pass on the `!reachable` check by accident.
-        v["regions"][1]["reachable"] = serde_json::json!(true);
-
-        let (text, degraded) = summarize_status(&v);
-        assert!(
-            degraded,
-            "a partition must never be softened by an unknown frontier: {text}"
-        );
-        assert_eq!(convergence_exit(degraded), EXIT_DEGRADED);
-        assert!(text.contains("PARTITIONED"), "{text}");
-        assert!(text.contains("NOT CONVERGED"), "{text}");
-    }
-
-    /// A genuine lag report must still say BEHIND, so tolerating unknowns cannot
-    /// hide a real follower falling behind.
     #[test]
     fn genuine_lag_is_still_reported_as_behind() {
         let mut v = converged();
@@ -680,32 +555,10 @@ mod tests {
         v["regions"][1]["lag_events"] = serde_json::json!(10);
         let (text, degraded) = summarize_status(&v);
         assert!(degraded);
-        assert_eq!(convergence_exit(degraded), EXIT_DEGRADED);
         assert!(text.contains("BEHIND"), "{text}");
         assert!(!text.contains("NO REPORT"), "{text}");
     }
 
-    /// A replica sitting at a MEASURED `applied: 0` with a real deficit is behind,
-    /// not unreported — the PVC-wipe shape. The old `applied == 0 && lag > 0`
-    /// inference rendered exactly this as NO REPORT; reading the wire instead
-    /// un-hides it.
-    #[test]
-    fn measured_zero_frontier_with_real_lag_is_behind_not_unreported() {
-        let mut v = converged();
-        v["regions"][1]["applied_events"] = serde_json::json!(0);
-        v["regions"][1]["lag_events"] = serde_json::json!(13_322_237_u64);
-        let (text, degraded) = summarize_status(&v);
-        assert!(degraded);
-        assert!(
-            text.contains("BEHIND"),
-            "a measured zero with a measured deficit IS behind: {text}"
-        );
-        assert!(
-            !text.contains("NO REPORT"),
-            "the server reported both numbers; the CLI must not invent a gap: {text}"
-        );
-    }
-
     /// Every "not converged" shape must drive the degraded exit code, so
     /// `tidalctl cluster-status && deploy` cannot pass on a broken cluster.
     #[test]
@@ -734,17 +587,6 @@ mod tests {
             status_is_degraded(&reseed),
             "pending reseed must be degraded"
         );
-
-        // The node's own view of a group it hosts. Load-bearing now that an
-        // UNKNOWN region frontier is tolerated: `shards[]` is the surface that
-        // stayed trustworthy through the whole fabrication, so a local deficit
-        // here must still fail the gate.
-        let mut shard_lag = converged();
-        shard_lag["shards"][0]["lag_events"] = serde_json::json!(11);
-        assert!(
-            status_is_degraded(&shard_lag),
-            "a local shard deficit must be degraded"
-        );
     }
 
     #[test]