feat(m11): continuous correctness (m11p9) — fault classes, invariant checkers, soak gates, nightly pipeline

- fault-injection cargo feature (compiled OUT of prod): slow-fsync + disk-full WAL hooks in tidal/src/fault.rs, inert until armed, tier-3 builds with feature - first-class invariant checkers (tests/support/invariants.rs): AckLedger no-acked-loss (now consumed by m11p3 gate), feed parity, single-leader-per-term, monotonic frontiers - cluster_faults.rs tier-3 suite 4/4: disk-full degrade+recover, slow-fsync lag+converge, both-slow quorum 503, asymmetric partition no-split-brain - tidal-stress soak gates: --json-summary + --max-p99-ms/--max-error-pct/ --fail-on-knee → non-zero exit on regression - Woodpecker cron nightly flow (chaos + gated soak), event-routed, not GH Actions - guarantee-traceability.md: roadmap §2 guarantees → named tests (closes G-C apparatus; 30-day-green is a calendar criterion)
2026-06-13 15:23:59 -06:00 · 2026-06-13 15:23:59 -06:00 · 1265140e28
commit 1265140e28
parent 005e292cbb
25 changed files with 2241 additions and 161 deletions
--- a/.woodpecker.yaml
+++ b/.woodpecker.yaml
@ -1,16 +1,23 @@
-# Pipeline: a rolling-upgrade RELEASE GATE (m11p8) runs FIRST; only if it passes
+# Two pipelines in one file, split by event (Woodpecker, NEVER GitHub Actions):
-# does Kaniko build the tidal-server image and push it to the in-cluster zot
+#
-# registry. DEPLOYMENT IS MANUAL (kustomize, from the orchard9-k3sf ops repo) —
+#   • event: push  → the m11p8 RELEASE GATE (rolling-upgrade tier-3 test) then a
-# the old auto-`kubectl set image deployment/tidaldb` step was removed because it
+#     Kaniko image build. Deployment is MANUAL (kustomize, orchard9-k3sf ops repo).
-# coupled every image build to a standalone roll and the cluster ops repo's
+#   • event: cron  → the m11p9 NIGHTLY CONTINUOUS-CORRECTNESS run: the tier-3
-# contract is "manual deploy via scripts, no CI/CD deploy". The same
+#     chaos suites (incl. the new disk-full / slow-fsync / asymmetric-partition
-# `tidal-server` binary serves every subcommand (standalone AND multi-process
+#     fault classes) with elevated kill-points, then a tidal-stress soak with
-# `cluster --region`), so one image covers both deployments.
+#     PASS/FAIL regression gates on p99 + error rate. A nightly failure flags a
 #     correctness or performance regression for that day.
 #
 # Per-step `when:` routes each step to its event; the workflow-level `when` admits
 # both. Configure a cron named "nightly" in the Woodpecker repo settings to fire
 # the cron pipeline (the same `tidal-server` binary serves standalone AND
 # multi-process `cluster --region`, so one image covers both deployments).
 when:
  branch: main
-  event: push
+  event: [push, cron]
 steps:
  # ── Release gate (push) ─────────────────────────────────────────────────────
  # m11p8 release gate: prove a rolling upgrade under load loses no acknowledged
  # write and never stalls (mp_rolling_upgrade_no_loss_no_stall — a tier-3 test
  # spawning three real OS processes with a graceful SIGTERM → version-tagged
@ -19,6 +26,8 @@ steps:
  # overlap. A failure here BLOCKS the image build below — the gate, not the start.
  rolling-upgrade-gate:
    image: rust:1-bookworm
    when:
      event: push
    commands:
      - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang
      - cargo test -p tidal-server --features cluster-e2e --test cluster_lifecycle
@ -26,6 +35,8 @@ steps:
  build:
    image: woodpeckerci/plugin-kaniko
    when:
      event: push
    settings:
      repo: tidal/server
      dockerfile: docker/standalone/Dockerfile
@ -42,3 +53,110 @@ steps:
      build_args:
        - TARGETPLATFORM=linux/amd64
      extra_args: --customPlatform=linux/amd64
  # ── Nightly chaos (cron) ────────────────────────────────────────────────────
  # The tier-3 chaos/correctness suites over REAL OS processes, run serially
  # (fixed ports + spawned processes must not overlap). Kill-points are elevated
  # above the per-PR defaults (8/5) but kept below the 100-point exit-gate run so
  # the nightly stays bounded; widen TIDAL_*_KILLPOINTS for a deeper sweep. The
  # `fault-injection` feature compiles in the WAL slow-fsync / disk-full hooks for
  # cluster_faults (inert in every other suite — see tidal/src/fault.rs). Boot /
  # convergence budgets are raised for a shared CI runner. A failure = a
  # correctness regression for the night.
  nightly-chaos:
    image: rust:1-bookworm
    when:
      event: cron
      cron: nightly
    environment:
      TIDAL_QUORUM_KILLPOINTS: "25"
      TIDAL_ELECTION_KILLPOINTS: "15"
      TIDAL_TEST_BOOT_BUDGET_SECS: "120"
      TIDAL_TEST_CONVERGENCE_BUDGET_SECS: "90"
    commands:
      - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang
      # New m11p9 fault classes first (disk-full, slow-fsync, asymmetric partition).
      - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_faults -- --nocapture --test-threads 1
      # The standing chaos / durability / availability / elasticity gates.
      - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_chaos -- --nocapture --test-threads 1
      - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_quorum -- --nocapture --test-threads 1
      - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_election -- --nocapture --test-threads 1
      - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_membership -- --nocapture --test-threads 1
      - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_reseed -- --nocapture --test-threads 1
      - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_lifecycle -- --nocapture --test-threads 1
      - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_runbook -- --nocapture --test-threads 1
  # ── Nightly soak (cron) ─────────────────────────────────────────────────────
  # A tidal-stress soak with PASS/FAIL regression gates (m11p9). By default it
  # boots a local standalone server and soaks it for a bounded window, gating on
  # error rate and p99 so a perf regression fails the step and shows on the trend
  # line (the JSON summary is the archived artifact). Point TIDAL_SOAK_TARGET at
  # the live Ref-A cluster and raise TIDAL_SOAK_SECS to 3600 for the GA-bar 1-hour
  # 100k-DAU soak (the standing k3s-access caveat applies — same as m11p1–p3).
  nightly-soak:
    image: rust:1-bookworm
    when:
      event: cron
      cron: nightly
    environment:
      TIDAL_SOAK_RPS: "1000"
      TIDAL_SOAK_SECS: "600"
      TIDAL_SOAK_MAX_P99_MS: "250"
      TIDAL_SOAK_MAX_ERROR_PCT: "1"
      # Soak target port — an UNCLAIMED slot in the project's reserved dev band
      # (59520-59529): 59520=site, 59521=iknowyou, so the soak uses 59526. One
      # source of truth for the literal (referenced by --listen AND the TARGET).
      TIDAL_SOAK_PORT: "59526"
    commands:
      - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang curl
      - cargo build -p tidal-server -p tidal-stress
      # Boot a standalone target unless TIDAL_SOAK_TARGET points elsewhere (Ref-A).
      # NOTE the `$${VAR}` escaping: Woodpecker substitutes a bare `${VAR}` in a
      # command BEFORE the shell runs, and step `environment:` vars are NOT in that
      # preprocessor namespace (they would blank to ""). `$${VAR}` passes a literal
      # `${VAR}` to the container shell, which expands it from the runtime env —
      # so the soak's ramp and gate thresholds are actually populated. Bare shell
      # locals (`$PORT`, `$TARGET`, `$SRV`) are never touched by the preprocessor.
      - |
        PORT="$${TIDAL_SOAK_PORT}"
        TARGET="$${TIDAL_SOAK_TARGET:-http://127.0.0.1:$PORT}"
        if [ -z "$TIDAL_SOAK_TARGET" ]; then
          mkdir -p /tmp/soak-data
          ./target/debug/tidal-server standalone --listen "127.0.0.1:$PORT" \
            --schema tidal-server/config/default-schema.yaml --data-dir /tmp/soak-data &
          SRV=$!
          # Reap the background server + scratch dir on ANY exit (success, gate
          # failure, or the boot-failure exit below) so the step is idempotent.
          trap 'kill "$SRV" 2>/dev/null; rm -rf /tmp/soak-data' EXIT
          up=0
          for i in $(seq 1 100); do
            curl -sf "$TARGET/health/startup" >/dev/null 2>&1 && { up=1; break; } || sleep 0.3
          done
          # Fail FAST and unambiguously on a boot failure (bad schema, port bound)
          # rather than running the soak against a dead target and mislabeling it
          # as an error-rate regression on the trend line.
          if [ "$up" != "1" ]; then echo "soak target failed to start at $TARGET"; exit 1; fi
        fi
        ./target/debug/tidal-stress --target "$TARGET" \
          --ramp "$${TIDAL_SOAK_RPS}:$${TIDAL_SOAK_SECS}" --corpus 5000 --mix peach \
          --json-summary soak-summary.json \
          --max-error-pct "$${TIDAL_SOAK_MAX_ERROR_PCT}" --max-p99-ms "$${TIDAL_SOAK_MAX_P99_MS}" --fail-on-knee
      - cat soak-summary.json
  # ── Nightly security + ops correctness (cron) ───────────────────────────────
  # The G-Sec and G-Op owner-tests run nightly too (not just on-demand): mTLS /
  # foreign-pod rejection / cert rotation (tidal-net mtls + cluster_security), the
  # tidalctl backup/restore round-trip, and the gap-free WAL-archival (PITR) unit
  # test (in the engine lib). In-process + fast; serial for the TLS port binds.
  nightly-security-ops:
    image: rust:1-bookworm
    when:
      event: cron
      cron: nightly
    commands:
      - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang
      - cargo test -p tidal-net --test mtls -- --test-threads 1
      - cargo test -p tidal-server --test cluster_security -- --test-threads 1
      - cargo test -p tidal-server --lib
      - cargo test -p tidalctl
      - cargo test -p tidaldb --lib wal::compaction
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -6,6 +6,44 @@ All notable changes to tidalDB will be documented in this file.
 ### Added
 **Continuous correctness (m11p9) — new fault classes as REAL faults, first-class invariant checkers, soak regression gates, a Woodpecker nightly chaos+soak pipeline, and a guarantee→test matrix**
 - **Fault injection compiled out of production.** A non-default `fault-injection`
  cargo feature (tidaldb + a tidal-server passthrough) adds two WAL hooks
  (`tidal/src/fault.rs`): **slow-fsync** (`TIDAL_FAULT_FSYNC_DELAY_MS` sleeps
  before each durable fsync) and **disk-full** (`TIDAL_FAULT_DISK_FULL_AFTER_BYTES`
  returns a real `ENOSPC`/errno-28 once cumulative segment bytes cross the
  threshold). The production image build never passes the feature, so the hooks
  are compiled out entirely — a disk/fsync fault a stray env var could trip in
  prod is a 3am footgun we refuse to ship; the safety is structural. Inert until
  armed even when compiled in. The tier-3 harness builds the spawned binary with
  the feature.
 - **First-class invariant checkers** (`tidal-server/tests/support/invariants.rs`):
  the no-acked-loss `AckLedger` (frontier + content, extracted from the m11p3
  ledger gate, which now consumes it), cross-replica `assert_feed_parity` /
  `feed_item_ids`, `assert_single_leader_per_term` (membership safety, reads the
  m11p6 `shards[]` rows), and `MonotonicCounters` (per-node applied frontier +
  leader commit index never regress, with legitimate epoch resets forgiven).
 - **New fault-class suite** (`tidal-server/tests/cluster_faults.rs`, tier-3): a
  disk-full follower degrades gracefully (receiver halts, node alive, zero acked
  loss, restart recovers to parity); a slow-fsync follower lags then converges
  while the fast follower supplies quorum; both followers slow → `ack=quorum`
  returns a retryable 503 naming the laggards while `ack=leader` is unaffected;
  an asymmetric partition (inbound severed, outbound up) causes no split brain
  (pre-vote + check-quorum) and no loss. 4/4 green.
 - **Soak + regression gates** in `tidal-stress`: `--json-summary <path>` (a
  machine-readable per-stage p99/throughput/error roll-up for trend lines) and
  `--max-p99-ms` / `--max-error-pct` / `--fail-on-knee` gates that exit non-zero
  on a regression (the tool always exited 0 before).
 - **Nightly pipeline** (`.woodpecker.yaml`): a cron `nightly` flow (chaos suites
  with elevated kill-points + the `fault-injection` feature, then the gated
  `tidal-stress` soak) beside the existing push release gate, event-routed by
  per-step `when`. Woodpecker, never GitHub Actions.
 - **Guarantee traceability** (`docs/planning/milestone-11/guarantee-traceability.md`):
  every roadmap §2 guarantee mapped to its named automated test(s). Closes the
  **G-C** apparatus; the 30-consecutive-days-green half of the GA bar is a
  calendar criterion the nightly pipeline accrues.
 **Observability + operations (m11p8) — complete cluster metric set on a per-node `/metrics` listener, request-id/tracing across hops, truthful status, self-driving heal, WAL PITR + backup/restore, rolling-upgrade release gate**
 - **Metrics.** Completed the `tidaldb_cluster_*` set with the two members the
--- a/docs/planning/ROADMAP.md
+++ b/docs/planning/ROADMAP.md
@ -37,7 +37,7 @@ A single embeddable database can replace the 6-system content ranking stack by t
 | M8  | Distributed Fabric    | Multi-region, multi-tenant replication keeps agent-memory semantics intact   | Hosted tidalDB, cloud/edge deployments, shared agent substrate — **✅ COMPLETE**: in-process primitives + multi-node replication over real gRPC + true multi-process cluster mode (one process per region, real process isolation) with full tier-3 UAT (partition injection via TCP-proxy, clock-skew, rolling-upgrade, runbook verification); G1 + G2 resolved. Post-M8 follow-ups: quorum-ack writes, automatic failure detection / leader election |
 | M9  | Community Sync & Revocation | Local embeddable profiles can opt into community personalization and safely leave/purge contributions | Community personalization, federated taste graphs, shared feeds — ✅ COMPLETE (2026-06-06) |
 | M10 | Governance & Agent Rights | Community rules and agent-scoped permissions control what signals influence ranking | User-owned AI personalization at scale, policy-compliant agents — ✅ COMPLETE (2026-06-06) |
-| M11 | Enterprise-Grade Cluster | The multi-process cluster becomes a system of record: fast unified log, quorum-durable, self-healing, horizontally scalable, secured, observable, chaos-tested | Paying-customer cluster deployments — **IN PROGRESS**: m11p1 ✅ + m11p2 ✅ + m11p3 ✅ + m11p4 ✅ (automatic failover: Raft-style election + term fencing + divergence quarantine — closes **G5**; v0.9 "Credible HA" wave complete 2026-06-12) + m11p5 ✅ (membership/discovery/elasticity: DNS peers, snapshot+stream reseed, kind-4 membership records, seed join, `k8s/cluster/` — 2026-06-12) + m11p7 ✅ (security hardening: gRPC mTLS default + zero-drop cert rotation, inter-node HTTP TLS + per-node signed tokens, admin audit log, per-principal rate limit — 2026-06-13) + m11p8 ✅ (observability + operations: completed `tidaldb_cluster_*` set incl. breaker/forwards/self-heal on the per-node `/metrics` listener + Grafana cluster row + alert group, request-id/tracing across hops, truthful status, self-driving heal, WAL PITR archival + `tidalctl` backup/restore, rolling-upgrade version handshake + Woodpecker release gate — closes **G-O** — 2026-06-13); m11p6 (sharding × replication) data plane in progress; p9 planned in [docs/roadmap-to-cluster.md](../roadmap-to-cluster.md) |
+| M11 | Enterprise-Grade Cluster | The multi-process cluster becomes a system of record: fast unified log, quorum-durable, self-healing, horizontally scalable, secured, observable, chaos-tested | Paying-customer cluster deployments — **IN PROGRESS**: m11p1 ✅ + m11p2 ✅ + m11p3 ✅ + m11p4 ✅ (automatic failover: Raft-style election + term fencing + divergence quarantine — closes **G5**; v0.9 "Credible HA" wave complete 2026-06-12) + m11p5 ✅ (membership/discovery/elasticity: DNS peers, snapshot+stream reseed, kind-4 membership records, seed join, `k8s/cluster/` — 2026-06-12) + m11p7 ✅ (security hardening: gRPC mTLS default + zero-drop cert rotation, inter-node HTTP TLS + per-node signed tokens, admin audit log, per-principal rate limit — 2026-06-13) + m11p8 ✅ (observability + operations: completed `tidaldb_cluster_*` set incl. breaker/forwards/self-heal on the per-node `/metrics` listener + Grafana cluster row + alert group, request-id/tracing across hops, truthful status, self-driving heal, WAL PITR archival + `tidalctl` backup/restore, rolling-upgrade version handshake + Woodpecker release gate — closes **G-O** — 2026-06-13) + m11p9 ✅ (continuous correctness: new fault classes (disk-full / slow-fsync / asymmetric partition) as REAL faults behind a production-compiled-out `fault-injection` feature, first-class invariant checkers, `tidal-stress` soak with regression gates + JSON summary, a Woodpecker cron nightly chaos+soak pipeline, and a guarantee→test traceability matrix — closes the **G-C** apparatus; the 30-day-green calendar accrues nightly — 2026-06-13); m11p6 (sharding × replication) data plane in progress in [docs/roadmap-to-cluster.md](../roadmap-to-cluster.md) |
 ### Embeddable → Distributed Path
@ -3285,7 +3285,7 @@ Full gap analysis, measured baselines, phase specs, and exit gates live in
 | m11p6 | Sharding × replication + rebalancing | Planned |
 | m11p7 | Security hardening (mTLS default, rotation, audit log) | ✅ **COMPLETE (2026-06-13)** — the cluster stops trusting the network. gRPC replication is served over a custom `tokio-rustls` acceptor fed a hot-swappable `DynamicCertResolver` (`ArcSwap<CertifiedKey>`) — mTLS preserved exactly (`WebPkiClientVerifier`; a foreign/absent client cert fails the handshake before any RPC), plaintext now an explicit `insecure: true` + loud WARN. **Cert + bearer rotation without restart**: a content-hash poller (catches k8s `..data` symlink swaps inotify misses) atomically swaps the cert with in-flight sessions untouched — **zero dropped requests under load** (verified). Inter-node HTTP gains TLS (same resolver — one rotation covers both planes; `https` forwards + cluster-CA reqwest clients) + **per-node identity** via a signed `x-tidal-node-token` (keyed-BLAKE3 MAC under a shared cluster key — no new crypto dep); the `x-tidal-internal` marker is honored ONLY from a verified sibling (marker-without-token → 403), never an auth bypass. Admin verbs (promote/partition/heal/join/remove/reseed) emit a structured audit record (principal, term, target, outcome) to a `tidal_audit` target + optional `TIDAL_AUDIT_LOG` JSONL, operator-leg only. Per-principal HTTP rate limit (engine `RateLimiter`, nodes exempt, 429 + Retry-After). All TLS/identity/audit/limit opt-in (`grpc_tls` / cluster key / env) — absent ⇒ pre-m11p7 behavior byte-for-byte. `k8s/cluster/` gains cert-manager + `grpc_tls` topology + cluster-key Secret; `scripts/gen-cluster-certs.sh` for non-cert-manager. Exit gate verified: foreign pod rejected (gRPC `mtls.rs` + HTTP `cluster_security.rs`), zero-drop rotation under load, zero plaintext inter-node links. See [milestone-11/phase-7.md](milestone-11/phase-7.md). |
 | m11p8 | Observability + operations (complete metric set, self-driving heal, backup/PITR, rolling-upgrade gate) | ✅ **COMPLETE (2026-06-13)** — operable by someone who didn't build it (closes **G-O** + the operability half of **G-Op**, and incident §1.4-3). **Metrics**: completed the `tidaldb_cluster_*` set with breaker (`tidaldb_cluster_peer_breaker_state` + `_breaker_opens_total`, surfaced read-only from the `tidal-net` breaker) and forwards (`_forwards_total` / `_forward_failures_total`) + the self-heal series; multi-shard nodes serve ONE `/metrics` listener with the co-located siblings' series `shard="N"`-labeled (S=1 byte-identical). A 12-panel Grafana cluster row + 8-rule `tidaldb-cluster` alert group ship beside the standalone ones. **Tracing**: both cluster routers gained the request-id + `TraceLayer` stack; the id rides the forward hop. **Truthful status**: the leader's own `applied_events` (was 0 — now its flushed frontier) and the post-promote `ShardId(0)` lag-keying undercount fixed; `/cluster/status` gained `version`. **Self-driving heal**: a standing leader duty re-arms a stuck peer's backlog re-ship every ~3s so it converges through breaker resets with NO operator heal loop (`healing_peers` gauge + alert). **Backup/PITR**: `wal.archive_dir` archives sealed segments before compaction deletes them (gap-free); `tidalctl backup`/`restore` (BLAKE3-verified round-trip). **Rolling upgrade**: wire `build_version` handshake (N/N+1 by proto3 compat; `>= 2`-major WARNs, never rejects) + `mp_rolling_upgrade_no_loss_no_stall` promoted to the Woodpecker release gate. Exit gates green (dashboard answers the golden signals; rolling-upgrade gate passed; backup/restore + archival round-trips verified). See [milestone-11/phase-8.md](milestone-11/phase-8.md). |
-| m11p9 | Continuous correctness (nightly chaos CI, invariant checkers, soak) | Planned |
+| m11p9 | Continuous correctness (nightly chaos CI, invariant checkers, soak) | ✅ **COMPLETE (2026-06-13)** — closes the **G-C** apparatus (the 30-consecutive-days-green half of the GA bar accrues nightly). New fault classes as REAL faults behind a production-compiled-out `fault-injection` feature: **slow-fsync** (`TIDAL_FAULT_FSYNC_DELAY_MS`) + **disk-full** (`TIDAL_FAULT_DISK_FULL_AFTER_BYTES` → real `ENOSPC`) WAL hooks (`tidal/src/fault.rs`); **asymmetric partition** via the harness's directed-edge proxies. First-class invariant checkers (`support/invariants.rs`): no-acked-loss `AckLedger` (the m11p3 ledger gate now consumes it), decay/feed parity, `assert_single_leader_per_term`, `MonotonicCounters`. New tier-3 `cluster_faults.rs` (disk-full / slow-fsync ×2 / asymmetric) **4/4 green** with zero acked loss. `tidal-stress` gained `--json-summary` + `--max-p99-ms`/`--max-error-pct`/`--fail-on-knee` regression gates (non-zero exit on breach — verified vs a real server). Nightly Woodpecker CRON pipeline (chaos suites with elevated kill-points + the gated soak) beside the push release gate. Every §2 guarantee → named test in [milestone-11/guarantee-traceability.md](milestone-11/guarantee-traceability.md). See [milestone-11/phase-9.md](milestone-11/phase-9.md). |
 ---
--- a/docs/planning/milestone-11/guarantee-traceability.md
+++ b/docs/planning/milestone-11/guarantee-traceability.md
@ -0,0 +1,66 @@
 # Enterprise-Guarantee → Automated-Test Traceability (m11p9)
 This is the m11p9 exit-gate artifact: **every guarantee in
 [roadmap-to-cluster.md §2](../../roadmap-to-cluster.md) maps to a named automated
 test** (the second half of the GA bar is the calendar one — the nightly suite
 green for 30 consecutive days; see [phase-9.md](phase-9.md)).
 Each guarantee lists the named test(s) that prove it, the fault class each one
 injects (where applicable), and the first-class invariant checker
 ([`tidal-server/tests/support/invariants.rs`](../../../tidal-server/tests/support/invariants.rs))
 it asserts through. All `mp_*` tests are tier-3 over REAL OS processes
 (`MultiProcCluster`); partitions sever real loopback TCP; the disk-full /
 slow-fsync faults return a real `ENOSPC` / sleep the real fsync behind the
 `fault-injection` feature (`tidal/src/fault.rs`).
 Run any suite locally:
 `cargo test -p tidal-server --features "cluster-e2e fault-injection" --test <suite> -- --test-threads 1 --nocapture`.
 Every named test that EXISTS below runs nightly via the cron pipeline in
 [`.woodpecker.yaml`](../../../.woodpecker.yaml): the chaos/correctness suites and
 the gated soak in `nightly-chaos`/`nightly-soak`, and the G-Sec/G-Op owner-tests
 (mTLS, `cluster_security`, `tidalctl` backup/restore, WAL-archival) in
 `nightly-security-ops`. The one exception is the **G-S** throughput gate, which
 is not yet written (m11p6 L4) — its row is marked accordingly.
 | Guarantee | Named automated test(s) | Suite | Fault / scenario | Invariant checker |
 |---|---|---|---|---|
 | **G-D Durability** — `ack=quorum` 2xx ⇒ majority-durable; SIGKILL any node ⇒ zero acked loss | `mp_quorum_ledger_zero_acked_loss_across_killpoints` | `cluster_quorum` | SIGKILL leader across N random kill-points under quorum load | `AckLedger` (frontier + content) |
 | | `mp_auto_failover_writes_resume_zero_acked_loss` | `cluster_election` | SIGKILL leader, auto-elect, across kill-points | `AckLedger` |
 | | `mp_disk_full_follower_degrades_no_acked_loss` | `cluster_faults` | **disk-full** (ENOSPC) on a follower | `AckLedger` + `MonotonicCounters` |
 | | `mp_slow_fsync_follower_lags_but_quorum_holds`, `mp_slow_fsync_both_followers_force_honest_quorum_timeout` | `cluster_faults` | **slow-fsync** on one / both followers | `AckLedger` |
 | | `mp_follower_reseeds_via_snapshot_after_compaction`, `mp_quarantined_node_reseeds_without_wipe` | `cluster_reseed` | WAL compaction past cursor / divergent-suffix quarantine | content probes |
 | **G-A Availability** — single-node loss ⇒ reads continue, writes resume <10 s, no dual-leader | `mp_auto_failover_writes_resume_zero_acked_loss` | `cluster_election` | leader SIGKILL → election (<10 s) | `AckLedger` |
 | | `mp_fenced_ex_leader_restart_cannot_write` | `cluster_election` | partition leader away, restart while fenced | fencing assertions |
 | | `mp_flapping_links_bounded_churn` | `cluster_election` | repeated link flaps | single-leader-per-term |
 | | `mp_asymmetric_partition_no_split_brain_no_loss` | `cluster_faults` | **asymmetric partition** (inbound severed, outbound up) | `assert_single_leader_per_term` + `AckLedger` |
 | | `mp_self_heal_converges_without_operator_verb` | `cluster_chaos` | gRPC link sever, no operator verb | feed parity |
 | **G-S Scalability** — write throughput scales with shard count at RF=3 (≥2.5× 1→3) | **(NOT YET WRITTEN — m11p6 L4)** the 3 shards × RF=3 ≥5,000 quorum signals/s gate + `tidal-stress --write-path` path-collapse comparison | _pending m11p6 L4_ | sharded × replicated scaling | throughput gate |
 | **G-E Elasticity** — online add/remove/replace; snapshot+stream catch-up; p99 <2× for <60 s | `mp_scale_3_5_3_under_load_zero_loss`, `mp_seed_join_snapshot_catchup`, `mp_dns_hostname_topology_replicates` | `cluster_membership` | scale 3→5→3 under load; seed-join after compaction | `AckLedger` (lost=0), p99 bound |
 | **G-Sec Security** — all inter-node links mTLS; authenticated RPC; rotation no downtime; admin audit | `http_tls_serves_ca_trusting_client_and_rejects_foreign`, `http_tls_cert_rotation_under_load_drops_zero` | `cluster_security` | foreign-CA client rejected; cert hot-rotation under load | zero-drop assertion |
 | | `mtls` (foreign-pod ship rejected), `cluster::security` unit tests (token mint/verify, marker-not-bypass) | `tidal-net/tests/mtls.rs`, engine unit | foreign pod cannot ship or call internal routes | negative tests |
 | **G-O Observability** — per-node Prometheus metrics; dashboard answers golden signals; alerts on lag / commit stall / election churn | per-node `/metrics` listener + `tidaldb_cluster_*` set (m11p8); `grafana-dashboard.json` "Cluster Replication" row (12 panels); `prometheus-alerts.yaml` `tidaldb-cluster` group (8 rules) | `cluster_runbook` (`runbook_s5_health_and_openapi`) + artifacts | metric/alert presence | artifact + status tests |
 | **G-Op Operability** — rolling upgrade N/N+1 under load (CI gate); backup/restore + PITR drilled; runbooks executable | `mp_rolling_upgrade_no_loss_no_stall` (the **Woodpecker release gate**) | `cluster_lifecycle` | graceful SIGTERM → version-tagged restart → heal → fixpoint under load | `AckLedger` |
 | | `backup_then_restore_roundtrips`, `restore_rejects_corrupted_backup`, `online_compaction_archives_before_deleting` | `tidalctl` / engine | offline backup/restore integrity; gap-free WAL archival (PITR) | BLAKE3 manifest |
 | | `runbook_s5..s11` (full operator playbook) | `cluster_runbook` | failover / partition / shutdown+recovery drills | per-drill |
 | **G-C Continuous correctness** — nightly chaos + soak green 30 consecutive days | the entire `cluster_faults` suite + the `nightly-chaos`/`nightly-soak` cron pipeline + `tidal-stress` soak regression gates + THIS matrix | `cluster_faults` / `.woodpecker.yaml` / `tidal-stress` | all fault classes nightly; soak p99/error regression gates | all four checkers |
 ## Honest status of the bar
 - **G-D, G-A, G-E, G-Sec, G-Op** — each maps to a green named test that runs in
  the nightly pipeline. ✅
 - **G-S (Scalability)** — the sharding × replication **data plane** is in place
  (m11p6 L0–L2); the ≥5,000/s 3-shard×RF=3 throughput gate and the `tidal-stress`
  path-collapse comparison land with m11p6 L4. The named owner-test is recorded
  here so the guarantee is not orphaned. ⏳ (tracked in m11p6.)
 - **G-O (Observability)** — proven by artifact + presence tests (the metric set,
  the dashboard, the alert group) rather than one behavioral test; the
  golden-signal coverage is the m11p8 exit-gate evidence.
 - **G-C (Continuous correctness)** — the *apparatus* is complete and green
  (the fault suite passes; the cron pipeline runs every named suite + the gated
  soak). The remaining half of the bar is **calendar**: 30 consecutive nightly-green
  days before GA. That cannot be completed in one session — the pipeline is what
  produces it. ⏳ (accrues nightly.)
 - **Ref-A line items** — the throughput / 100k-item / 1-hour-soak figures that
  must run on the live 3-region k3s cluster remain blocked on k3s access (the
  standing M11 caveat from m11p1–p3); the in-runner soak gates regressions
  locally and the same `tidal-stress` invocation targets Ref-A when reachable.
--- a/docs/planning/milestone-11/phase-9.md
+++ b/docs/planning/milestone-11/phase-9.md
@ -0,0 +1,223 @@
 # m11p9 — Continuous Correctness (COMPLETE — 2026-06-13)
 Phase spec and exit gate: [docs/roadmap-to-cluster.md §4/m11p9](../../roadmap-to-cluster.md).
 Closes the roadmap's **G-C (Continuous correctness)** — trust is a pipeline, not
 a milestone — and wires the named owner-test for every other §2 guarantee
 ([guarantee-traceability.md](guarantee-traceability.md)).
 Predecessors: every prior m11 phase (this phase industrializes their tier-3
 suites and adds the fault classes they lacked).
 **Goal:** make correctness continuous — new fault classes (disk-full, slow-fsync,
 asymmetric partition) as REAL faults; the invariant checks the durability and
 election gates carried inline promoted to first-class reusable checkers; a
 nightly chaos + soak pipeline that fails on a regression; and a guarantee→test
 matrix so the GA bar is auditable.
 ## Design (as adopted)
 ### 1. Fault injection compiled out of production (`tidal/src/fault.rs`)
 Two new fault classes need REAL faults at the storage boundary, not engine flags:
 - **`TIDAL_FAULT_FSYNC_DELAY_MS`** — sleep before every durable WAL fsync
  (`wal::sync_file_durable`). Models a slow disk: data still reaches stable
  storage, each group-commit fsync costs `delay + real`.
 - **`TIDAL_FAULT_DISK_FULL_AFTER_BYTES`** — after N cumulative segment bytes this
  process lifetime, every segment write returns a real `ENOSPC`
  (`std::io::Error::from_raw_os_error(28)` — the errno on Linux AND macOS,
  wrapped in the same `WalError::Io` a genuine full disk produces) at
  `segment::write_batch_bytes`.
 Both are behind the **non-default `fault-injection` cargo feature** (tidaldb +
 a tidal-server passthrough). Production builds (`docker/standalone/Dockerfile`:
 `cargo build -p tidal-server --release --locked`) never pass it, so the hooks
 are **compiled out entirely** — a disk-write or fsync fault a stray env var could
 trip in production is exactly the 3am footgun we refuse to ship, so the safety is
 structural, not a default. Even compiled in, faults are inert until an env var
 arms them (the posture of `TIDAL_HLC_SKEW_MS` / the kill-point knobs). The
 `MultiProcCluster` harness's `tidal_server_bin()` builds the spawned binary with
 the feature; every non-fault suite spawns the same binary and is unaffected.
 The injection returns `ENOSPC` *before* the partial `write_all` (the standard
 clean-fault model). The follower apply path already handles a WAL write error
 correctly (proven by the suite below): the receiver halts (degraded, alive),
 NEVER advances its applied frontier past the fault (no gap-swallowing), and a
 restart re-pulls the suffix via the catch-up stream.
 ### 2. First-class invariant checkers (`tidal-server/tests/support/invariants.rs`)
 The roadmap's named invariants were inline copies in three suites; m11p9 promotes
 them to one audited module every suite consumes:
 - **`AckLedger`** — no acknowledged-write loss: records every write the client
  saw a 2xx + `x-tidal-seq` for, then proves (A) no acked seqno exceeds the
  max-applied survivor's contiguous frontier, and (B) every acked item is present
  on the promoted/recovered leader via `/search`. Extracted verbatim from the
  m11p3 ledger gate, which now CONSUMES it (re-verified: 3/3 kill-points zero
  acked loss after the refactor).
 - **`assert_feed_parity` / `feed_pairs`** — cross-replica decay/score parity to
  `1e-6` between continuously-running replicas.
 - **`feed_item_ids`** — cross-replica DATA parity (the materialized item set),
  robust across a node RESTART where a velocity/rate score legitimately differs
  (its time-bucketed windowed counts are rebuilt from a burst WAL-replay rather
  than continuous accumulation, even with identical durable data — see the
  disk-full test).
 - **`assert_single_leader_per_term`** — membership safety: at most one leader per
  `(shard, term)` across a set of status snapshots (reads the m11p6 `shards[]`
  rows or the flat fields). A second leader at the same term is a split brain.
 - **`MonotonicCounters`** — per-entity monotonic counters: a node's applied
  frontier and the leader's commit index never move backward across observations
  (legitimate epoch resets — a reseeding node, a leader change — are forgotten,
  not flagged), so a regression means durably-acked state was lost.
 ### 3. New fault-class suite (`tidal-server/tests/cluster_faults.rs`)
 Four tier-3 tests over real OS processes, asserting through the checkers:
 - `mp_disk_full_follower_degrades_no_acked_loss` — a follower hits ENOSPC
  mid-replication, its receiver halts (degraded, alive), the healthy majority
  keeps acking quorum writes, NO acked write is lost, and a restart with space
  recovers it to full content + item-set parity.
 - `mp_slow_fsync_follower_lags_but_quorum_holds` — a slow disk lags one follower;
  the fast follower supplies quorum so every write commits; the slow node
  converges once the burst ends; no loss.
 - `mp_slow_fsync_both_followers_force_honest_quorum_timeout` — neither follower
  can confirm inside the budget ⇒ `ack=quorum` returns a retryable 503 naming the
  laggards (never a false 2xx) while `ack=leader` is unaffected; the followers
  recover.
 - `mp_asymmetric_partition_no_split_brain_no_loss` — a follower that loses INBOUND
  links (can still send) cannot disrupt the cluster: pre-vote + check-quorum keep
  the standing leader, there is never a second leader at the same term, no acked
  write is lost, and it rejoins cleanly on heal.
 The asymmetric partition reuses the harness's directed-edge proxies
 (`proxied_rewrite(&["ap-south"])` + `region("ap-south").sever_all()` severs only
 inbound to ap-south; its outbound dials are unproxied, so it keeps sending the
 disruptive RequestVotes pre-vote neutralizes) — no new mechanism, the existing
 `PartitionProxy` was already directional.
 ### 4. Soak + regression gates (`tidal-stress`)
 `tidal-stress` gained, for the nightly soak:
 - **`--json-summary <path>`** — a flat, machine-readable per-stage roll-up
  (p99 / throughput / error-rate / client-shed + the gate verdict), hand-rolled
  (no serializer dep, matching the metrics module's posture) for trend-line
  archival.
 - **`--max-p99-ms`, `--max-error-pct`, `--fail-on-knee`** — PASS/FAIL regression
  gates. A breach returns `StressError::Gate` → non-zero exit (the tool always
  exited 0 before, so a regression scrolled past in green). Unarmed = the
  original informational behavior, byte-for-byte.
 ### 5. Nightly pipeline (`.woodpecker.yaml`) — Woodpecker, never GitHub Actions
 The pipeline now carries two event-routed flows (workflow-level `when:
 event: [push, cron]`, per-step `when`):
 - **push** — the m11p8 rolling-upgrade release gate, then the Kaniko image build
  (unchanged).
 - **cron `nightly`** — `nightly-chaos` runs every tier-3 suite serially with
  elevated kill-points (`TIDAL_QUORUM_KILLPOINTS=25`, `TIDAL_ELECTION_KILLPOINTS=15`)
  and the `fault-injection` feature (so `cluster_faults` runs); `nightly-soak`
  boots a target and runs `tidal-stress` with the regression gates, archiving the
  JSON summary. Point `TIDAL_SOAK_TARGET` at the live Ref-A cluster and raise
  `TIDAL_SOAK_SECS=3600` for the GA-bar 1-hour 100k-DAU soak.
 ### 6. Guarantee traceability
 [guarantee-traceability.md](guarantee-traceability.md) maps every §2 guarantee
 (G-D, G-A, G-S, G-E, G-Sec, G-O, G-Op, G-C) to its named automated test(s), the
 fault each injects, and the checker each asserts through.
 ## Exit gate (from the roadmap)
 - Every guarantee in §2 maps to a named automated test.
 - Nightly suite green 30 consecutive days before GA.
 ## Exit-gate evidence (local; debug builds, real OS processes)
 | Gate | Target | Measured |
 |------|--------|----------|
 | New fault classes pass | disk-full / slow-fsync / asymmetric | **`cluster_faults` 4/4 green** (73s): disk-full follower froze at applied 71 while node1 reached 240, recovered after restart to 120-item parity, zero acked loss; slow follower lagged 15 vs 0 then converged; both-slow → honest 503 naming `["eu-west","ap-south"]` while `ack=leader` 204'd; asymmetric → leader held us-east, single-leader-per-term, ap-south term bounded (0→0), rejoined. |
 | First-class checkers faithful | extraction = no regression | `cluster_quorum::mp_quorum_ledger_zero_acked_loss_across_killpoints` refactored onto `AckLedger`: **3/3 kill-points zero acked loss** (29s) — extraction verified against the live gate. |
 | Soak regression gate works | PASS exits 0, breach exits ≠0 | Against a real standalone server: a within-threshold run **exited 0** with a valid JSON summary (p99 39.7ms, 0% error); an impossible `--max-p99-ms 0.01` run **exited 1** with the breach in stderr and `"passed": false` in the JSON. |
 | Every guarantee → named test | §2 coverage | [guarantee-traceability.md](guarantee-traceability.md): G-D/G-A/G-E/G-Sec/G-Op each map to a green named test; G-S to the m11p6 owner-test (in progress); G-O to the m11p8 metric/dashboard/alert artifacts; G-C to this apparatus. |
 | Nightly green 30 days | calendar | **Pipeline established, not yet accrued** — this is a wall-clock criterion the cron pipeline produces over 30 nights; it cannot be completed in one session. Honestly out-of-session (like the Ref-A k3s runs). |
 ## Status
 - [x] `fault-injection` feature (slow-fsync + disk-full WAL hooks), compiled out of production
 - [x] First-class invariant checkers (`support/invariants.rs`): ledger, decay parity, single-leader-per-term, monotonic counters
 - [x] `cluster_quorum` refactored onto `AckLedger` (extraction re-verified)
 - [x] `cluster_faults.rs`: disk-full, slow-fsync (×2), asymmetric partition — 4/4 green
 - [x] `tidal-stress` `--json-summary` + `--max-p99-ms` / `--max-error-pct` / `--fail-on-knee` + non-zero exit (verified vs a real server)
 - [x] Woodpecker nightly cron pipeline (chaos suites + gated soak) beside the push release gate
 - [x] Guarantee → automated-test traceability matrix
 - [x] Docs (this phase, runbook chaos/soak section, monitoring soak note) + CHANGELOG + ROADMAP + roadmap-to-cluster as-built
 - [ ] **Calendar:** nightly green for 30 consecutive days before GA (the pipeline accrues this; not an in-session deliverable)
 ## Adversarial review (6-dimension, read-only)
 A read-only review fanned out over six dimensions (production-safety, checkers,
 fault-tests, soak-gates, Woodpecker, doc-honesty), each finding adversarially
 verified before adjudication. **10 confirmed findings, all fixed:**
 - **Woodpecker (blocker):** the soak step referenced `environment:`-block vars as
  `${VAR}`, which Woodpecker's preprocessor blanks BEFORE the shell — the gate
  would have run with an empty ramp/thresholds (never executing). Fixed with the
  `$${VAR}` escape so the container shell expands them at runtime.
 - **Gate threshold (blocker):** a `--max-p99-ms nan`/`inf` reported the gate ARMED
  while every `> NaN` comparison is false — silently passing a regression. Now
  rejected (finite, non-negative) at the CLI boundary.
 - **JSON validity (blocker):** a non-finite `target_rps` (`--ramp "inf:30"`) emitted
  bare `inf`/`NaN`, invalid JSON. Fixed at the parse boundary + a `to_json`
  null-sanitizer (defense in depth).
 - **Checkers (warnings):** `assert_single_leader_per_term` false-failed on a
  duplicate same-region snapshot (`assert_ne!` → gated `if prev != region`);
  `MonotonicCounters` now re-baselines a region's role-relative applied frontier
  on a leader↔follower flip (not just reseed); `assert_items_present` distinguishes
  a transport/non-2xx probe failure from true absence (no false ACKNOWLEDGED LOSS).
 - **Coverage (warning):** added a `summary.rs` test module (gate breach/pass,
  fail-on-knee, JSON round-trip through a real parser, null-sanitize) — 6 tests.
 - **Doc honesty (warnings):** the G-Sec/G-Op owner-tests (mTLS, `cluster_security`,
  `tidalctl` backup/restore, WAL-archival) were NOT in the nightly — ADDED a
  `nightly-security-ops` cron step so the "runs nightly" claim is true; the G-S
  row is marked "(NOT YET WRITTEN — m11p6 L4)" so it doesn't read as delivered.
 - **Cron name (nit, external):** the nightly is silently-green until a cron named
  `nightly` is created in Woodpecker repo settings — documented in the pipeline
  header; a one-time repo-settings step.
 After the fixes the fault suite re-ran **4/4 green** and the ledger gate **3/3**
 (the checker changes are faithful); the slow-fsync and asymmetric tests moved to
 item-SET parity (a burst catch-up reconstructs velocity differently — the same
 restart-robustness rationale as disk-full).
 ### Formal seven-dimension review (`/review-code` → `fix-all`)
 A second pass ran the `code-reviewer` seven-dimension protocol (each dimension a
 distinct frame, each finding adversarially verified): **0 blockers, 0 critical, 7
 warnings, 12 suggestions** (APPROVE-with-fixes). All 7 warnings + 10 of 12
 suggestions fixed; final score 90/100. Notable fixes: a real diagnostic bug
 (`assert_items_present`'s two guards were swapped — a true acknowledged-loss
 printed "PROBE UNREACHABLE" and vice-versa); `StressError::Gate` was overloaded
 across breach / invalid-threshold / IO-write and split into `Gate`/`BadGate`/
 `Summary` so the operator-facing message names the real cause; the soak port
 moved off iknowyou's 59521 to a `TIDAL_SOAK_PORT` (59526); the soak step now
 fails fast on a boot failure and reaps its background server; doc-truthing the
 "every suite consumes the shared checkers" claim (only `cluster_quorum` +
 `cluster_faults` do — the legacy suites' migration is a tracked follow-up);
 S=1-scope notes on the flat-field checkers; a real `from_stage` unit test; and
 the `observe_cluster_frontiers` double-status-fetch removed. Two suggestions
 deferred with justification: consolidating the `role`-vs-`is_leader` leadership
 read into one helper (steady-state-equal, >5min, regression risk on green gates)
 and Woodpecker YAML anchors for the repeated apt-get/cargo lines (untestable CI
 refactor, benign duplication).
 ## Verification status
 - Workspace `cargo fmt --all -- --check`: clean.
 - `cargo clippy` `-D warnings`: clean on tidaldb (`fault-injection` on + off), tidal-server
  (`cluster-e2e fault-injection`), and tidal-stress (all targets).
 - Production safety: `cargo check -p tidaldb` with the feature OFF compiles the
  fault module and both WAL hooks out entirely (the shipped binary is unchanged).
 - The tree is UNCOMMITTED (continues the m11p1–p8 uncommitted tree; the user commits).
--- a/docs/roadmap-to-cluster.md
+++ b/docs/roadmap-to-cluster.md
@ -1,6 +1,6 @@
 # Roadmap to an Enterprise-Grade Cluster
-**Status:** ADOPTED as M11 — m11p1 ✅, m11p2 ✅, m11p3 ✅, m11p4 ✅, m11p5 ✅, m11p7 ✅, m11p8 ✅ complete (m11p8 2026-06-13 — observability + operations: completed `tidaldb_cluster_*` set incl. breaker/forwards/self-heal on the per-node `/metrics` listener + Grafana cluster row + alert group, request-id/tracing across hops, truthful status, self-driving heal, WAL PITR archival + `tidalctl` backup/restore, rolling-upgrade version handshake + Woodpecker release gate — closes **G-O**) · m11p6 (sharding × replication) data plane in progress · p9 planned · **Date:** 2026-06-10 · **Baseline evidence:**
+**Status:** ADOPTED as M11 — m11p1 ✅, m11p2 ✅, m11p3 ✅, m11p4 ✅, m11p5 ✅, m11p7 ✅, m11p8 ✅, m11p9 ✅ complete (m11p9 2026-06-13 — continuous correctness: new fault classes (disk-full / slow-fsync / asymmetric partition) as REAL faults behind a production-compiled-out `fault-injection` feature, first-class invariant checkers (no-acked-loss ledger, decay parity, single-leader-per-term, monotonic counters), a `tidal-stress` soak with PASS/FAIL regression gates + JSON summary, a Woodpecker cron nightly chaos+soak pipeline beside the push release gate, and a guarantee→test traceability matrix — closes **G-C** apparatus; the 30-day-green calendar accrues nightly) · m11p6 (sharding × replication) data plane in progress · **Date:** 2026-06-10 · **Baseline evidence:**
 [stress-test-thepeach.md](ops/stress-test-thepeach.md), [cluster runbook](runbooks/cluster.md),
 [ROADMAP M8 Known Gaps](planning/ROADMAP.md) (G4/G5/G6), live k3s deployment (3 regions × 2-vCPU pods).
@ -485,6 +485,32 @@ current "replicated XOR sharded" split ends.
 - **Exit gate (GA):** every guarantee in §2 maps to a named automated test;
  nightly suite green 30 consecutive days.
 > **✅ COMPLETE (2026-06-13), as built:** the new fault classes are REAL faults
 > at the storage boundary, not engine flags — a `fault-injection` cargo feature
 > (NOT default, never passed by the production image build, so compiled OUT of the
 > shipped binary) adds WAL hooks for **slow-fsync** (`TIDAL_FAULT_FSYNC_DELAY_MS`,
 > sleep before the durable fsync) and **disk-full** (`TIDAL_FAULT_DISK_FULL_AFTER_BYTES`,
 > a real `ENOSPC` once cumulative segment bytes cross the threshold);
 > **asymmetric partition** reuses the harness's directed-edge proxies (sever
 > inbound-only). The invariant checks the m11p3/m11p4 gates carried inline became
 > a first-class `support/invariants.rs` (no-acked-loss `AckLedger`, decay/feed
 > parity, `assert_single_leader_per_term`, monotonic frontiers) — the m11p3 ledger
 > gate now consumes it (re-verified). A new tier-3 `cluster_faults.rs` proves all
 > three fault classes degrade gracefully with zero acked loss (**4/4 green**:
 > disk-full follower froze then recovered to parity; slow follower lagged then
 > converged; both-slow → honest retryable 503 naming laggards while `ack=leader`
 > kept 204'ing; asymmetric partition → leader held, single-leader-per-term, no
 > split brain). `tidal-stress` gained a `--json-summary` + `--max-p99-ms` /
 > `--max-error-pct` / `--fail-on-knee` regression gate (non-zero exit on breach —
 > verified vs a real server). The nightly is a Woodpecker CRON pipeline (chaos
 > suites with elevated kill-points + the gated soak) beside the push release gate
 > — Woodpecker, never GitHub Actions. Every §2 guarantee maps to a named test in
 > [guarantee-traceability.md](planning/milestone-11/guarantee-traceability.md);
 > G-S's owner-test is the (in-progress) m11p6 throughput gate. The remaining GA
 > half — nightly green for **30 consecutive days** — is a calendar criterion the
 > pipeline now produces (out-of-session, like the Ref-A k3s runs). Details:
 > [planning/milestone-11/phase-9.md](planning/milestone-11/phase-9.md).
 ---
 ## 5. Sequencing, sizing, releases
--- a/docs/runbooks/cluster.md
+++ b/docs/runbooks/cluster.md
@ -1303,6 +1303,53 @@ macOS, where `F_FULLFSYNC` averages ~7.4ms with a 10–50ms tail
 the p99 gate on the reference environment, and tune `wal.batch_timeout_ms`
 against the measured fsync histogram.
 ## 15. Continuous correctness: chaos suites + soak (m11p9)
 Correctness is a pipeline, not a one-time gate. The tier-3 suites run REAL OS
 processes; the new fault classes inject REAL faults.
 **Run the chaos suites locally** (serial — they bind fixed ports and spawn
 processes, so suites must not overlap):
 ```
 cargo test -p tidal-server --features "cluster-e2e fault-injection" \
  --test cluster_faults -- --test-threads 1 --nocapture
 ```
 `cluster_faults` covers the fault classes the partition/crash/skew suites lacked:
 | Test | Fault | Asserts |
 |------|-------|---------|
 | `mp_disk_full_follower_degrades_no_acked_loss` | follower WAL `ENOSPC` | receiver halts (degraded, alive), healthy majority keeps acking quorum, zero acked loss, restart recovers to parity |
 | `mp_slow_fsync_follower_lags_but_quorum_holds` | one follower's fsync slowed | fast follower supplies quorum, slow node lags then converges, no loss |
 | `mp_slow_fsync_both_followers_force_honest_quorum_timeout` | both followers slowed below the budget | `ack=quorum` → retryable 503 naming laggards; `ack=leader` → 204; recover |
 | `mp_asymmetric_partition_no_split_brain_no_loss` | inbound to one node severed (outbound up) | pre-vote + check-quorum hold the leader; single-leader-per-term; no loss |
 **Fault knobs** (behind the `fault-injection` feature — compiled OUT of the
 production image, inert until armed): `TIDAL_FAULT_FSYNC_DELAY_MS=<ms>` slows
 every durable WAL fsync; `TIDAL_FAULT_DISK_FULL_AFTER_BYTES=<n>` fails segment
 writes with `ENOSPC` after `n` cumulative bytes this process lifetime (arm it on
 a node at restart to fail after `n` bytes of post-restart writes). NEVER set
 these on a production node.
 **Soak with regression gates** (`tidal-stress`):
 ```
 tidal-stress --target http://<gateway> --ramp "3900:3600" --mix peach \
  --json-summary soak.json --max-error-pct 1 --max-p99-ms 250 --fail-on-knee
 ```
 `--fail-on-knee` (built-in SLO), `--max-p99-ms`, and `--max-error-pct` make the
 run exit non-zero on a regression; `--json-summary` writes a machine-readable
 per-stage roll-up for trend lines. A bounded version runs nightly; the GA-bar
 1-hour 100k-DAU soak points `--target` at the live Ref-A cluster (`--ramp 3900:3600`).
 **Nightly CI** (`.woodpecker.yaml`, cron `nightly` — Woodpecker, never GitHub
 Actions): the chaos suites with elevated kill-points (`TIDAL_QUORUM_KILLPOINTS`,
 `TIDAL_ELECTION_KILLPOINTS`) then the gated soak. A nightly failure flags the
 day's correctness or performance regression. The guarantee→test map is
 [docs/planning/milestone-11/guarantee-traceability.md](../planning/milestone-11/guarantee-traceability.md).
 ## Cross-references
 - **Kubernetes deployment** — [docs/runbooks/kubernetes.md](kubernetes.md)
@ -1319,9 +1366,10 @@ against the measured fsync histogram.
 - **Roadmap / cluster status & known gaps** —
  [docs/planning/ROADMAP.md](../planning/ROADMAP.md) and
  [docs/roadmap-to-cluster.md](../roadmap-to-cluster.md) for the M11 cluster
-  status — quorum-ack writes (m11p3), automatic failover (m11p4), and
+  status — quorum-ack writes (m11p3), automatic failover (m11p4),
-  membership/discovery/elasticity (m11p5) all shipped; sharding × replication
+  membership/discovery/elasticity (m11p5), security hardening (m11p7),
-  (p6), security hardening (p7), and continuous correctness (p9) remain.
+  observability + operations (m11p8), and continuous correctness (m11p9) all
  shipped; sharding × replication (p6) data plane in progress.
 - **API & schema reference** — [API.md](../../API.md),
  [QUICKSTART.md](../../QUICKSTART.md), and the live `/openapi.json` document.
 - **Scope & vision** — [VISION.md](../../VISION.md).
--- a/tidal-server/Cargo.toml
+++ b/tidal-server/Cargo.toml
@ -95,6 +95,9 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus
 [features]
 cluster-e2e = []
 # m11p9 chaos testing: passes through to tidaldb's WAL fault injection. The
 # tier-3 fault suite spawns a binary built with this; production never sets it.
 fault-injection = ["tidaldb/fault-injection"]
 [dev-dependencies]
 tempfile = "3"
--- a/tidal-server/tests/cluster_chaos.rs
+++ b/tidal-server/tests/cluster_chaos.rs
@ -94,7 +94,8 @@ const AP_SOUTH: usize = 2;
 /// A short partition window during which the leader keeps writing.
 const PARTITION_WRITES: u64 = 8;
-// ── Shared seeding / feed helpers (canonical bodies in support::multiproc) ──────
+// ── Feed helpers (LOCAL copies; canonical home is support::invariants —
 //    this suite's migration onto the shared checkers is a tracked follow-up) ──────
 /// A node's local-region feed as a sorted `(entity_id, score)` vector.
 fn feed_pairs(
--- a/tidal-server/tests/cluster_faults.rs
+++ b/tidal-server/tests/cluster_faults.rs
@ -0,0 +1,404 @@
 //! Tier-3 fault-injection suite (m11p9 continuous correctness).
 //!
 //! The chaos gate (`cluster_chaos.rs`) covers partition / crash / clock-skew.
 //! This suite adds the fault classes the roadmap (§4/m11p9) names as missing:
 //!
 //! 1. **Disk-full on a follower** (`mp_disk_full_follower_degrades_no_acked_loss`)
 //!    — a follower whose WAL hits `ENOSPC` mid-replication halts its receiver
 //!    (degraded, alive), the healthy majority keeps acking quorum writes, NO
 //!    acknowledged write is lost, and a restart with space recovers it to full
 //!    parity.
 //! 2. **Slow-fsync on a follower** (`mp_slow_fsync_follower_lags_but_quorum_holds`)
 //!    — a slow disk lags one follower; the fast follower supplies quorum so
 //!    every write still commits; the slow node converges once the burst ends; no
 //!    loss.
 //! 3. **Slow-fsync on BOTH followers** (`mp_slow_fsync_both_followers_force_honest_quorum_timeout`)
 //!    — when no follower can confirm inside the budget, `ack=quorum` returns a
 //!    retryable 503 naming the laggards (never a false success) while
 //!    `ack=leader` is unaffected; the followers recover.
 //! 4. **Asymmetric partition** (`mp_asymmetric_partition_no_split_brain_no_loss`)
 //!    — a follower that loses INBOUND links (can still send) cannot disrupt the
 //!    cluster: pre-vote + check-quorum keep the standing leader, there is never a
 //!    second leader at the same term, and no acked write is lost.
 //!
 //! These are REAL faults exercising REAL recovery paths: the WAL hooks return a
 //! genuine `ENOSPC` / sleep the real fsync (behind the `fault-injection` feature,
 //! compiled out of production — see `tidal/src/fault.rs`); partitions sever real
 //! loopback TCP. Every test asserts through the first-class invariant checkers
 //! (`support::invariants`): the no-acked-loss ledger, cross-replica feed parity,
 //! single-leader-per-term, and per-node monotonic frontiers.
 //!
 //! Run: `cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_faults -- --nocapture --test-threads 1`
 #![cfg(feature = "cluster-e2e")]
 #![allow(
    clippy::unwrap_used,
    clippy::expect_used,
    clippy::panic,
    clippy::cast_possible_truncation,
    clippy::cast_precision_loss,
    clippy::too_many_lines
 )]
 mod support;
 use std::time::{Duration, Instant};
 use support::{
    invariants::{AckLedger, MonotonicCounters, assert_single_leader_now, feed_item_ids},
    multiproc::{BREAKER_RESET, ClusterOptions, MultiProcCluster, convergence_budget},
    partition::proxied_rewrite,
 };
 const LEADER: usize = 0;
 const FSYNC_DELAY: &str = "TIDAL_FAULT_FSYNC_DELAY_MS";
 const DISK_FULL: &str = "TIDAL_FAULT_DISK_FULL_AFTER_BYTES";
 /// Stable single-leader posture (no auto-election) + quorum default, for the
 /// fault tests whose subject is the data plane, not the election. A faulted
 /// node must not trigger a spurious failover that muddies the assertion.
 const STABLE_QUORUM_YAML: &str =
    "election:\n  auto_election: false\nreplication:\n  ack: quorum\n  quorum_timeout_ms: 2000";
 /// Drive `count` quorum item+view writes for ids `id_base..id_base+count`
 /// through `base`, recording the client-observed acks into `ledger`. Returns
 /// how many items the client saw acknowledged.
 fn drive_quorum_writes(base: &str, id_base: u64, count: u64, ledger: &mut AckLedger) -> u64 {
    let client = reqwest::blocking::Client::builder()
        .timeout(Duration::from_secs(4))
        .build()
        .unwrap();
    let mut acked = 0;
    for i in 0..count {
        if ledger.write_item_and_view(&client, base, "quorum", id_base + i) {
            acked += 1;
        }
    }
    acked
 }
 /// **Disk-full on a follower.** A follower whose WAL fills mid-replication must
 /// degrade gracefully (halt apply, stay alive — never corrupt, never crash),
 /// the healthy majority must keep acking `ack=quorum` writes, NO acknowledged
 /// write may be lost, and a restart with space must recover the follower to full
 /// content + score parity.
 #[test]
 fn mp_disk_full_follower_degrades_no_acked_loss() {
    // ap-south (node 2) hits ENOSPC after ~8 KiB of post-boot WAL writes — far
    // past the empty-cluster boot (it writes nothing until it applies), but
    // crossed after a few dozen replicated records.
    let opts = ClusterOptions::new(3)
        .with_env(2, DISK_FULL, "8192")
        .with_topology_extra(STABLE_QUORUM_YAML);
    let mut cluster = MultiProcCluster::start_with(opts);
    cluster.wait_converged_all(convergence_budget());
    // Write a burst of quorum item+view writes through the leader. node 1
    // (healthy) supplies the one-of-two follower quorum, so every write commits
    // even after node 2's disk fills and its receiver halts.
    let mut ledger = AckLedger::new();
    let mut mono = MonotonicCounters::new("disk-full");
    let mut acked = 0;
    for round in 0..6 {
        acked += drive_quorum_writes(&cluster.node(LEADER), 10_000 + round * 20, 20, &mut ledger);
        mono.observe_cluster_frontiers(&cluster);
    }
    assert!(acked >= 100, "expected ~120 acked writes, got {acked}");
    // node 2 is FROZEN behind: its WAL is full, its receiver halted, so its
    // applied frontier stalled while node 1 stayed current. Poll briefly to let
    // node 1 finish converging and confirm the asymmetry.
    let leader_seq = cluster.leader_last_seq().expect("leader serves status");
    let deadline = Instant::now() + convergence_budget();
    loop {
        let n1 = cluster.local_status(1).expect("node1 status");
        let n2 = cluster.local_status(2).expect("node2 status");
        let n1_applied = n1["applied_events"].as_u64().unwrap_or(0);
        let n2_applied = n2["applied_events"].as_u64().unwrap_or(0);
        if n1_applied >= leader_seq && n2_applied < leader_seq {
            println!(
                "[disk-full] node1 caught up (applied {n1_applied}/{leader_seq}); \
                 node2 frozen at applied {n2_applied} (disk full)"
            );
            break;
        }
        assert!(
            Instant::now() <= deadline,
            "node1 must converge while node2 stays frozen: \
             n1_applied={n1_applied} n2_applied={n2_applied} leader_seq={leader_seq}"
        );
        std::thread::sleep(Duration::from_millis(100));
    }
    // No acked loss: every acked write is durable on the HEALTHY follower's
    // contiguous frontier (the real quorum-durability claim under a disk-full
    // minority), and every acked item is content-present on it.
    let (_keep, _) = ledger.assert_frontier_covers_acks(&cluster, &[1usize], "disk-full");
    ledger.assert_items_present(&cluster, 1, Duration::from_secs(10), "disk-full");
    // HEAL: restart node 2 with the fault disarmed (0 = off). A fresh receiver
    // catches the full log up via the stream — no operator verb beyond restart.
    cluster.restart(2, &[(DISK_FULL, "0")]);
    cluster.wait_converged_all(convergence_budget() + BREAKER_RESET);
    // The recovered disk-full node now holds every acked write. Convergence is
    // proven three ways: the contiguous frontier matched the leader
    // (wait_converged_all above), the materialized item SET matches the leader,
    // and every acked item is content-present via /search. We assert feed item-SET
    // parity, NOT score parity: `trending` scores are a windowed velocity
    // (count / window), and a node rebuilt from a burst WAL-replay aligns its time
    // buckets differently than one that accumulated continuously — so a velocity
    // score legitimately differs across a restart even with identical durable data.
    // Decay/score parity to 1e-6 between CONTINUOUS replicas is asserted by the
    // slow-fsync and asymmetric tests (and the chaos gate).
    let leader_ids = feed_item_ids(&cluster, LEADER, "trending", 200);
    let recovered_ids = feed_item_ids(&cluster, 2, "trending", 200);
    assert_eq!(
        leader_ids, recovered_ids,
        "recovered disk-full node must materialize the same item set as the leader"
    );
    ledger.assert_items_present(
        &cluster,
        2,
        Duration::from_secs(10),
        "disk-full (post-recovery)",
    );
    println!(
        "[disk-full] node2 recovered after restart: {acked} acked writes all present, \
         item set parity with leader ({} items)",
        leader_ids.len()
    );
 }
 /// **Slow-fsync on one follower.** A slow disk lags one follower; the fast
 /// follower supplies quorum so every `ack=quorum` write still commits; the slow
 /// node is visibly behind during the burst, converges once it ends, and loses
 /// nothing.
 #[test]
 fn mp_slow_fsync_follower_lags_but_quorum_holds() {
    // ap-south (node 2) sleeps 250ms before every durable fsync.
    let opts = ClusterOptions::new(3)
        .with_env(2, FSYNC_DELAY, "250")
        .with_topology_extra(STABLE_QUORUM_YAML);
    let cluster = MultiProcCluster::start_with(opts);
    cluster.wait_converged_all(convergence_budget());
    let mut ledger = AckLedger::new();
    let mut mono = MonotonicCounters::new("slow-fsync-follower");
    let acked = drive_quorum_writes(&cluster.node(LEADER), 20_000, 60, &mut ledger);
    mono.observe_cluster_frontiers(&cluster);
    assert!(
        acked >= 55,
        "fast follower must supply quorum: only {acked} acked"
    );
    // Right after the burst the slow follower trails the fast one (250ms/fsync
    // ≫ the fast nodes' ~2ms group-commit), proving the slow disk is visible.
    let n1 = cluster.local_status(1).expect("node1 status");
    let n2 = cluster.local_status(2).expect("node2 status");
    let n1_lag = n1["lag_events"].as_u64().unwrap_or(u64::MAX);
    let n2_lag = n2["lag_events"].as_u64().unwrap_or(0);
    println!("[slow-fsync] after burst: node1 lag={n1_lag} node2(slow) lag={n2_lag}");
    assert!(
        n2_lag > n1_lag,
        "the slow follower must trail the fast one: node1 lag={n1_lag} node2 lag={n2_lag}"
    );
    // A slow disk is not a broken one: the burst over, the slow follower drains
    // and converges; no acked write is lost; frontiers never regressed.
    cluster.wait_converged_all(convergence_budget() * 2);
    mono.observe_cluster_frontiers(&cluster);
    let (_keep, _) =
        ledger.assert_frontier_covers_acks(&cluster, &[1usize, 2usize], "slow-fsync-follower");
    // Item-SET parity, not score: the slow follower drains its backlog as a burst
    // once the write burst ends, which reconstructs its time-bucketed velocity
    // differently than the leader's continuous accumulation — so a `trending`
    // (velocity) score can differ across the catch-up even with identical durable
    // data. The frontier + content + set checks prove data convergence robustly.
    let leader_ids = feed_item_ids(&cluster, LEADER, "trending", 200);
    let slow_ids = feed_item_ids(&cluster, 2, "trending", 200);
    assert_eq!(
        leader_ids, slow_ids,
        "converged slow follower must materialize the same item set as the leader"
    );
    ledger.assert_items_present(&cluster, 2, Duration::from_secs(10), "slow-fsync-follower");
    println!("[slow-fsync] {acked} acked writes, slow follower converged to parity, no loss");
 }
 /// **Slow-fsync on BOTH followers + tight budget → honest quorum timeout.** When
 /// neither follower can confirm a write inside `quorum_timeout_ms`, `ack=quorum`
 /// must return a retryable 503 naming the laggards — never a false 2xx — while
 /// `ack=leader` on the same cluster keeps succeeding (the leader's disk is fast).
 /// The slow followers are not broken, so the cluster converges once load eases.
 #[test]
 fn mp_slow_fsync_both_followers_force_honest_quorum_timeout() {
    // Both followers sleep 1500ms per fsync; the quorum budget is 300ms — so the
    // commit index physically cannot advance to a fresh write inside the budget.
    let yaml =
        "election:\n  auto_election: false\nreplication:\n  ack: quorum\n  quorum_timeout_ms: 300";
    let opts = ClusterOptions::new(3)
        .with_env(1, FSYNC_DELAY, "1500")
        .with_env(2, FSYNC_DELAY, "1500")
        .with_topology_extra(yaml);
    let cluster = MultiProcCluster::start_with(opts);
    cluster.wait_converged_all(convergence_budget());
    let client = cluster.client();
    let leader = cluster.node(LEADER);
    // ack=quorum (the topology default): the budget expires before either slow
    // follower fsyncs the apply → retryable 503 naming the laggards.
    let q = client
        .post(format!("{leader}/signals"))
        .json(&serde_json::json!({ "entity_id": 30_001, "signal": "view", "weight": 1.0 }))
        .send()
        .expect("quorum write sends");
    assert_eq!(
        q.status().as_u16(),
        503,
        "ack=quorum must time out honestly against two slow followers"
    );
    let body: serde_json::Value = q.json().expect("503 carries a JSON body");
    assert_eq!(
        body["retryable"].as_bool(),
        Some(true),
        "503 must be retryable: {body}"
    );
    assert!(
        body["laggards"].as_array().is_some_and(|l| !l.is_empty()),
        "503 must name the laggards: {body}"
    );
    println!(
        "[both-slow] ack=quorum → honest 503 retryable, laggards={}",
        body["laggards"]
    );
    // ack=leader on the same cluster is unaffected (the leader's disk is fast):
    // the caller's durability choice is the caller's, never the deployment's.
    let l = client
        .post(format!("{leader}/signals"))
        .header("x-tidal-ack", "leader")
        .json(&serde_json::json!({ "entity_id": 30_002, "signal": "view", "weight": 1.0 }))
        .send()
        .expect("leader write sends");
    assert_eq!(
        l.status().as_u16(),
        204,
        "ack=leader must keep succeeding while ack=quorum times out: {}",
        l.status()
    );
    println!("[both-slow] ack=leader → 204 (unaffected)");
    // The slow followers are not broken: they DO apply, just slowly. Once the
    // single in-flight write drains, the cluster converges (the timed-out write
    // is in the leader's log and commits late — at-least-once, never lost).
    cluster.wait_converged_all(convergence_budget() * 3);
    println!("[both-slow] slow followers drained and converged");
 }
 /// **Asymmetric partition.** A follower that loses its INBOUND links (peers
 /// cannot reach it; it can still send) must not disrupt the cluster: pre-vote +
 /// check-quorum keep the standing leader, there is never a second leader at the
 /// same term, the reachable majority keeps acking quorum writes with zero loss,
 /// and the follower rejoins cleanly on heal.
 #[test]
 fn mp_asymmetric_partition_no_split_brain_no_loss() {
    // Proxy only the edges INTO ap-south; ap-south's own outbound dials are
    // unproxied (identity), so severing the inbound edges is a true ASYMMETRIC
    // cut: ap-south stops RECEIVING heartbeats/ships but can still SEND the
    // disruptive RequestVotes pre-vote is designed to neutralize.
    let (rewrite, proxies) = proxied_rewrite(&["ap-south"]);
    let fast_election = "election:\n  heartbeat_interval_ms: 100\n  election_timeout_min_ms: 500\n  \
         election_timeout_max_ms: 1000\n  leader_lease_ms: 350\nreplication:\n  ack: quorum";
    let opts = ClusterOptions::new(3)
        .with_rewrite(rewrite)
        .with_topology_extra(fast_election);
    let cluster = MultiProcCluster::start_with(opts);
    cluster.wait_leader_agreed("us-east", convergence_budget());
    cluster.wait_converged_all(convergence_budget());
    // Baseline writes; capture the standing leader's term.
    let mut ledger = AckLedger::new();
    drive_quorum_writes(&cluster.node(LEADER), 40_000, 30, &mut ledger);
    let base_term = cluster.local_status(LEADER).expect("leader status")["term"]
        .as_u64()
        .unwrap_or(0);
    // ── Asymmetric sever: cut every inbound edge to ap-south ──────────────────
    proxies.region("ap-south").sever_all();
    // The reachable majority keeps acking quorum writes (us-east + eu-west).
    let acked = drive_quorum_writes(&cluster.node(LEADER), 41_000, 30, &mut ledger);
    assert!(
        acked >= 25,
        "reachable majority must keep acking: only {acked}"
    );
    // Through several election timeouts: the standing leader is unchanged, there
    // is never a second leader at any term, and ap-south's term does not explode
    // (pre-vote blocks a node that can't receive grants from bumping its term).
    let watch_until = Instant::now() + Duration::from_secs(12);
    let mut max_apsouth_term = base_term;
    while Instant::now() < watch_until {
        // The shared checker snapshots every live node and asserts
        // single-leader-per-term; reuse its snapshots for the rest of the checks
        // (no second fetch, no hardcoded node count).
        let statuses = assert_single_leader_now(&cluster, "asymmetric-partition");
        // The two reachable nodes still agree us-east leads.
        for st in &statuses {
            let region = st["region"].as_str().unwrap_or("?");
            if region == "us-east" || region == "eu-west" {
                assert_eq!(
                    st["leader"].as_str(),
                    Some("us-east"),
                    "reachable node {region} must still see us-east as leader: {st}"
                );
            }
            if region == "ap-south" {
                max_apsouth_term = max_apsouth_term.max(st["term"].as_u64().unwrap_or(0));
            }
        }
        std::thread::sleep(Duration::from_millis(200));
    }
    assert!(
        max_apsouth_term <= base_term + 2,
        "ap-south term exploded under asymmetric partition ({base_term} → {max_apsouth_term}); \
         pre-vote should have neutralized the disruptive node"
    );
    println!(
        "[asymmetric] leader held us-east, single-leader-per-term, ap-south term bounded \
         ({base_term} → {max_apsouth_term})"
    );
    // No acked loss: every acked write is durable on the reachable follower.
    let (_keep, _) =
        ledger.assert_frontier_covers_acks(&cluster, &[1usize], "asymmetric-partition");
    ledger.assert_items_present(
        &cluster,
        LEADER,
        Duration::from_secs(10),
        "asymmetric-partition",
    );
    // HEAL: ap-south's inbound returns; it catches up and rejoins as a follower.
    proxies.region("ap-south").heal_all();
    cluster.wait_converged_all(convergence_budget() + BREAKER_RESET);
    // Item-SET parity, not score: ap-south applies the partition backlog as a
    // burst on heal, reconstructing its velocity buckets differently than the
    // leader's continuous accumulation — so trending (velocity) scores can differ
    // across the catch-up even with identical durable data. Set + content +
    // frontier prove data convergence robustly.
    let leader_ids = feed_item_ids(&cluster, LEADER, "trending", 200);
    let rejoined_ids = feed_item_ids(&cluster, 2, "trending", 200);
    assert_eq!(
        leader_ids, rejoined_ids,
        "rejoined ap-south must materialize the same item set as the leader"
    );
    ledger.assert_items_present(
        &cluster,
        2,
        Duration::from_secs(10),
        "asymmetric-partition (rejoined)",
    );
    println!("[asymmetric] ap-south rejoined to parity, no acked loss");
 }
--- a/tidal-server/tests/cluster_quorum.rs
+++ b/tidal-server/tests/cluster_quorum.rs
@ -53,6 +53,7 @@ use std::sync::{
 use std::time::{Duration, Instant};
 use support::{
    invariants::{AckLedger, post_acked},
    multiproc::{BREAKER_RESET, ClusterOptions, MultiProcCluster, convergence_budget},
    partition::proxied_rewrite,
 };
@ -68,44 +69,6 @@ fn killpoints() -> usize {
        .unwrap_or(8)
 }
 /// A unique all-alpha search token for an entity id (digits 0-9 → letters
 /// a-j), so `/search?query=<token>` is an exact item-presence probe under
 /// the default tokenizer.
 fn item_token(entity_id: u64) -> String {
    let mut token = String::from("kpq");
    for d in entity_id.to_string().bytes() {
        token.push(char::from(b'a' + (d - b'0')));
    }
    token
 }
 /// POST with the `x-tidal-ack` header through a dedicated client. Returns
 /// `Some(seq)` only for a 2xx carrying `x-tidal-seq` — the ledger's
 /// definition of "acknowledged".
 fn post_acked(
    client: &reqwest::blocking::Client,
    base: &str,
    path: &str,
    ack: &str,
    body: &serde_json::Value,
 ) -> Option<u64> {
    let resp = client
        .post(format!("{base}{path}"))
        .header("x-tidal-ack", ack)
        .json(body)
        .send()
        .ok()?;
    if !resp.status().is_success() {
        return None;
    }
    resp.headers()
        .get("x-tidal-seq")?
        .to_str()
        .ok()?
        .parse()
        .ok()
 }
 /// m11p3 quorum semantics over a REAL 3-process cluster with real TCP
 /// partitions:
 ///
@ -268,7 +231,8 @@ fn mp_quorum_ledger_zero_acked_loss_across_killpoints() {
            cluster.node(1), // forwarded quorum writes through a follower
        ];
-        // ── Concurrent quorum writers, ledger = client-observed acks ───────
+        // ── Concurrent quorum writers; each builds an AckLedger of the writes
        // the client saw acknowledged (the shared first-class checker). ──────
        let stop = Arc::new(AtomicBool::new(false));
        let id_base = 1_000 * (round as u64 + 1);
        let mut writers = Vec::new();
@ -280,38 +244,16 @@ fn mp_quorum_ledger_zero_acked_loss_across_killpoints() {
                    .timeout(Duration::from_secs(4))
                    .build()
                    .unwrap();
-                // (entity_id, item_seq, view_seq-if-acked)
+                let mut ledger = AckLedger::new();
                let mut acked: Vec<(u64, u64, Option<u64>)> = Vec::new();
                let mut n = 0u64;
                while !stop.load(Ordering::Acquire) {
                    let entity_id = id_base + (w as u64) * 500 + n;
                    n += 1;
-                    let Some(item_seq) = post_acked(
+                    // An un-acked item (leader dying/dead, forward failed, or
-                        &client,
+                    // quorum timeout) is owed nothing by contract — skip it.
-                        &base,
+                    let _ = ledger.write_item_and_view(&client, &base, "quorum", entity_id);
                        "/items",
                        "quorum",
                        &serde_json::json!({
                            "entity_id": entity_id,
                            "metadata": { "title": item_token(entity_id) }
                        }),
                    ) else {
                        // Not acknowledged (leader dying/dead, forward failed,
                        // or quorum timeout): by contract it owes us nothing.
                        continue;
                    };
                    let view_seq = post_acked(
                        &client,
                        &base,
                        "/signals",
                        "quorum",
                        &serde_json::json!({
                            "entity_id": entity_id, "signal": "view", "weight": 1.0
                        }),
                    );
                    acked.push((entity_id, item_seq, view_seq));
                }
-                acked
+                ledger.writes().to_vec()
            }));
        }
@ -322,42 +264,18 @@ fn mp_quorum_ledger_zero_acked_loss_across_killpoints() {
        cluster.kill_hard(LEADER);
        stop.store(true, Ordering::Release);
        let _ = client_drain(&leader_base); // flush any half-open socket
-        let mut ledger: Vec<(u64, u64, Option<u64>)> = Vec::new();
+        let mut ledger = AckLedger::new();
        for w in writers {
-            ledger.extend(w.join().expect("writer thread"));
+            ledger.extend(w.join().expect("writer thread").into_iter());
        }
-        let max_acked_seq = ledger
+        let max_acked_seq = ledger.max_acked_seq();
            .iter()
            .map(|(_, item_seq, view_seq)| view_seq.unwrap_or(*item_seq).max(*item_seq))
            .max()
            .unwrap_or(0);
-        // ── Operator rule: promote the max-applied survivor ────────────────
+        // ── INVARIANT A (frontier): no acked seqno above the max-applied
-        let survivors = [1usize, 2usize];
+        // survivor's contiguous durable frontier — the shared checker also
-        let applied: Vec<(usize, u64)> = survivors
+        // returns the survivor the operator rule says to promote. ───────────
-            .iter()
+        let ctx = format!("round {round}");
-            .map(|&idx| {
+        let (chosen, chosen_applied) =
-                let status = cluster
+            ledger.assert_frontier_covers_acks(&cluster, &[1usize, 2usize], &ctx);
                    .local_status(idx)
                    .expect("survivor must serve status");
                (idx, status["applied_events"].as_u64().unwrap())
            })
            .collect();
        let (chosen, chosen_applied) = applied
            .iter()
            .copied()
            .max_by_key(|&(_, a)| a)
            .expect("two survivors");
        // ── INVARIANT A (frontier): no acked seqno above the chosen
        // survivor's contiguous durable frontier ───────────────────────────
        assert!(
            max_acked_seq <= chosen_applied,
            "round {round}: ACKNOWLEDGED LOSS — max acked seq {max_acked_seq} exceeds the \
             max-applied survivor's frontier {chosen_applied} (applied: {applied:?}, \
             {} acked writes)",
            ledger.len()
        );
        // m11p4: promote is a FENCED transfer — the election's up-to-date
        // restriction can refuse a target that fell behind between this
@ -372,10 +290,9 @@ fn mp_quorum_ledger_zero_acked_loss_across_killpoints() {
            &serde_json::json!({ "region": new_leader }),
        );
        if resp.status().as_u16() != 200 {
-            let (other, _) = applied
+            let other = [1usize, 2usize]
-                .iter()
+                .into_iter()
-                .copied()
+                .find(|&idx| idx != chosen)
                .find(|&(idx, _)| idx != chosen)
                .expect("two survivors");
            println!(
                "[ledger] round {round}: promote of {new_leader} refused (it fell \
@ -391,50 +308,12 @@ fn mp_quorum_ledger_zero_acked_loss_across_killpoints() {
        }
        cluster.wait_leader_agreed(&new_leader, Duration::from_secs(10));
-        // ── INVARIANT B (content): every acked item is on the new leader ───
+        // ── INVARIANT B (content): every acked item is present on the new
-        let client = reqwest::blocking::Client::builder()
+        // leader (the shared checker polls past the text index's 2s commit). ─
            .timeout(Duration::from_secs(4))
            .build()
            .unwrap();
        // The text index auto-commits every 2s (engine default), so the FIRST
        // probe polls past the commit interval; data presence is what is
        // asserted, not commit timing.
        let winner_idx = (0..3)
            .find(|&i| cluster.region_name(i) == new_leader)
            .expect("winner index");
-        let new_leader_base = cluster.node(winner_idx);
+        ledger.assert_items_present(&cluster, winner_idx, Duration::from_secs(10), &ctx);
        let search_deadline = Instant::now() + Duration::from_secs(10);
        for (entity_id, item_seq, _) in &ledger {
            let token = item_token(*entity_id);
            // Definitely assigned: the loop body's first statement writes it
            // before any break can be reached.
            let mut last: serde_json::Value;
            let present = loop {
                last = client
                    .get(format!("{new_leader_base}/search?query={token}&limit=5"))
                    .send()
                    .unwrap()
                    .json()
                    .unwrap();
                let hit = last["items"]
                    .as_array()
                    .unwrap_or(&Vec::new())
                    .iter()
                    .any(|it| it["entity_id"].as_u64() == Some(*entity_id));
                if hit {
                    break true;
                }
                if Instant::now() > search_deadline {
                    break false;
                }
                std::thread::sleep(Duration::from_millis(200));
            };
            assert!(
                present,
                "round {round}: ACKNOWLEDGED LOSS — item {entity_id} (seq {item_seq}, \
                 acked at quorum) is missing on promoted leader {new_leader}: {last}"
            );
        }
        println!(
            "[ledger] round {round}: kill@{kill_after:?} → {} acked writes (max seq \
             {max_acked_seq}) all present on {new_leader} (applied {chosen_applied})",
--- a/tidal-server/tests/support/invariants.rs
+++ b/tidal-server/tests/support/invariants.rs
@ -0,0 +1,588 @@
 //! First-class correctness invariant checkers for the tier-3 chaos suites (m11p9).
 //!
 //! Before m11p9 each suite carried its own inline copy of these checks — the
 //! zero-acked-loss ledger lived in `cluster_quorum.rs`, feed-score parity in
 //! `cluster_chaos.rs`, the single-leader-per-term assertion in
 //! `cluster_election.rs`. m11p9 makes this module the **canonical, audited home**
 //! so a new fault class proves the SAME invariants the durability gate does, and
 //! a fix to a checker fixes it for every CONSUMER at once.
 //!
 //! Consumers today: `cluster_quorum.rs` (the m11p3 ledger gate, migrated onto
 //! `AckLedger`) and the new `cluster_faults.rs`. The legacy suites
 //! (`cluster_chaos`, `cluster_election`, `cluster_membership`, `cluster_reseed`,
 //! `cluster_multiproc`, `cluster_lifecycle`) still carry their own pre-m11p9
 //! local copies of `item_token`/`post_acked`/`feed_pairs`/`assert_feed_parity`
 //! and their inline single-leader assertions — migrating them onto this module is
 //! a tracked follow-up (note: some are deliberate variants, e.g. `cluster_chaos`'s
 //! `feed_pairs` panics via `.unwrap()` where the shared one defaults via
 //! `.unwrap_or`, so the migration is a behavior review, not a blind swap).
 //!
 //! The four checkers map to the roadmap's named invariants:
 //!
 //! 1. [`AckLedger`] — **no acknowledged-write loss**. Records every write the
 //!    client saw a 2xx + `x-tidal-seq` for, then proves after a leader kill that
 //!    (A, frontier) no acked seqno exceeds the max-applied survivor's contiguous
 //!    durable frontier, and (B, content) every acked item is actually present on
 //!    the promoted leader. This is the m11p3 exit-gate proof, extracted verbatim.
 //! 2. [`assert_feed_parity`] — **cross-replica decay parity to 1e-6**. Two
 //!    replicas that have applied the same log must rank identical items with
 //!    identical decayed scores (a doubly-applied or dropped segment inflates or
 //!    deflates a score past the tolerance).
 //! 3. [`assert_single_leader_per_term`] — **membership safety**: at most one
 //!    leader per term per shard, across a set of node status snapshots. A second
 //!    leader at the same term is a split brain.
 //! 4. [`MonotonicCounters`] — **per-entity monotonic counters**: a named u64
 //!    quantity (a node's applied frontier, a leader's commit index, an entity's
 //!    durable visibility) must never move backward across observations. A
 //!    regression means durably-acknowledged state was forgotten — the failure
 //!    mode disk-full / slow-fsync chaos is most likely to expose.
 #![allow(dead_code)]
 use std::{
    collections::HashMap,
    time::{Duration, Instant},
 };
 use super::multiproc::MultiProcCluster;
 // ── No acknowledged-write loss (G-D durability) ──────────────────────────────
 /// A unique all-alpha search token for an entity id (digits 0-9 → letters a-j),
 /// so `/search?query=<token>` is an exact item-presence probe under the default
 /// tokenizer. Stable across suites so a recorded item is probeable everywhere.
 #[must_use]
 pub fn item_token(entity_id: u64) -> String {
    let mut token = String::from("kpq");
    for d in entity_id.to_string().bytes() {
        token.push(char::from(b'a' + (d - b'0')));
    }
    token
 }
 /// POST with the `x-tidal-ack` header through a caller-supplied client. Returns
 /// `Some(seq)` only for a 2xx carrying `x-tidal-seq` — the ledger's definition
 /// of "acknowledged". A non-2xx, a missing header, or any transport error is
 /// `None`: by the at-least-once contract an un-acked write is owed nothing.
 #[must_use]
 pub fn post_acked(
    client: &reqwest::blocking::Client,
    base: &str,
    path: &str,
    ack: &str,
    body: &serde_json::Value,
 ) -> Option<u64> {
    let resp = client
        .post(format!("{base}{path}"))
        .header("x-tidal-ack", ack)
        .json(body)
        .send()
        .ok()?;
    if !resp.status().is_success() {
        return None;
    }
    resp.headers()
        .get("x-tidal-seq")?
        .to_str()
        .ok()?
        .parse()
        .ok()
 }
 /// One acknowledged write the ledger tracks: the entity, the item record's
 /// seqno, and the view signal's seqno if it too was acked.
 #[derive(Clone, Copy, Debug)]
 pub struct AckedWrite {
    pub entity_id: u64,
    pub item_seq: u64,
    pub view_seq: Option<u64>,
 }
 /// The client-observed ledger of acknowledged writes — the ground truth a
 /// no-acked-loss proof replays against post-recovery state.
 #[derive(Default)]
 pub struct AckLedger {
    acked: Vec<AckedWrite>,
 }
 impl AckLedger {
    #[must_use]
    pub fn new() -> Self {
        Self::default()
    }
    /// Write one item (+ its `view` signal) at `ack` durability through `base`
    /// and record whatever the client saw acknowledged. The item is recorded
    /// only if its own write was acked (an un-acked item owes nothing); the view
    /// seqno is recorded as `Some` only if the view was acked too. Returns
    /// whether the item was acked (so a writer loop can pace itself).
    pub fn write_item_and_view(
        &mut self,
        client: &reqwest::blocking::Client,
        base: &str,
        ack: &str,
        entity_id: u64,
    ) -> bool {
        let Some(item_seq) = post_acked(
            client,
            base,
            "/items",
            ack,
            &serde_json::json!({
                "entity_id": entity_id,
                "metadata": { "title": item_token(entity_id) }
            }),
        ) else {
            return false;
        };
        let view_seq = post_acked(
            client,
            base,
            "/signals",
            ack,
            &serde_json::json!({ "entity_id": entity_id, "signal": "view", "weight": 1.0 }),
        );
        self.acked.push(AckedWrite {
            entity_id,
            item_seq,
            view_seq,
        });
        true
    }
    /// Fold another writer thread's ledger into this one.
    pub fn extend(&mut self, other: impl IntoIterator<Item = AckedWrite>) {
        self.acked.extend(other);
    }
    /// Record a raw acked write (for callers that post through their own path).
    pub fn record(&mut self, entity_id: u64, item_seq: u64, view_seq: Option<u64>) {
        self.acked.push(AckedWrite {
            entity_id,
            item_seq,
            view_seq,
        });
    }
    #[must_use]
    pub const fn len(&self) -> usize {
        self.acked.len()
    }
    #[must_use]
    pub const fn is_empty(&self) -> bool {
        self.acked.is_empty()
    }
    #[must_use]
    pub fn writes(&self) -> &[AckedWrite] {
        &self.acked
    }
    /// The highest seqno the client ever saw acknowledged (item or view).
    #[must_use]
    pub fn max_acked_seq(&self) -> u64 {
        self.acked
            .iter()
            .map(|w| w.view_seq.unwrap_or(w.item_seq).max(w.item_seq))
            .max()
            .unwrap_or(0)
    }
    /// INVARIANT A (frontier): no acked seqno exceeds the max-applied survivor's
    /// contiguous durable frontier. A quorum ack for seqno S means some
    /// follower's applied frontier reached S durably; the operator rule "promote
    /// the max-applied survivor" therefore guarantees the promoted node holds
    /// every acked seqno. Returns the chosen survivor's `(index, applied)` so the
    /// caller can promote it.
    ///
    /// SCOPE: reads the FLAT `applied_events` field, which under m11p6 sharding
    /// mirrors only the node's DEFAULT (lowest-id) hosted group — exact for S=1
    /// (the shipped topology). A multi-shard ledger must record each acked write's
    /// shard and compare against that shard's `shards[]` frontier (the follow-up,
    /// alongside the shard-aware path in [`assert_single_leader_per_term`]).
    ///
    /// # Panics
    ///
    /// Panics (ACKNOWLEDGED LOSS) if `max_acked_seq > max survivor applied`, or
    /// if no survivor serves status.
    #[must_use]
    pub fn assert_frontier_covers_acks(
        &self,
        cluster: &MultiProcCluster,
        survivors: &[usize],
        ctx: &str,
    ) -> (usize, u64) {
        let applied: Vec<(usize, u64)> = survivors
            .iter()
            .map(|&idx| {
                let status = cluster
                    .local_status(idx)
                    .unwrap_or_else(|| panic!("{ctx}: survivor {idx} must serve status"));
                (idx, status["applied_events"].as_u64().unwrap_or(0))
            })
            .collect();
        let (chosen, chosen_applied) = applied
            .iter()
            .copied()
            .max_by_key(|&(_, a)| a)
            .unwrap_or_else(|| panic!("{ctx}: no survivors supplied"));
        let max_acked = self.max_acked_seq();
        assert!(
            max_acked <= chosen_applied,
            "{ctx}: ACKNOWLEDGED LOSS — max acked seq {max_acked} exceeds the max-applied \
             survivor's frontier {chosen_applied} (applied: {applied:?}, {} acked writes)",
            self.acked.len()
        );
        (chosen, chosen_applied)
    }
    /// INVARIANT B (content): every acked item is present on `leader_idx` via
    /// `/search`, polling up to `budget` (the text index auto-commits every 2s,
    /// so a fresh item can race the first probe — presence is asserted, not
    /// commit timing). The frontier can't lie about data it lacks, but this
    /// catches a frontier that lies about data it claims.
    ///
    /// # Panics
    ///
    /// Panics (ACKNOWLEDGED LOSS) if any acked item is absent within `budget`.
    pub fn assert_items_present(
        &self,
        cluster: &MultiProcCluster,
        leader_idx: usize,
        budget: Duration,
        ctx: &str,
    ) {
        let base = cluster.node(leader_idx);
        let client = reqwest::blocking::Client::builder()
            .timeout(Duration::from_secs(4))
            .build()
            .expect("build probe client");
        let deadline = Instant::now() + budget;
        for w in &self.acked {
            let token = item_token(w.entity_id);
            // The text index auto-commits every 2s, so a fresh item can race the
            // first probe — poll up to `budget`, asserting presence not timing.
            // Track whether we ever got a CLEAN 2xx-with-no-hit: a poll that only
            // ever saw transport errors / non-2xx is a REACHABILITY failure (a
            // just-promoted leader still warming up), NOT durability loss — it
            // must not be charged as ACKNOWLEDGED LOSS.
            let mut saw_clean_miss = false;
            let mut last_note = String::from("no response");
            let present = loop {
                match client
                    .get(format!("{base}/search?query={token}&limit=5"))
                    .send()
                {
                    Ok(resp) if resp.status().is_success() => {
                        let body: serde_json::Value =
                            resp.json().unwrap_or(serde_json::Value::Null);
                        let hit = body["items"]
                            .as_array()
                            .unwrap_or(&Vec::new())
                            .iter()
                            .any(|it| it["entity_id"].as_u64() == Some(w.entity_id));
                        if hit {
                            break true;
                        }
                        saw_clean_miss = true;
                        last_note = format!("2xx, item absent: {body}");
                    }
                    Ok(resp) => last_note = format!("HTTP {}", resp.status()),
                    Err(e) => last_note = format!("transport error: {e}"),
                }
                if Instant::now() > deadline {
                    break false;
                }
                std::thread::sleep(Duration::from_millis(200));
            };
            // A clean 2xx-with-no-hit through the whole budget is true loss; a
            // budget spent only on transport/non-2xx failures is a reachability
            // bug in the probe path, surfaced distinctly so it is never
            // misattributed to data loss. The two guards must match their
            // messages: PROBE UNREACHABLE fires only when we NEVER saw a clean
            // miss (transport/non-2xx the whole time); ACKNOWLEDGED LOSS fires
            // only when we DID see a clean 2xx-with-no-hit (the item is genuinely
            // absent on a reachable leader).
            assert!(
                present || saw_clean_miss,
                "{ctx}: PROBE UNREACHABLE — item {} (seq {}, acked) could not be probed on \
                 leader node {leader_idx} within the budget (last: {last_note}); this is a \
                 reachability failure, not necessarily data loss",
                w.entity_id,
                w.item_seq
            );
            assert!(
                present || !saw_clean_miss,
                "{ctx}: ACKNOWLEDGED LOSS — item {} (seq {}, acked) is missing on leader \
                 node {leader_idx} (last: {last_note})",
                w.entity_id,
                w.item_seq
            );
        }
    }
 }
 // ── Cross-replica decay parity (G-D / G-C) ───────────────────────────────────
 /// A node's local-region feed as a sorted `(entity_id, score)` vector.
 ///
 /// # Panics
 ///
 /// Panics if the feed body is not the expected shape.
 #[must_use]
 pub fn feed_pairs(
    cluster: &MultiProcCluster,
    idx: usize,
    profile: &str,
    limit: u32,
 ) -> Vec<(u64, f64)> {
    let body = cluster.get_json(idx, &format!("/feed?profile={profile}&limit={limit}"));
    let mut pairs: Vec<(u64, f64)> = body["items"]
        .as_array()
        .unwrap_or(&Vec::new())
        .iter()
        .map(|it| {
            (
                it["entity_id"].as_u64().unwrap_or(0),
                it["score"].as_f64().unwrap_or(f64::NAN),
            )
        })
        .collect();
    pairs.sort_by_key(|(id, _)| *id);
    pairs
 }
 /// A node's local-region feed as a sorted entity-id vector — the data-convergence
 /// view (which items are materialized), independent of score. Robust across a
 /// node RESTART, where a velocity/rate score (`trending`/`hot`) legitimately
 /// differs because its time-bucketed windowed counts are rebuilt from a burst
 /// WAL-replay rather than continuous accumulation, even though the durable data
 /// is identical. Use this for cross-replica DATA parity after a restart; use
 /// [`assert_feed_parity`] for decay/score parity between continuously-running
 /// replicas.
 ///
 /// # Panics
 ///
 /// Panics if the feed body is not the expected shape.
 #[must_use]
 pub fn feed_item_ids(
    cluster: &MultiProcCluster,
    idx: usize,
    profile: &str,
    limit: u32,
 ) -> Vec<u64> {
    feed_pairs(cluster, idx, profile, limit)
        .into_iter()
        .map(|(id, _)| id)
        .collect()
 }
 /// Assert two replicas' feed views carry the SAME items with scores equal to
 /// `1e-6`. A doubly-applied segment inflates decayed scores past the tolerance;
 /// a dropped one deflates them — either trips this.
 ///
 /// # Panics
 ///
 /// Panics if the item sets differ or any score differs by more than `1e-6`.
 pub fn assert_feed_parity(label: &str, a: &[(u64, f64)], b: &[(u64, f64)]) {
    assert_eq!(
        a.iter().map(|(id, _)| *id).collect::<Vec<_>>(),
        b.iter().map(|(id, _)| *id).collect::<Vec<_>>(),
        "{label}: feed item sets differ"
    );
    for ((id_a, score_a), (_, score_b)) in a.iter().zip(b.iter()) {
        assert!(
            (score_a - score_b).abs() <= 1e-6,
            "{label}: score for item {id_a} differs: {score_a} vs {score_b}"
        );
    }
 }
 // ── Membership safety: single leader per term per shard (G-A availability) ────
 /// Assert that across `statuses` (each a `/cluster/status/local` body) no two
 /// nodes claim to be leader at the same term. Reads the per-shard `shards[]`
 /// rows when present (m11p6 multi-shard) and falls back to the flat
 /// `role`/`term` fields (S=1), so it is correct on both surfaces. A second
 /// leader at the same `(shard, term)` is a split brain.
 ///
 /// # Panics
 ///
 /// Panics naming both offending nodes if a `(shard, term)` has two leaders.
 pub fn assert_single_leader_per_term(statuses: &[serde_json::Value], ctx: &str) {
    // (shard, term) -> the region already seen leading it.
    let mut leader_of: HashMap<(u64, u64), String> = HashMap::new();
    for st in statuses {
        let region = st["region"].as_str().unwrap_or("<unknown>").to_string();
        let rows = st["shards"].as_array();
        let claims: Vec<(u64, u64, bool)> = match rows {
            Some(rows) if !rows.is_empty() => rows
                .iter()
                .map(|r| {
                    (
                        r["shard"].as_u64().unwrap_or(0),
                        r["term"].as_u64().unwrap_or(0),
                        r["role"].as_str() == Some("leader"),
                    )
                })
                .collect(),
            _ => vec![(
                0,
                st["term"].as_u64().unwrap_or(0),
                st["role"].as_str() == Some("leader"),
            )],
        };
        for (shard, term, is_leader) in claims {
            if !is_leader {
                continue;
            }
            // A genuine split brain is TWO DISTINCT regions leading the same
            // (shard, term). The same region's status observed twice in one
            // snapshot (a duplicate sample, a retry-appended list) is NOT a split
            // brain — gate on `prev != region` so it is idempotently ignored.
            if let Some(prev) = leader_of.insert((shard, term), region.clone())
                && prev != region
            {
                panic!(
                    "{ctx}: SPLIT BRAIN — {prev} and {region} both lead shard {shard} term {term}"
                );
            }
        }
    }
 }
 /// Convenience: snapshot every live node's `/cluster/status/local` and assert
 /// single-leader-per-term. Returns the snapshots for further assertions.
 #[must_use]
 pub fn assert_single_leader_now(cluster: &MultiProcCluster, ctx: &str) -> Vec<serde_json::Value> {
    let statuses: Vec<serde_json::Value> = (0..cluster.len())
        .filter_map(|i| cluster.local_status(i))
        .collect();
    assert_single_leader_per_term(&statuses, ctx);
    statuses
 }
 // ── Per-entity monotonic counters (G-C continuous correctness) ────────────────
 /// Asserts a set of named `u64` counters never moves backward across
 /// observations. Each [`observe`](Self::observe) checks the new value against
 /// the last seen for that key; a decrease panics. Used for per-node applied
 /// frontiers, a leader's commit index, and per-entity durable visibility
 /// (0→1, never 1→0): quantities that are monotonic by construction, so a
 /// regression is durably-acknowledged state being forgotten.
 pub struct MonotonicCounters {
    label: String,
    last: HashMap<String, u64>,
    /// The leader region observed on the previous frontier sweep, so a leader
    /// change (a new epoch, where `commit_index` legitimately resets to the
    /// promote baseline) clears the commit-index tracking instead of false-firing.
    last_leader: Option<String>,
    /// Per-region role (`true` = leader) observed on the previous sweep. A
    /// region's `applied_events` is role-RELATIVE — a leader reports its own
    /// flushed frontier, a follower reports how far it applied the CURRENT
    /// leader's stream — so a leader↔follower flip re-baselines the value and is
    /// NOT a regression. We re-baseline `applied:<region>` on any role flip.
    last_role: HashMap<String, bool>,
 }
 impl MonotonicCounters {
    #[must_use]
    pub fn new(label: &str) -> Self {
        Self {
            label: label.to_string(),
            last: HashMap::new(),
            last_leader: None,
            last_role: HashMap::new(),
        }
    }
    /// Observe `value` for `key`; panic if it is below the last value seen.
    ///
    /// # Panics
    ///
    /// Panics if `value` regressed below a prior observation of `key`.
    pub fn observe(&mut self, key: &str, value: u64) {
        if let Some(&prev) = self.last.get(key) {
            assert!(
                value >= prev,
                "{}: NON-MONOTONIC — {key} regressed {prev} → {value} (acknowledged state lost)",
                self.label
            );
        }
        self.last.insert(key.to_string(), value);
    }
    /// Forget any tracking for `key` (e.g. when a node legitimately resets its
    /// frontier by reseeding, so the next observation starts a fresh baseline).
    pub fn reset(&mut self, key: &str) {
        self.last.remove(key);
    }
    /// Observe every live node's `applied_events` frontier (keyed by region) and
    /// the current leader's `commit_index`. Call repeatedly through a fault
    /// window: within a stable leadership epoch no node's durable frontier and
    /// no leader's commit index may ever regress.
    ///
    /// Legitimate epoch resets are handled, not flagged. `applied_events` is
    /// role-RELATIVE (a leader reports its own flushed frontier; a follower
    /// reports how far it applied the current leader's stream), so it is
    /// re-baselined — not asserted — whenever a region (a) reports
    /// `reseeding`/`quarantined` (mid snapshot-reinstall) OR (b) flips role
    /// leader↔follower (a clean failover, where the value changes WHAT it
    /// measures). `commit_index` is leadership-scoped and re-baselined on a
    /// leader-region change. What remains asserted is the real invariant: within
    /// a stable role/epoch a node never forgets durably-applied data, and a leader
    /// never un-commits a quorum-durable write within its term.
    /// SCOPE: this reads the FLAT top-level status fields (`is_leader`,
    /// `applied_events`, `commit_index`), which under m11p6 sharding mirror only
    /// the node's DEFAULT (lowest-id) hosted group — so it is exact for S=1 (the
    /// shipped topology) and tracks only the default group's frontier under S>1.
    /// The per-group `shards[]` rows (which [`assert_single_leader_per_term`]
    /// already consults) are the follow-up for multi-group frontier tracking.
    pub fn observe_cluster_frontiers(&mut self, cluster: &MultiProcCluster) {
        let mut leader_now: Option<String> = None;
        // Capture the leader's commit_index in the SAME sweep (no second N-request
        // scan): the value is in `st` while we hold it.
        let mut leader_commit: u64 = 0;
        for idx in 0..cluster.len() {
            let Some(st) = cluster.local_status(idx) else {
                continue;
            };
            let region = st["region"].as_str().unwrap_or("?").to_string();
            let is_leader = st["is_leader"].as_bool() == Some(true);
            let reseeding = st["reseeding"].as_bool() == Some(true)
                || st["quarantined"].as_bool() == Some(true);
            // A role flip (leader↔follower) re-baselines the role-relative
            // applied frontier; the first observation of a region is not a flip.
            let role_flipped = self
                .last_role
                .insert(region.clone(), is_leader)
                .is_some_and(|was| was != is_leader);
            let applied_key = format!("applied:{region}");
            if reseeding || role_flipped {
                self.reset(&applied_key);
                // Re-seed the baseline at the current value so the NEXT sweep
                // asserts from here, not from the stale pre-flip frontier.
                self.last
                    .insert(applied_key, st["applied_events"].as_u64().unwrap_or(0));
            } else {
                // A leader reports its flushed frontier as applied (m11p8); a
                // follower reports its applied frontier. Both are durable.
                self.observe(&applied_key, st["applied_events"].as_u64().unwrap_or(0));
            }
            if is_leader {
                leader_now = Some(region);
                leader_commit = st["commit_index"].as_u64().unwrap_or(0);
            }
        }
        // commit_index is leadership-scoped: clear it on a leader change.
        if leader_now != self.last_leader {
            self.reset("commit_index");
            self.last_leader.clone_from(&leader_now);
        }
        if leader_now.is_some() {
            self.observe("commit_index", leader_commit);
        }
    }
 }
--- a/tidal-server/tests/support/mod.rs
+++ b/tidal-server/tests/support/mod.rs
@ -14,5 +14,6 @@
 #![allow(dead_code)]
 pub mod invariants;
 pub mod multiproc;
 pub mod partition;
--- a/tidal-server/tests/support/multiproc.rs
+++ b/tidal-server/tests/support/multiproc.rs
@ -1159,16 +1159,26 @@ fn tidal_server_bin() -> PathBuf {
            .or_else(|| manifest_dir.parent())
            .expect("locate workspace root");
        // Build with `fault-injection` so the spawned cluster processes carry the
        // m11p9 WAL slow-fsync / disk-full hooks (inert unless a `TIDAL_FAULT_*`
        // env var arms them — every non-fault suite spawns the same binary and is
        // unaffected). Production never passes this feature, so the shipped image
        // compiles the hooks out entirely.
        let status = Command::new("cargo")
            .arg("build")
            .arg("-p")
            .arg("tidal-server")
            .arg("--features")
            .arg("fault-injection")
            .current_dir(workspace_root)
            .stdout(std::process::Stdio::null())
            .stderr(std::process::Stdio::null())
            .status()
            .expect("run cargo build");
-        assert!(status.success(), "cargo build -p tidal-server failed");
+        assert!(
            status.success(),
            "cargo build -p tidal-server --features fault-injection failed"
        );
        let bin = workspace_root.join("target/debug/tidal-server");
        assert!(
--- a/tidal-stress/Cargo.toml
+++ b/tidal-stress/Cargo.toml
@ -60,6 +60,10 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus
 # network): bench workload.next() construction and the latency-histogram record
 # path in isolation. Matches the engine crate's criterion posture.
 criterion = { version = "0.5", features = ["html_reports"] }
 # Test-only: the summary module's JSON is hand-rolled (no serializer dep on the
 # hot path), but the round-trip unit test parses it with a real strict parser so
 # a malformed-JSON regression is caught.
 serde_json = "1"
 [[bench]]
 name = "hotpath"
--- a/tidal-stress/src/error.rs
+++ b/tidal-stress/src/error.rs
@ -35,6 +35,26 @@ pub enum StressError {
    /// header value (contains control or non-visible characters).
    #[error("invalid api key: {0}")]
    Auth(String),
    /// A regression gate (`--max-p99-ms`, `--max-error-pct`, `--fail-on-knee`)
    /// was breached. The run completed and reported normally; this is the
    /// non-zero exit that fails the nightly CI step (m11p9).
    #[error("regression gate breached: {0}")]
    Gate(String),
    /// A gate threshold flag was invalid (non-finite or negative) — an operator
    /// CLI mistake caught before the run starts, NOT a regression.
    #[error("invalid gate threshold: {0}")]
    BadGate(String),
    /// The `--json-summary` artifact could not be written (an IO/disk fault, not
    /// a gate breach).
    #[error("could not write summary {path}: {source}")]
    Summary {
        path: String,
        #[source]
        source: std::io::Error,
    },
 }
 /// Crate-local result alias.
--- a/tidal-stress/src/lib.rs
+++ b/tidal-stress/src/lib.rs
@ -12,4 +12,5 @@ pub mod client;
 pub mod error;
 pub mod metrics;
 pub mod scheduler;
 pub mod summary;
 pub mod workload;
--- a/tidal-stress/src/main.rs
+++ b/tidal-stress/src/main.rs
@ -20,6 +20,7 @@ use tidal_stress::client::{HttpClient, seed_corpus};
 use tidal_stress::error::{Result, StressError};
 use tidal_stress::metrics::{self, StageStats};
 use tidal_stress::scheduler::{Stage, parse_ramp, run_stage};
 use tidal_stress::summary::{GateConfig, RunSummary, StageSummary};
 use tidal_stress::workload::{OpKind, Workload, WritePath, parse_mix};
 #[derive(Parser)]
@ -27,6 +28,9 @@ use tidal_stress::workload::{OpKind, Workload, WritePath, parse_mix};
    version,
    about = "Open-loop capacity ramp for tidalDB (thepeach feed workload)"
 )]
 // A CLI args struct: each bool is an independent flag, not a state machine — the
 // standard exception to `struct_excessive_bools`.
 #[allow(clippy::struct_excessive_bools)]
 struct Cli {
    /// Region gateway base URL, e.g. http://10.43.99.11:9500 . Repeat for each
    /// region; reads round-robin across them (each serves locally), writes too
@ -119,6 +123,26 @@ struct Cli {
    /// feed p99 >150ms) instead of pushing every stage.
    #[arg(long, default_value_t = false)]
    stop_on_knee: bool,
    /// Write a machine-readable JSON run summary to this path (per-stage p99 /
    /// throughput / error rate + the gate verdict). For nightly soak trend lines.
    #[arg(long)]
    json_summary: Option<String>,
    /// Regression gate: exit non-zero if any stage's worst-op p99 exceeds this
    /// (milliseconds). Omitted = no p99 gate.
    #[arg(long)]
    max_p99_ms: Option<f64>,
    /// Regression gate: exit non-zero if any stage's error rate exceeds this
    /// (percent, e.g. 1.0 = 1%). Omitted = no error-rate gate.
    #[arg(long)]
    max_error_pct: Option<f64>,
    /// Regression gate: exit non-zero if any stage breached the built-in SLO
    /// (the capacity knee). The nightly soak's PASS/FAIL signal.
    #[arg(long, default_value_t = false)]
    fail_on_knee: bool,
 }
 // SLO thresholds for the per-stage verdict. tidalDB's own RETRIEVE SLA is p99
@ -169,6 +193,22 @@ async fn run() -> Result<()> {
            "--ack must be leader|quorum, got {ack:?}"
        )));
    }
    // A gate threshold of NaN/inf would report the gate ARMED while every
    // `value > NaN` comparison is false — silently letting a regression through
    // (clap's default f64 parser accepts "nan"/"inf"). Reject non-finite or
    // negative thresholds so the gate cannot be disarmed by a typo.
    for (flag, val) in [
        ("--max-p99-ms", cli.max_p99_ms),
        ("--max-error-pct", cli.max_error_pct),
    ] {
        if let Some(v) = val
            && (!v.is_finite() || v < 0.0)
        {
            return Err(StressError::BadGate(format!(
                "{flag} must be finite and non-negative, got {v}"
            )));
        }
    }
    let client = Arc::new(HttpClient::new(
        Duration::from_secs(cli.request_timeout_secs),
        cli.api_key.clone(),
@ -267,6 +307,7 @@ async fn run() -> Result<()> {
    // ── Ramp ────────────────────────────────────────────────────────────────
    let mut best_pass: Option<(usize, f64, f64)> = None; // (stage idx, total ok/s, signal ok/s)
    let mut knee: Option<usize> = None;
    let mut stage_summaries: Vec<StageSummary> = Vec::new();
    for (i, stage) in stages.iter().enumerate() {
        if cli.poll_status
@ -285,6 +326,12 @@ async fn run() -> Result<()> {
        let _ = std::io::stdout().flush();
        let (passed, signal_ok_rps) = stage_verdict(&stats);
        stage_summaries.push(StageSummary::from_stage(
            stage.target_rps,
            &stats,
            passed,
            signal_ok_rps,
        ));
        if passed {
            best_pass = Some((i, stats.achieved_rps(), signal_ok_rps));
        } else if knee.is_none() {
@ -304,6 +351,35 @@ async fn run() -> Result<()> {
    }
    print_verdict(&cli, &stages, best_pass, knee);
    // ── Machine-readable summary + regression gates (m11p9) ──────────────────
    let run = RunSummary {
        stages: stage_summaries,
        knee_stage: knee,
        gate: GateConfig {
            max_p99_ms: cli.max_p99_ms,
            max_error_pct: cli.max_error_pct,
            fail_on_knee: cli.fail_on_knee,
        },
    };
    if let Some(path) = &cli.json_summary {
        std::fs::write(path, run.to_json()).map_err(|source| StressError::Summary {
            path: path.clone(),
            source,
        })?;
        println!("\nwrote JSON summary → {path}");
    }
    if run.gate.is_armed() {
        let breaches = run.breaches();
        if breaches.is_empty() {
            println!(
                "\nregression gates: PASS ({} stage(s) within thresholds)",
                run.stages.len()
            );
        } else {
            return Err(StressError::Gate(breaches.join("; ")));
        }
    }
    Ok(())
 }
--- a/tidal-stress/src/scheduler.rs
+++ b/tidal-stress/src/scheduler.rs
@ -191,6 +191,15 @@ pub fn parse_ramp(spec: &str, stage_secs: u64) -> Result<Vec<Stage>> {
                    .trim()
                    .parse()
                    .map_err(|_| StressError::Ramp(format!("bad rps '{r}'")))?;
                // `f64::from_str` accepts "inf"/"nan" and overflows ("1e400") to
                // +inf; a non-finite or negative rps is meaningless and would
                // poison the scheduler and the JSON summary (inf/NaN are not valid
                // JSON). Reject it at the boundary.
                if !target_rps.is_finite() || target_rps < 0.0 {
                    return Err(StressError::Ramp(format!(
                        "rps must be finite and non-negative, got '{r}'"
                    )));
                }
                let secs: u64 = s
                    .trim()
                    .parse()
--- a/tidal-stress/src/summary.rs
+++ b/tidal-stress/src/summary.rs
@ -0,0 +1,395 @@
 //! Machine-readable run summary + regression gates (m11p9 continuous correctness).
 //!
 //! The human-readable per-stage tables and the capacity verdict are for an
 //! operator reading the terminal. A NIGHTLY SOAK needs two more things the
 //! original tool lacked: a machine-readable artifact a trend-line job can archive
 //! and diff, and a PASS/FAIL exit code so a regression (p99 or error-rate creep,
 //! or the built-in SLO knee) fails the CI step instead of scrolling past in green.
 //!
 //! This module builds a flat JSON summary (hand-rolled — every value is a number,
 //! bool, or null, so no serializer dependency is needed, matching the
 //! dependency-light posture of [`crate::metrics`]) and evaluates the regression
 //! gates. A breach is surfaced as a [`crate::error::StressError::Gate`], which
 //! `main` turns into a non-zero exit.
 use std::fmt::Write as _;
 use crate::metrics::StageStats;
 use crate::workload::OpKind;
 /// One stage's machine-readable roll-up.
 pub struct StageSummary {
    pub target_rps: f64,
    pub achieved_rps: f64,
    pub ok_per_sec: f64,
    pub signal_ok_per_sec: f64,
    pub error_rate: f64,
    pub client_shed: u64,
    /// Feed-read p99 (ms), 0 if no feed reads in the mix.
    pub feed_p99_ms: f64,
    /// Worst signal-write p99 (ms) across view/like/skip, 0 if none.
    pub signal_p99_ms: f64,
    /// Worst p99 (ms) across every op that took OK traffic — the single number
    /// the `--max-p99-ms` gate compares.
    pub overall_p99_ms: f64,
    /// Whether this stage held the built-in SLO (error ≤1%, feed p99 ≤150ms exact,
    /// zero client shed).
    pub slo_passed: bool,
 }
 impl StageSummary {
    /// Build from a finished stage's stats and its built-in-SLO verdict.
    #[must_use]
    pub fn from_stage(
        target_rps: f64,
        stats: &StageStats,
        slo_passed: bool,
        signal_ok_per_sec: f64,
    ) -> Self {
        let p99_ms = |k: OpKind| {
            let op = &stats.ops[k.idx()];
            if op.ok() == 0 {
                0.0
            } else {
                op.hist.percentile(0.99).as_secs_f64() * 1000.0
            }
        };
        let feed_p99_ms = p99_ms(OpKind::FeedRead);
        let signal_p99_ms = [OpKind::SignalView, OpKind::SignalLike, OpKind::SignalSkip]
            .into_iter()
            .map(p99_ms)
            .fold(0.0_f64, f64::max);
        // The worst p99 across every op (zero-ok ops contribute 0.0 via the
        // closure's guard) — the single number the `--max-p99-ms` gate compares.
        // One source of truth for the per-op p99 math (the `p99_ms` closure).
        let overall_p99_ms = OpKind::ALL.into_iter().map(p99_ms).fold(0.0_f64, f64::max);
        Self {
            target_rps,
            achieved_rps: stats.achieved_rps(),
            ok_per_sec: stats.total_ok() as f64 / stats.elapsed.as_secs_f64().max(1e-9),
            signal_ok_per_sec,
            error_rate: stats.error_rate(),
            client_shed: stats.client_shed,
            feed_p99_ms,
            signal_p99_ms,
            overall_p99_ms,
            slo_passed,
        }
    }
 }
 /// Regression-gate thresholds resolved from the CLI.
 #[derive(Clone, Copy, Default)]
 pub struct GateConfig {
    /// Fail if any stage's worst-op p99 exceeds this (ms).
    pub max_p99_ms: Option<f64>,
    /// Fail if any stage's error rate exceeds this (percent, e.g. 1.0 = 1%).
    pub max_error_pct: Option<f64>,
    /// Fail if any stage breached the built-in SLO (the capacity knee).
    pub fail_on_knee: bool,
 }
 impl GateConfig {
    /// Whether any gate is armed (if not, the run is informational and always
    /// exits 0, preserving the original tool's behavior).
    #[must_use]
    pub const fn is_armed(&self) -> bool {
        self.max_p99_ms.is_some() || self.max_error_pct.is_some() || self.fail_on_knee
    }
 }
 /// The whole run's machine-readable summary.
 pub struct RunSummary {
    pub stages: Vec<StageSummary>,
    /// First stage index that breached the built-in SLO, if any.
    pub knee_stage: Option<usize>,
    pub gate: GateConfig,
 }
 impl RunSummary {
    /// Evaluate the armed regression gates. Returns the list of human-readable
    /// breach reasons (empty = passed). Unarmed gates never breach.
    #[must_use]
    pub fn breaches(&self) -> Vec<String> {
        let mut out = Vec::new();
        if self.gate.fail_on_knee
            && let Some(k) = self.knee_stage
        {
            out.push(format!(
                "built-in SLO breached at stage {} (target {:.0} rps) [--fail-on-knee]",
                k + 1,
                self.stages.get(k).map_or(0.0, |s| s.target_rps)
            ));
        }
        if let Some(max) = self.gate.max_p99_ms {
            for (i, s) in self.stages.iter().enumerate() {
                if s.overall_p99_ms > max {
                    out.push(format!(
                        "stage {} p99 {:.1}ms exceeds --max-p99-ms {max:.1}",
                        i + 1,
                        s.overall_p99_ms
                    ));
                }
            }
        }
        if let Some(max) = self.gate.max_error_pct {
            for (i, s) in self.stages.iter().enumerate() {
                if s.error_rate * 100.0 > max {
                    out.push(format!(
                        "stage {} error rate {:.2}% exceeds --max-error-pct {max:.2}",
                        i + 1,
                        s.error_rate * 100.0
                    ));
                }
            }
        }
        out
    }
    /// Whether the run passed every armed gate.
    #[must_use]
    pub fn passed(&self) -> bool {
        self.breaches().is_empty()
    }
    /// Render the flat JSON summary (hand-rolled; all values numeric/bool/null).
    #[must_use]
    pub fn to_json(&self) -> String {
        let mut s = String::new();
        let breaches = self.breaches();
        s.push_str("{\n");
        let _ = writeln!(s, "  \"passed\": {},", self.passed());
        let _ = writeln!(s, "  \"gates_armed\": {},", self.gate.is_armed());
        match self.knee_stage {
            Some(k) => {
                let _ = writeln!(s, "  \"knee_stage\": {},", k + 1);
            }
            None => s.push_str("  \"knee_stage\": null,\n"),
        }
        // Breach reasons as a JSON string array (these strings are tool-authored,
        // but escape quotes/backslashes defensively).
        s.push_str("  \"breaches\": [");
        for (i, b) in breaches.iter().enumerate() {
            if i > 0 {
                s.push_str(", ");
            }
            let _ = write!(s, "\"{}\"", b.replace('\\', "\\\\").replace('"', "\\\""));
        }
        s.push_str("],\n");
        s.push_str("  \"stages\": [\n");
        for (i, st) in self.stages.iter().enumerate() {
            s.push_str("    {");
            let _ = write!(s, "\"target_rps\": {}, ", jnum(st.target_rps, 1));
            let _ = write!(s, "\"achieved_rps\": {}, ", jnum(st.achieved_rps, 1));
            let _ = write!(s, "\"ok_per_sec\": {}, ", jnum(st.ok_per_sec, 1));
            let _ = write!(
                s,
                "\"signal_ok_per_sec\": {}, ",
                jnum(st.signal_ok_per_sec, 1)
            );
            let _ = write!(s, "\"error_rate\": {}, ", jnum(st.error_rate, 6));
            let _ = write!(s, "\"client_shed\": {}, ", st.client_shed);
            let _ = write!(s, "\"feed_p99_ms\": {}, ", jnum(st.feed_p99_ms, 3));
            let _ = write!(s, "\"signal_p99_ms\": {}, ", jnum(st.signal_p99_ms, 3));
            let _ = write!(s, "\"overall_p99_ms\": {}, ", jnum(st.overall_p99_ms, 3));
            let _ = write!(s, "\"slo_passed\": {}", st.slo_passed);
            s.push('}');
            if i + 1 < self.stages.len() {
                s.push(',');
            }
            s.push('\n');
        }
        s.push_str("  ]\n}\n");
        s
    }
 }
 /// Format an `f64` for JSON at `decimals` precision, or `null` if it is
 /// non-finite (NaN/inf are not valid JSON — a single unguarded field would
 /// otherwise corrupt the whole artifact). Defense in depth: the inputs are
 /// already validated finite at the CLI/parse boundary, but this guarantees the
 /// emitter can never produce invalid JSON regardless of upstream.
 fn jnum(v: f64, decimals: usize) -> String {
    if v.is_finite() {
        format!("{v:.decimals$}")
    } else {
        "null".to_string()
    }
 }
 #[cfg(test)]
 mod tests {
    use super::*;
    fn stage(
        target_rps: f64,
        error_rate: f64,
        overall_p99_ms: f64,
        slo_passed: bool,
    ) -> StageSummary {
        StageSummary {
            target_rps,
            achieved_rps: target_rps,
            ok_per_sec: target_rps,
            signal_ok_per_sec: target_rps,
            error_rate,
            client_shed: 0,
            feed_p99_ms: overall_p99_ms,
            signal_p99_ms: overall_p99_ms,
            overall_p99_ms,
            slo_passed,
        }
    }
    #[test]
    fn unarmed_gate_never_breaches() {
        let run = RunSummary {
            stages: vec![stage(1000.0, 0.5, 9999.0, false)],
            knee_stage: Some(0),
            gate: GateConfig::default(),
        };
        assert!(!run.gate.is_armed());
        assert!(
            run.passed(),
            "an unarmed run is informational — always passes"
        );
        assert!(run.breaches().is_empty());
    }
    #[test]
    fn p99_and_error_gates_breach_over_threshold_and_pass_within() {
        let over = RunSummary {
            stages: vec![stage(1000.0, 0.05, 300.0, true)],
            knee_stage: None,
            gate: GateConfig {
                max_p99_ms: Some(250.0),
                max_error_pct: Some(1.0),
                fail_on_knee: false,
            },
        };
        assert!(!over.passed(), "p99 300>250 and error 5%>1% must breach");
        assert_eq!(over.breaches().len(), 2);
        let within = RunSummary {
            stages: vec![stage(1000.0, 0.001, 100.0, true)],
            knee_stage: None,
            gate: GateConfig {
                max_p99_ms: Some(250.0),
                max_error_pct: Some(1.0),
                fail_on_knee: false,
            },
        };
        assert!(within.passed(), "p99 100<=250 and error 0.1%<=1% must pass");
    }
    #[test]
    fn fail_on_knee_breaches_only_with_a_knee() {
        let cfg = GateConfig {
            max_p99_ms: None,
            max_error_pct: None,
            fail_on_knee: true,
        };
        let with_knee = RunSummary {
            stages: vec![stage(1000.0, 0.0, 10.0, false)],
            knee_stage: Some(0),
            gate: cfg,
        };
        assert!(!with_knee.passed());
        let no_knee = RunSummary {
            stages: vec![stage(1000.0, 0.0, 10.0, true)],
            knee_stage: None,
            gate: cfg,
        };
        assert!(no_knee.passed());
    }
    #[test]
    fn json_round_trips_through_a_real_parser() {
        let run = RunSummary {
            stages: vec![
                stage(1500.0, 0.012, 142.5, false),
                stage(3000.0, 0.2, 410.0, false),
            ],
            knee_stage: Some(2),
            gate: GateConfig {
                max_p99_ms: Some(250.0),
                max_error_pct: Some(1.0),
                fail_on_knee: true,
            },
        };
        let parsed: serde_json::Value =
            serde_json::from_str(&run.to_json()).expect("to_json must be valid JSON");
        assert_eq!(parsed["passed"], false);
        assert_eq!(parsed["gates_armed"], true);
        assert_eq!(parsed["knee_stage"], 3);
        assert_eq!(parsed["stages"].as_array().expect("stages array").len(), 2);
        assert!(
            !parsed["breaches"]
                .as_array()
                .expect("breaches array")
                .is_empty()
        );
    }
    #[test]
    fn non_finite_field_serializes_as_null_not_invalid_json() {
        // Defense in depth: even if a non-finite value reached a field, the
        // emitter must still produce parseable JSON (null), never `inf`/`NaN`.
        let run = RunSummary {
            stages: vec![stage(f64::INFINITY, f64::NAN, 100.0, true)],
            knee_stage: None,
            gate: GateConfig::default(),
        };
        let json = run.to_json();
        assert!(
            !json.contains("inf") && !json.contains("NaN"),
            "no bare inf/NaN: {json}"
        );
        let parsed: serde_json::Value =
            serde_json::from_str(&json).expect("still valid JSON with null-sanitized fields");
        assert!(parsed["stages"][0]["target_rps"].is_null());
        assert!(parsed["stages"][0]["error_rate"].is_null());
    }
    #[test]
    fn from_stage_overall_p99_is_worst_busy_op_ignoring_zero_ok() {
        use std::time::Duration;
        use crate::metrics::StageStats;
        // A real StageStats: FeedRead busy with a ~120ms tail, SignalView busy
        // with a ~40ms tail, every other op left zero-ok (no traffic).
        let mut stats = StageStats::new();
        stats.elapsed = Duration::from_secs(10);
        let feed = OpKind::FeedRead.idx();
        let view = OpKind::SignalView.idx();
        for _ in 0..100 {
            stats.ops[feed].status[0] += 1; // count as Ok
            stats.ops[feed].hist.record(Duration::from_millis(120));
            stats.ops[view].status[0] += 1;
            stats.ops[view].hist.record(Duration::from_millis(40));
        }
        let s = StageSummary::from_stage(1000.0, &stats, true, 0.0);
        // The histogram is bucketed (~3-4%), so assert with tolerance.
        assert!(
            (100.0..150.0).contains(&s.feed_p99_ms),
            "feed p99 {} ~120ms",
            s.feed_p99_ms
        );
        assert!(
            (30.0..55.0).contains(&s.signal_p99_ms),
            "signal p99 {} ~40ms",
            s.signal_p99_ms
        );
        // overall = the WORST busy op (FeedRead), and the zero-ok ops (which would
        // each contribute 0.0) never inflate or drag it.
        assert!(
            (s.overall_p99_ms - s.feed_p99_ms).abs() < 1e-9,
            "overall p99 {} must equal the worst busy op's p99 {} (zero-ok ops excluded)",
            s.overall_p99_ms,
            s.feed_p99_ms
        );
        assert!(s.overall_p99_ms >= s.signal_p99_ms);
    }
 }
--- a/tidal/Cargo.toml
+++ b/tidal/Cargo.toml
@ -10,6 +10,11 @@ license = "MIT"
 default = ["metrics"]
 test-utils = ["dep:tempfile"]
 metrics = []   # hand-rolled HTTP, no new crate deps
 # m11p9 chaos testing: WAL slow-fsync + disk-full injection (src/fault.rs).
 # NOT in `default` and never passed by the production image build, so the
 # shipped binary compiles it out entirely. Inert until an env var arms it even
 # when compiled in. See src/fault.rs.
 fault-injection = []
 [dependencies]
 base64 = "0.22"
--- a/tidal/src/fault.rs
+++ b/tidal/src/fault.rs
@ -0,0 +1,147 @@
 //! Test-only fault injection for continuous-correctness chaos testing (m11p9).
 //!
 //! # Compiled out of production
 //!
 //! Every item in this module is behind the `fault-injection` cargo feature,
 //! which is **not** in tidaldb's `default` set and is **never** passed by the
 //! production image build (`docker/standalone/Dockerfile`: `cargo build -p
 //! tidal-server --release --locked`). With the feature off this module does not
 //! exist and the two WAL call sites that reference it
 //! ([`crate::wal::sync_file_durable`] and
 //! [`crate::wal::segment::SegmentWriter::write_batch_bytes`]) compile to their
 //! original instructions — the production binary is byte-identical to one built
 //! from a tree without this file.
 //!
 //! A disk-write or fsync fault that a stray env var could trip in production is
 //! exactly the kind of 3am footgun we refuse to ship, so the safety is
 //! structural (the code is absent), not merely a default. The chaos suites and
 //! the nightly CI build the binary with `--features fault-injection`; the
 //! `MultiProcCluster` harness's `tidal_server_bin()` does so explicitly.
 //!
 //! # Inert until armed
 //!
 //! Even *with* the feature compiled in, every fault is INERT until an env var
 //! explicitly arms it — the exact posture of the existing `TIDAL_HLC_SKEW_MS`,
 //! `TIDAL_QUORUM_KILLPOINTS`, and `TIDAL_ELECTION_KILLPOINTS` test knobs. The
 //! configuration is read **once** into a process-global `OnceLock` on first
 //! access, so an un-armed process pays at most one relaxed atomic load per WAL
 //! batch/fsync and no env lookups on the hot path.
 //!
 //! # The faults
 //!
 //! - **`TIDAL_FAULT_FSYNC_DELAY_MS`** — sleep this many milliseconds *before*
 //!   every durable WAL fsync. Models a slow disk / degraded NVMe: the data still
 //!   reaches stable storage, but each group-commit fsync costs `delay + real`.
 //!   On a follower this throttles apply (it WAL-firsts every replicated batch),
 //!   so under `ack=quorum` the commit index lags honestly; on a leader it raises
 //!   write latency without ever losing an acknowledged write.
 //!
 //! - **`TIDAL_FAULT_DISK_FULL_AFTER_BYTES`** — after this process has written N
 //!   cumulative bytes to WAL segments, every subsequent segment write fails with
 //!   a real `ENOSPC` (`std::io::Error::from_raw_os_error(28)`, the errno on both
 //!   Linux and macOS), wrapped in [`crate::wal::error::WalError::Io`] — the same
 //!   variant a genuine full disk produces. Arming it on a node *at restart*
 //!   (the cumulative counter starts at zero each process lifetime) makes the
 //!   threshold "fail after N bytes of post-restart writes", which is how the
 //!   chaos suite drives a node into disk-full mid-replication and then proves the
 //!   writer notifies its waiters, halts cleanly (no torn-batch corruption past
 //!   WAL recovery's BLAKE3 truncation), and the cluster loses no acknowledged
 //!   write because the healthy majority held quorum throughout.
 //!
 //! The injection returns `ENOSPC` *before* issuing the partial `write_all`, the
 //! standard chaos model (a clean, deterministic fault). A real full disk can
 //! also tear a batch mid-write; that case is already covered by the WAL's
 //! crash-recovery tail truncation and is not what this knob targets.
 //!
 //! # Granularity (per-PROCESS, not per-WAL)
 //!
 //! Both knobs are PROCESS-GLOBAL: the disk-full byte counter and the fsync delay
 //! apply to every WAL in the process collectively. Today (S=1, one
 //! `ShardReplica`/WAL per process) that is exactly one WAL, so the model is
 //! clean. Under m11p6 multi-shard co-location (a `BTreeMap<ShardId, ShardReplica>`
 //! in one process, each with its own `SegmentWriter`) the counter would aggregate
 //! across all hosted shards and fire `ENOSPC` on whichever shard first crosses
 //! the shared threshold — so a *per-shard* disk-full (fill shard 2 but not shard
 //! 1) needs either one-shard-per-process or keying these statics by shard/WAL.
 //! That keying is deferred until the m11p6 (S>1) fault suite actually needs it.
 use std::{
    sync::{
        OnceLock,
        atomic::{AtomicU64, Ordering},
    },
    time::Duration,
 };
 use crate::wal::error::WalError;
 /// Process-global fault configuration, parsed once from the environment.
 struct FaultConfig {
    /// Sleep before each durable fsync (slow-disk injection).
    fsync_delay: Option<Duration>,
    /// Fail segment writes once cumulative bytes exceed this (disk-full).
    disk_full_after: Option<u64>,
 }
 impl FaultConfig {
    /// Parse the fault env vars. A missing/blank/unparsable value disarms that
    /// fault (never a panic — an un-armed process must boot normally).
    fn from_env() -> Self {
        Self {
            fsync_delay: env_u64("TIDAL_FAULT_FSYNC_DELAY_MS").map(Duration::from_millis),
            disk_full_after: env_u64("TIDAL_FAULT_DISK_FULL_AFTER_BYTES"),
        }
    }
 }
 /// Read a non-negative `u64` env var, or `None` if unset/blank/zero/unparsable.
 /// Zero disarms (a `*_AFTER_BYTES=0` would otherwise fail the very first write,
 /// which is never the intent of an opt-in knob).
 fn env_u64(var: &str) -> Option<u64> {
    std::env::var(var)
        .ok()
        .and_then(|v| v.trim().parse::<u64>().ok())
        .filter(|&n| n > 0)
 }
 /// The parsed config, initialized once on first access.
 fn config() -> &'static FaultConfig {
    static CONFIG: OnceLock<FaultConfig> = OnceLock::new();
    CONFIG.get_or_init(FaultConfig::from_env)
 }
 /// Cumulative bytes this process has written to WAL segments (disk-full input).
 static SEGMENT_BYTES_WRITTEN: AtomicU64 = AtomicU64::new(0);
 /// Hook called at the top of every durable WAL fsync. With the slow-fsync fault
 /// armed, sleeps `TIDAL_FAULT_FSYNC_DELAY_MS` before the real sync; otherwise a
 /// single `Option` check and return.
 pub fn before_fsync() {
    if let Some(delay) = config().fsync_delay {
        std::thread::sleep(delay);
    }
 }
 /// Hook called at the top of every WAL segment write. With the disk-full fault
 /// armed, accounts `len` against the cumulative counter and returns a real
 /// `ENOSPC` once the threshold is crossed; otherwise returns `Ok` immediately
 /// without even touching the counter.
 ///
 /// # Errors
 ///
 /// Returns [`WalError::Io`] wrapping `ENOSPC` once cumulative segment bytes
 /// exceed `TIDAL_FAULT_DISK_FULL_AFTER_BYTES`.
 pub fn before_segment_write(len: usize) -> Result<(), WalError> {
    let Some(threshold) = config().disk_full_after else {
        return Ok(());
    };
    let total = SEGMENT_BYTES_WRITTEN.fetch_add(len as u64, Ordering::Relaxed) + len as u64;
    if total > threshold {
        // ENOSPC is errno 28 on both Linux and macOS. `from_raw_os_error`
        // produces an error whose `.kind()` the platform maps exactly as a real
        // full-disk write would — the same `WalError::Io` the writer's error
        // path already handles (notify waiters, keep serving).
        return Err(WalError::Io(std::io::Error::from_raw_os_error(28)));
    }
    Ok(())
 }
--- a/tidal/src/lib.rs
+++ b/tidal/src/lib.rs
@ -27,6 +27,12 @@ pub mod storage;
 pub mod text;
 pub mod wal;
 /// Test-only WAL fault injection (slow-fsync, disk-full) for the m11p9
 /// continuous-correctness chaos suites. Behind the non-default `fault-injection`
 /// feature, so production builds never compile it — see the module docs.
 #[cfg(feature = "fault-injection")]
 pub(crate) mod fault;
 /// Build hash compiled in from the `TIDALDB_BUILD_HASH` environment variable.
 ///
 /// Falls back to `"dev"` if `TIDALDB_BUILD_HASH` is unset or `build.rs` is not
--- a/tidal/src/wal/mod.rs
+++ b/tidal/src/wal/mod.rs
@ -77,6 +77,11 @@ pub type WalOpenResult = (
 /// appends, checkpoint temp). For directory-entry durability use
 /// [`sync_dir_durable`].
 pub(crate) fn sync_file_durable(file: &std::fs::File) -> Result<(), WalError> {
    // Test-only slow-disk injection (m11p9): inert unless the `fault-injection`
    // feature is compiled AND `TIDAL_FAULT_FSYNC_DELAY_MS` is armed. Compiled out
    // of production entirely.
    #[cfg(feature = "fault-injection")]
    crate::fault::before_fsync();
    #[cfg(target_os = "macos")]
    {
        // F_FULLFSYNC flushes the device write cache; plain fsync on macOS does
--- a/tidal/src/wal/segment.rs
+++ b/tidal/src/wal/segment.rs
@ -327,6 +327,13 @@ impl SegmentWriter {
    ///
    /// Returns `WalError::Io` on write failure.
    pub fn write_batch_bytes(&mut self, bytes: &[u8]) -> Result<u64, WalError> {
        // Test-only disk-full injection (m11p9): inert unless the
        // `fault-injection` feature is compiled AND
        // `TIDAL_FAULT_DISK_FULL_AFTER_BYTES` is armed. Returns a real ENOSPC
        // before any partial write, so the writer's error path runs unchanged.
        // Compiled out of production entirely.
        #[cfg(feature = "fault-injection")]
        crate::fault::before_segment_write(bytes.len())?;
        let offset = self.current_size;
        self.file.write_all(bytes)?;
        self.current_size += bytes.len() as u64;