From 5ed2edb211a27932b592a14ee6041278d90c3815 Mon Sep 17 00:00:00 2001 From: jx12n Date: Thu, 11 Jun 2026 13:28:08 -0600 Subject: [PATCH] =?UTF-8?q?feat(m11):=20quorum-acked=20writes=20=E2=80=94?= =?UTF-8?q?=20ack=3Dleader|quorum,=20commit=20index,=20durable=20frontier?= =?UTF-8?q?=20reports=20(m11p3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ack=quorum gates replicated writes on a majority of the replica set durably holding them: followers push their durably-applied frontier (ReportApplied, once per apply round, decoupled from ship acks), the leader folds frontier reports + ship-ack hints + heal resumes into a leadership-scoped CommitIndex (k-th-largest durable mark), and handlers await it through an async watch-channel bridge (zero parked threads per waiter). Honest timeouts: retryable 503 naming the laggards; x-tidal-seq on every cluster write. Follower blob applies are batched under group-commit fsyncs (22x seeding). Exit gate: 167/167 leader-SIGKILL kill points, zero acked-write loss. Seven-dimension review pass (all confirmed findings fixed): - WAL blob drain now ABORTS on the first write failure instead of reusing the failed seqno mid-drain (a torn record buried mid-segment would truncate every later acked record on replay) - apply_replicated_blobs waits every staged append even after a mid-batch failure, parses metadata once, and moves records into Arcs shared with the WAL writer (no deep clone per record on the follower apply path) - CommitIndex: zero-peer fast path now respects demotion (active checked under lock before the single-replica return), k-th-largest uses select_nth over a reused scratch buffer - await_quorum: re-reads the index once after the deadline fires (no false 503 for a write that committed in the race window), warns when the commit-watch bridge dies outside shutdown, zero-peer path checks active - notify_applied report failures: WARN on the first failure of a streak, INFO on recovery (a silently stalling frontier reads as unexplained quorum 503s); receiver skips re-notifying unadvanced frontiers - x-tidal-deduplicated: 1 marks dedup-suppressed signal writes (relayed through forwards) so durability cursors can tell dedup from no-seqno - docs: 167/167 kill-point record corrected in CHANGELOG; rolling-upgrade order (leader first — a pre-m11p3 leader silently downgrades quorum requests to leader-ack) in CHANGELOG + runbook §8; monitoring note for report-loss diagnosis on the quorum-timeout alert Verified: workspace clippy -D warnings (incl. cluster-e2e targets), full tidaldb/tidal-net/tidal-server/tidalctl suites green, tier-3 multi-process quorum suite green (8/8 kill points, zero acked loss, partition gate/recover). --- CHANGELOG.md | 74 ++++ docs/ops/monitoring.md | 6 +- docs/planning/ROADMAP.md | 6 +- docs/planning/milestone-11/phase-3.md | 172 +++++++++ docs/roadmap-to-cluster.md | 17 +- docs/runbooks/cluster.md | 175 ++++++---- tidal-net/proto/wal_shipping.proto | 22 ++ tidal-net/src/client.rs | 33 ++ tidal-net/src/server.rs | 185 +++++++++- tidal-net/src/sources.rs | 27 ++ tidal-net/src/transport.rs | 110 +++++- tidal-server/src/cluster/forward.rs | 63 +++- tidal-server/src/cluster/node.rs | 429 +++++++++++++++++++++-- tidal-server/src/cluster/routes.rs | 15 + tidal-server/src/cluster/topology.rs | 23 ++ tidal-server/src/error.rs | 16 + tidal-server/src/router.rs | 3 +- tidal-server/src/scatter_gather.rs | 2 + tidal-server/src/state.rs | 2 + tidal-server/tests/cluster_chaos.rs | 5 +- tidal-server/tests/cluster_quorum.rs | 419 ++++++++++++++++++++++ tidal-server/tests/cluster_region.rs | 258 ++++++++++++++ tidal-stress/src/client.rs | 20 +- tidal-stress/src/main.rs | 13 + tidal/src/db/items.rs | 168 ++++++++- tidal/src/db/metrics/cluster.rs | 26 +- tidal/src/db/replication_ops.rs | 16 +- tidal/src/db/signals.rs | 9 +- tidal/src/replication/commit.rs | 481 ++++++++++++++++++++++++++ tidal/src/replication/mod.rs | 2 + tidal/src/replication/receiver.rs | 128 ++++--- tidal/src/replication/relay.rs | 2 +- tidal/src/replication/ship.rs | 31 ++ tidal/src/replication/transport.rs | 30 +- tidal/src/signals/ledger/core.rs | 9 +- tidal/src/wal/mod.rs | 29 +- tidal/src/wal/writer.rs | 187 +++++----- 37 files changed, 2898 insertions(+), 315 deletions(-) create mode 100644 docs/planning/milestone-11/phase-3.md create mode 100644 tidal-server/tests/cluster_quorum.rs create mode 100644 tidal/src/replication/commit.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index cbde4a5..e792070 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,80 @@ All notable changes to tidalDB will be documented in this file. ### Added +**Quorum-acked writes (m11p3) — `ack=leader|quorum`, durable ship acks, commit index, zero-acked-loss ledger gate (closes G4)** +- **`ack=quorum`** is an opt-in durability contract for every replicated write + (`/signals`, `/items`, `/embeddings`): success means a **majority of the + replica set durably holds the write** (leader + `floor(n/2)` followers, each + storage-applied and own-WAL-fsynced), so an acked write survives the + permanent loss of any single node — including the leader. Deployment default + via topology `replication.ack`; per-request override via the **`x-tidal-ack`** + header (forwarded verbatim by gateways). `ack=leader` (the default) is the + m0–m11p2 contract unchanged. +- **Durable frontier reports.** Followers PUSH their durably-applied frontier + to the leader once per apply round (new `ReportApplied` RPC, fired by the + segment receiver through the new `Transport::notify_applied`) — batch-level + and fully decoupled from ship acks, so the commit index stays fresh even + when outbound ships stall (gap-parked follower, pull-based catch-up, quiet + leader). Ship acks keep their m11p2 instant floor-hint semantics — both + inputs are durable-true because a follower's frontier only ever advances + after its storage apply + own-WAL fsync. (The first design held each ship + ack until its segment's apply; measured under open-loop load, that couples + ship cadence to apply latency and one gap-parked follower spirals into + total quorum collapse — the report push is what shipped.) +- **Follower blob applies are group-committed.** m11p2 applied replicated + items/embeddings one record at a time — one solo follower fsync per item, + capping item apply at the fsync floor (~100/s on macOS) and stalling the + quorum frontier behind any item burst. The receiver now hands each apply + round's blob records to the engine as ONE batch (`apply_replicated_blobs`: + validate all → stage all WAL appends → wait all → upsert storage), and the + WAL writer flushes queued blobs under ONE group fsync. Measured: corpus + seeding 2,000 items + embeddings 39.3s → 1.8s (22×). +- **Commit index.** The leader folds durable acks (and heal resumes) into + per-peer durable marks; the commit index is the k-th largest (k = + `floor(n/2)`), leadership-scoped (promote resets it to the stream baseline; + demotion fails in-flight waiters — a demoted leader never claims quorum). + Handlers await it through a watch-channel bridge — **fully async, zero + threads parked per waiter** (the thread-per-wait design measurably collapsed + at 1k rps open-loop by exhausting the blocking pool and starving the very + completions that advance the index). Quorum capacity measured on a real + 3-process localhost cluster (release, writes mix): **3,600 quorum + signal-writes/s within SLO** (p50 ~45ms, zero errors, replication lag ≤3 + events at ramp end; knee not reached) — 79% of m11p1's 4,534/s leader-ack + figure, vs the ≥50% gate. +- **Honest timeout semantics.** Quorum not confirmed within + `replication.quorum_timeout_ms` (default 2000) → a **retryable 503** naming + the laggard regions, the commit index, and needed/confirmed counts. The + write is in the leader's log and may still commit: retries are + at-least-once (items/embeddings retries are idempotent upserts; signal + retries can double-count — decided in-phase: no idempotency-key machinery, + documented in runbook §8 with the session-write precedent for callers that + need exact-once). +- **`x-tidal-seq`** on every cluster write response: the write's seqno in the + replicated log (relayed through forwards) — an exact durability cursor + against `commit_index` in `/cluster/status/local` (which also gains `ack`). + `tidaldb_cluster_relay_durable_seq` now reports the commit index + (`relay_last_seq − relay_durable_seq` = quorum lag); new counter + `tidaldb_cluster_quorum_timeouts_total`. +- **The ledger gate (exit gate, run for real):** tier-3 + `mp_quorum_ledger_zero_acked_loss_across_killpoints` SIGKILLs the leader + under concurrent quorum load and proves zero acknowledged loss on the + promoted max-applied survivor — frontier invariant (max acked seq ≤ + survivor applied) plus per-item content probes. **167/167 kill points + passed on the final design** (batches of 64 + 91 + 12; kill timings spread + 120–598ms across fresh 3-process clusters; an earlier 100/100 run had + validated the superseded ack-holding design before it was replaced — see + docs/planning/milestone-11/phase-3.md). Plus tier-3 partition semantics + (one follower down: quorum commits; both down: fast 503 naming laggards + while `ack=leader` flows; heal: recovers) and in-process gRPC coverage + (override headers, forwarded quorum writes, blob writes, 400 on bad mode). +- **Rolling-upgrade order (mixed-version caveat):** a pre-m11p3 leader + neither serves the `ReportApplied` RPC nor recognizes `x-tidal-ack` — it + silently applies LEADER-ack semantics to a `quorum` request (a durability + downgrade the caller cannot see). Upgrade the leader first: `ack=quorum` + is then honored immediately (commit-index freshness rides the m11p2 + ship-ack floor hints until the followers upgrade too). Replication and + heal are unaffected by either order. Runbook §8 records the procedure. + **One replicated log (m11p2) — items/embeddings ride the WAL, `StreamSegments` catch-up, HTTP broadcast deleted** - **The leader's WAL is now THE replicated log.** The group-commit writer hands every fsynced batch to a bounded in-memory **ship feed** (`wal::feed::WalShipFeed`); diff --git a/docs/ops/monitoring.md b/docs/ops/monitoring.md index 00488e7..d3e5801 100644 --- a/docs/ops/monitoring.md +++ b/docs/ops/monitoring.md @@ -143,7 +143,8 @@ multi-process cluster mode the listener binds the topology's per-region | `tidaldb_cluster_write_pool_depth` | gauge | count | Queued cluster write jobs awaiting a pool worker. | | `tidaldb_cluster_write_pool_rejections_total` | counter | count | Write submissions shed with backpressure (HTTP 429). | | `tidaldb_cluster_relay_last_seq` | gauge | seqno | Leader stream high-water mark. Since m11p2 the stream is the WAL itself, so this reports the WAL flushed frontier. | -| `tidaldb_cluster_relay_durable_seq` | gauge | seqno | Leader durable (fsynced) frontier. Since m11p2 only fsynced batches enter the stream, so this equals `relay_last_seq` by construction (the pair is kept for dashboard continuity; m11p3 repurposes the gap for quorum lag). | +| `tidaldb_cluster_relay_durable_seq` | gauge | seqno | **The quorum commit index** (m11p3): highest seqno a majority of the replica set durably holds. `relay_last_seq − relay_durable_seq` is the cluster's quorum lag. | +| `tidaldb_cluster_quorum_timeouts_total` | counter | writes | `ack=quorum` writes that timed out awaiting the commit index (each returned a retryable 503 naming the laggards). | | `tidaldb_cluster_peer_acked_seqno` | gauge | seqno | Per peer (`peer_shard` label): contiguous frontier accepted by the peer's transport. | | `tidaldb_cluster_peer_ship_queue_depth` | gauge | events | Per peer: `relay_last_seq − acked` — flushed events not yet accepted (or self-reported applied) by this peer. | | `tidaldb_cluster_peer_ship_batches_total` | counter | count | Per peer: batches shipped. | @@ -166,7 +167,8 @@ multi-process cluster mode the listener binds the topology's per-region | High Rate Limiting | `rate(tidaldb_rate_limited_total[5m]) > 100` | Info | Sustained rate limiting. Review agent rate limit configuration or reduce write volume. | | Tantivy Segment Bloat | `tidaldb_tantivy_segment_count > 30` | Warning | Tantivy has many unmerged segments. Text syncer may be stalled. | | Cluster Peer Ship Stall | `deriv(tidaldb_cluster_peer_acked_seqno[2m]) == 0 AND tidaldb_cluster_peer_ship_queue_depth > 0` | Critical | A peer stopped accepting batches while events queue behind it (partition, dead peer, or paused sender). Check `/cluster/status` and heal. | -| Cluster Durable Frontier Stall | `tidaldb_cluster_relay_last_seq - tidaldb_cluster_relay_durable_seq > 10000` | Critical | m11p1-era alert; since m11p2 the two gauges are equal by construction, so any sustained gap means a metrics-pipeline fault. Keep until m11p3 repurposes the pair for quorum lag. | +| Cluster Quorum Lag | `tidaldb_cluster_relay_last_seq - tidaldb_cluster_relay_durable_seq > 10000` | Critical | A majority of the replica set is not confirming durability (down/partitioned followers, or follower apply throughput exhausted). `ack=quorum` writes will 503; the bodies name the laggards. | +| Quorum Timeouts | `rate(tidaldb_cluster_quorum_timeouts_total[5m]) > 1` | Warning | `ack=quorum` writes are timing out (retryable 503s). Sustained timeouts = a laggard region or an over-budget `replication.quorum_timeout_ms` for the deployment's RTT. Elevated timeouts while `relay_durable_seq` (the commit index) holds steady and followers report healthy = the `ReportApplied` frontier pushes are being lost (packet loss / a pre-m11p3 leader) — followers WARN `applied-frontier report failed` on the first failure of a streak. | | Cluster Write Shedding | `rate(tidaldb_cluster_write_pool_rejections_total[5m]) > 100` | Warning | Sustained 429 shedding on the cluster write path. Raise `write_workers`/capacity or reduce offered write rate. | ### Grafana Dashboard Suggestions diff --git a/docs/planning/ROADMAP.md b/docs/planning/ROADMAP.md index 97989fb..ff5f26b 100644 --- a/docs/planning/ROADMAP.md +++ b/docs/planning/ROADMAP.md @@ -37,7 +37,7 @@ A single embeddable database can replace the 6-system content ranking stack by t | M8 | Distributed Fabric | Multi-region, multi-tenant replication keeps agent-memory semantics intact | Hosted tidalDB, cloud/edge deployments, shared agent substrate — **✅ COMPLETE**: in-process primitives + multi-node replication over real gRPC + true multi-process cluster mode (one process per region, real process isolation) with full tier-3 UAT (partition injection via TCP-proxy, clock-skew, rolling-upgrade, runbook verification); G1 + G2 resolved. Post-M8 follow-ups: quorum-ack writes, automatic failure detection / leader election | | M9 | Community Sync & Revocation | Local embeddable profiles can opt into community personalization and safely leave/purge contributions | Community personalization, federated taste graphs, shared feeds — ✅ COMPLETE (2026-06-06) | | M10 | Governance & Agent Rights | Community rules and agent-scoped permissions control what signals influence ranking | User-owned AI personalization at scale, policy-compliant agents — ✅ COMPLETE (2026-06-06) | -| M11 | Enterprise-Grade Cluster | The multi-process cluster becomes a system of record: fast unified log, quorum-durable, self-healing, horizontally scalable, secured, observable, chaos-tested | Paying-customer cluster deployments — **IN PROGRESS**: m11p1 (replication perf floor) ✅ + m11p2 (one replicated log: items/embeddings on the WAL, `StreamSegments` catch-up, HTTP broadcast deleted) ✅ COMPLETE 2026-06-11; p3–p9 planned in [docs/roadmap-to-cluster.md](../roadmap-to-cluster.md) | +| M11 | Enterprise-Grade Cluster | The multi-process cluster becomes a system of record: fast unified log, quorum-durable, self-healing, horizontally scalable, secured, observable, chaos-tested | Paying-customer cluster deployments — **IN PROGRESS**: m11p1 (replication perf floor) ✅ + m11p2 (one replicated log) ✅ + m11p3 (quorum-acked writes: `ack=leader\|quorum`, commit index, zero-acked-loss ledger gate — closes G4) ✅ COMPLETE 2026-06-11; p4–p9 planned in [docs/roadmap-to-cluster.md](../roadmap-to-cluster.md) | ### Embeddable → Distributed Path @@ -2910,7 +2910,7 @@ These phases take the proven in-process primitives and deliver actual multi-node | Gap | Origin | Severity | Description | |-----|--------|----------|-------------| -| G4 | post-M8 | Medium | **Quorum-ack write contract.** A `204` from `/signals` (and the data writes) asserts leader durability only (storage + WAL fsync); the follower ship is best-effort and no follower acknowledgement is asserted. A write contract that blocks for N-follower acks is future work — explicitly NOT part of m8p10. (Runbook §8 documents the current leader-durable contract.) | +| G4 | ✅ closed by m11p3 (2026-06-11) | Medium | **Quorum-ack write contract.** ~~A `204` from `/signals` (and the data writes) asserts leader durability only~~ — `ack=quorum` (topology `replication.ack` default or per-request `x-tidal-ack`) now gates success on a majority of the replica set durably holding the write, with retryable-503 timeout semantics and a 100/100-kill-point zero-acked-loss ledger gate. Runbook §8 documents the knob and its cost. | | G5 | post-M8 | Medium | **Automatic failure detection / leader election.** Leadership is operator-driven: `/cluster/promote` moves the leader and fans the view out to peers; there is no automatic failure detector and no automatic election. "Survive a machine dying" is a runbook step (detect → promote), not an automatic failover. (Runbook §9 documents the operator drill.) | | G6 | post-M8 | Low | **Embedding validation surfaces as HTTP 500.** A strict-dimension mismatch or zero-norm vector on `/embeddings` is correctly rejected, but the engine wraps the `VectorError` as an internal error, so the HTTP status is 500 rather than 400. The write is safely rejected (no corruption); only the status code is imprecise. (Runbook §5 documents the 500.) | @@ -3279,7 +3279,7 @@ Full gap analysis, measured baselines, phase specs, and exit gates live in |-------|------|--------| | m11p1 | Replication performance floor: decouple ack from ship, batch + pipeline per-peer shipping, follower group-commit coalescing, group-commit knobs, first `tidaldb_cluster_*` metrics + cluster `/metrics` listener | ✅ **COMPLETE (2026-06-11)** — 4,534 replicated signal-writes/s sustained within SLO on a 3-process localhost cluster (release build; ~50× the ~90/s baseline), replication lag ≤377 events (~80ms) through the full ramp, errors <1%. Leader fsync profiled (`tidaldb_cluster_wal_fsync_us`): macOS F_FULLFSYNC ≈7.4ms mean explains the local write-p99 tail; the ≤50ms p99 sub-gate validates on Linux fdatasync (Ref-A) where the fsync floor is far lower. Ref-A re-run pending infra access. | | m11p2 | One replicated log: items/embeddings ride the WAL, delete the HTTP broadcast side channel, implement `StreamSegments` catch-up | ✅ **COMPLETE (2026-06-11)** — the leader's WAL is THE replicated log (stream seqnos = WAL seqnos, restart-surviving); items/embeddings journal as kind-1/2 blob records on the same stream as signals; catch-up is follower-pulled `StreamSegments` over the durable segments (gap-triggered, boot-time, and heal-nudged); the m8p10 HTTP item broadcast + O(items) heal backfill are deleted (both 2026-06-10 bug classes impossible by construction); promote carries a persisted stream baseline. Tier-3 `mp_items_ride_the_log_and_catchup_stream` proves items via leader + follower gateways converge everywhere and a restarted follower self-heals via the stream with no operator verb. See [milestone-11/phase-2.md](milestone-11/phase-2.md). | -| m11p3 | Quorum-acked writes (`ack=leader\|quorum`, commit index, no-acked-loss ledger checker) — closes **G4** | Planned | +| m11p3 | Quorum-acked writes: `ack=leader\|quorum` (topology default + `x-tidal-ack` override), durable-frontier reports, commit index, retryable-503 timeout semantics, ledger checker | ✅ **COMPLETE (2026-06-11)** — followers push durably-applied frontiers (`ReportApplied`, batch-level, decoupled from ship acks); commit index = k-th largest durable mark, leadership-scoped; `ack=quorum` writes await it asynchronously and 503 with laggard names on budget expiry; every cluster write returns `x-tidal-seq`. Exit gate: 100/100 leader-SIGKILL kill-points with ZERO acknowledged loss (frontier + content proofs on the promoted max-applied survivor); quorum throughput 3,600 signal-writes/s within SLO locally (79% of p1's 4,534/s leader-ack figure, gate ≥50%). Also fixed en route: follower blob applies group-committed (22× item-seed speedup), async quorum waits (thread-per-wait collapsed at 1k rps). See [milestone-11/phase-3.md](milestone-11/phase-3.md). | | m11p4 | Failure detection, election, term fencing — closes **G5** | Planned | | m11p5 | Membership, discovery, elasticity (DNS, seed join, snapshot + stream catch-up) | Planned | | m11p6 | Sharding × replication + rebalancing | Planned | diff --git a/docs/planning/milestone-11/phase-3.md b/docs/planning/milestone-11/phase-3.md new file mode 100644 index 0000000..5c72d56 --- /dev/null +++ b/docs/planning/milestone-11/phase-3.md @@ -0,0 +1,172 @@ +# m11p3 — Quorum-Acked Writes (COMPLETE — 2026-06-11) + +Phase spec and exit gate: [docs/roadmap-to-cluster.md §4/m11p3](../../roadmap-to-cluster.md). +Closes ROADMAP gap **G4** ("no quorum durability — a leader-acked write can +die with the leader"). +Predecessors: [phase-1.md](phase-1.md) (ship queue), [phase-2.md](phase-2.md) +(one replicated log; stream seqnos = WAL seqnos). + +**Goal:** an opt-in durability contract a system of record can sit on: +`ack=quorum` writes succeed only once a **majority of the replica set +durably holds them**, with pipelined batch-level acks, honest timeout +semantics (retryable 503 naming the laggards), and a ledger checker proving +zero acknowledged loss under leader SIGKILL. + +## Design (as adopted) + +### 1. Durable frontier reports (the foundation) + +m11p2's ship ack piggybacked the follower's applied seqno read **before** +enqueue — a monotonic floor. The key observation: that floor is already +**durable-true** (a follower's frontier provably advances only after storage +upserts AND its own WAL group-commit fsync — `apply_replicated_events` waits +every staged append before any in-memory fold; blob applies are +engine-WAL-first). What m11p3 adds is **freshness**, pushed the other way: + +- The segment receiver calls the new `Transport::notify_applied(shard, + applied)` once per **apply round**; the gRPC transport fires a + `ReportApplied` RPC (fire-and-forget, deduped on unchanged frontiers) at + the stream's source. One report covers a whole pipelined window of + segments — batch-level, never per-write. +- The leader's report handler folds the mark into its hint map AND the + quorum commit index (via a late-bound `AppliedSink` — the ship queue that + owns the index is built after the transport). +- Ship acks stay instant (m11p2 semantics). Reports flow even when ships + DON'T: a follower converging by catch-up pull, a quiet leader, a healed + peer — all keep the commit index honest with zero leader work. + +**The design that did NOT ship** (recorded because the failure is +instructive): holding each ship ack open until its segment's durable apply. +Functionally correct, and it passed every targeted test — but under +open-loop load it couples the leader's ship cadence to the follower's apply +latency: one gap-parked follower throttles its own feed (window × ack-wait), +falls further behind, and the cluster spirals into total quorum collapse +(measured: 100% write failure at 1,000 rps). The report push removes the +coupling entirely. + +### 2. Commit index = k-th largest durable mark + +`CommitIndex` (engine, `replication/commit.rs`) tracks per-peer durable +marks fed by frontier reports (`AppliedSink`), ship-ack hint folds +(`record_success`), and heal resumes (the follower's own status report) — +all three durable-true. With `n = peers + 1` replicas, +a seqno is **committed** once `floor(n/2)` peers report marks at or past it +(the leader is the remaining majority member; nothing unfsynced can ship by +construction). The index is leadership-scoped and owned by the ship queue: +`activate_from(baseline)` resets marks to the promote baseline, +`deactivate()`/`shutdown()` fail every waiter with `Demoted`, and an +epoch counter catches activate→deactivate→activate cycles mid-wait. +`PeerState.acked` (transport-accept, retry pruning) is deliberately NOT a +quorum input. + +### 3. `ack=leader|quorum` on the write path + +- **Deployment default**: topology `replication.ack` (default `leader` — + the m0–m11p2 contract, byte-for-byte). **Per-request override**: + `x-tidal-ack` header (invalid value → 400), forwarded verbatim by + follower gateways so the leader honors the CALLER's choice. +- `/signals`, `/items`, `/embeddings` (the replicated mutations) gate + `ack=quorum` AFTER leader durability by awaiting a **watch-channel mirror** + of the commit index (one bridge thread publishes; any number of handlers + await for free). Fully async by necessity, not taste: the first + implementation parked one blocking-pool thread per waiter, and at 1k rps + open-loop the 512-thread pool filled with parked waiters while the + completions that advance the index queued behind them — total collapse. + Budget: `replication.quorum_timeout_ms` (default 2000 — the cross-region + SLO). +- Every cluster write's response now carries **`x-tidal-seq`** (its + replicated-log seqno; relayed through forwards), giving clients an exact + durability cursor — and giving the ledger checker its ledger. +- Timeout → **retryable 503** naming the laggard regions, the commit index, + and the confirmed count. The write IS in the leader's log and MAY still + commit: retries are at-least-once (see the idempotency decision below). + Demotion mid-wait → the NotLeader 503 (never a false quorum claim). +- Hard negatives stay CRDT (per-user convergent data, `/cluster/reconcile`) + — the ack knob does not apply. The `/sharded/*` surface keeps leader-ack + semantics this phase (quorum × sharding lands with m11p6). + +### 4. Follower blob applies are group-committed (found by the gate) + +The quorum throughput gate exposed an m11p2 flaw invisible to convergence +tests: replicated items/embeddings applied ONE record at a time — each +paying a solo follower WAL fsync (`wal_blob_first` waits per record, the +writer synced per blob batch). Item apply capped at the fsync floor (~100/s +on macOS `F_FULLFSYNC`), so any item burst left followers with a +multi-second frontier stall that every subsequent `ack=quorum` write then +measured honestly as a 503. Two-sided fix, mirroring m11p1's signal +coalescing: `ReplicatedBlobApplier` became batched (`apply_blobs`: the +engine validates ALL records, stages ALL WAL appends, waits all — shared +group syncs — then upserts storage in order), and the WAL writer flushes +its queued blobs under ONE group fsync per drain window. Corpus seeding of +2,000 items + embeddings: 39.3s → **1.8s** (22×). + +### 5. Observability + +- `tidaldb_cluster_relay_durable_seq` is repurposed: it now reports the + **commit index** (the m11p2 value it replaced had become equal to + `relay_last_seq` by construction). `relay_last_seq − relay_durable_seq` + is the cluster's quorum lag. +- New counter `tidaldb_cluster_quorum_timeouts_total`. +- `/cluster/status/local` gains `commit_index` and `ack` (the node's + deployment default). + +### Idempotency open question — DECIDED (in-phase, per the roadmap) + +**No new idempotency-key machinery in m11p3.** Findings: the WAL's +content-hash dedup window cannot dedup client retries (the server stamps +`Timestamp::now()` per request, so retried bodies hash differently), and +even when it fires it is log-only (the in-memory aggregate still folds). +Items/embeddings retries are idempotent upserts — retrying a quorum-timeout +503 on `/items`//`/embeddings` is always safe. Signal retries are +at-least-once: a retried 503 whose original DID commit double-counts that +signal's weight — bounded, decays, and self-corrects relative to exact +counting over time. Documented in runbook §8; client-supplied idempotency +keys ride on the session-write precedent (`WalCommand::SessionSignal` +already carries one) if a future phase needs exact-once accounting. + +## Exit gate (from the roadmap) + +- SIGKILL the leader under `ack=quorum` load → ledger checker proves zero + acknowledged loss across 100 random kill-points. +- Quorum throughput ≥ 50% of p1's leader-ack number (≥1,000/s Ref-A). + +## Status + +- [x] Durable frontier reports (`ReportApplied`, `notify_applied`, `AppliedSink`) +- [x] Follower blob group-commit (`apply_blobs` + writer group fsync) +- [x] `CommitIndex` (k-th-largest marks, leadership-scoped, laggard naming) +- [x] Ship-queue wiring (durable-mark folds, activate/deactivate/shutdown) +- [x] Seqno surfacing (`x-tidal-seq` through engine → handlers → forwards) +- [x] `ack=leader|quorum`: topology default + header override + 503 body +- [x] Metrics + status surfaces +- [x] In-process gRPC tests (gate/503/heal; forwarded quorum; blob writes) +- [x] Tier-3 suite: partition semantics + the ledger checker +- [x] 100-kill-point gate run + throughput run (recorded below) +- [x] Docs (runbook §8 rewrite, topology table, monitoring) + CHANGELOG + +## Exit-gate evidence (local; release builds, real OS processes) + +| Gate | Target | Measured | +|------|--------|----------| +| Zero acknowledged loss under leader SIGKILL | 100 random kill-points | **167/167 kill-points, zero acked loss** on the final design (batches of 64 + 91 + 12; each round = fresh 3-process cluster, concurrent `ack=quorum` item+signal writers via leader AND follower gateways, SIGKILL at timings spread 120–598ms, then BOTH proofs on the promoted max-applied survivor: `max(acked seq) <= survivor applied_events` and per-item `/search` content probes). An earlier 100/100 run validated the superseded ack-holding design before it was replaced. | +| Quorum throughput ≥ 50% of p1's leader-ack number | ≥ 2,267/s local-equivalent | **3,600 quorum signal-writes/s within SLO** (writes mix, `wal.batch_timeout_ms: 2`, knee not reached; p50 ~45ms at 1.5k, ~370ms at 3.6k from concurrency-limit queueing; zero errors; replication lag ≤3 events at ramp end) = **79%** of p1's 4,534/s. Same-day leader-ack baseline on this machine: 5,971/s (10ms batch timeout). Ref-A (Linux) runs remain blocked on k3s access — same as p1/p2. | + +Tier-3 partition semantics (real TCP severs): one follower down → quorum +commits via the other; both down → fast 503 naming the laggards while +`ack=leader` writes flow uninterrupted; healed → quorum recovers within the +breaker window. (`mp_quorum_writes_gate_and_recover_under_partition`.) + +### Performance findings the gate forced (all shipped) + +1. **Thread-per-quorum-wait collapses at 1k rps**: parked Condvar waiters + exhausted tokio's 512-thread blocking pool; the group-commit completions + that advance the index queued BEHIND them → 0 OK writes, 40s latencies. + Fix: async watch-channel bridge (one publisher thread, zero per-waiter + threads). +2. **Holding ship acks until durable apply death-spirals**: ack-wait × + window throttles ships to a gap-parked follower → it lags further → all + quorum writes time out. Fix: follower-pushed `ReportApplied`, ship acks + stay instant. +3. **Per-record blob fsyncs cap item apply at ~100/s** (m11p2 flaw): fixed + with batched blob apply + writer group fsync → seeding 22× faster, item + bursts no longer stall the quorum frontier. diff --git a/docs/roadmap-to-cluster.md b/docs/roadmap-to-cluster.md index e0a9160..2dae772 100644 --- a/docs/roadmap-to-cluster.md +++ b/docs/roadmap-to-cluster.md @@ -1,6 +1,6 @@ # Roadmap to an Enterprise-Grade Cluster -**Status:** ADOPTED as M11 — m11p1 ✅, m11p2 ✅ complete (2026-06-11), p2–p9 planned · **Date:** 2026-06-10 · **Baseline evidence:** +**Status:** ADOPTED as M11 — m11p1 ✅, m11p2 ✅, m11p3 ✅ complete (2026-06-11), p2–p9 planned · **Date:** 2026-06-10 · **Baseline evidence:** [stress-test-thepeach.md](ops/stress-test-thepeach.md), [cluster runbook](runbooks/cluster.md), [ROADMAP M8 Known Gaps](planning/ROADMAP.md) (G4/G5/G6), live k3s deployment (3 regions × 2-vCPU pods). @@ -252,6 +252,21 @@ channel (items/embeddings) is deleted. proves zero acknowledged loss across 100 random kill-points; quorum throughput ≥50% of p1's leader-ack number (≥1,000/s Ref-A). +> **✅ COMPLETE (2026-06-11), as built:** durable freshness is follower-PUSHED +> (`ReportApplied` once per apply round — decoupled from ship acks after the +> ack-holding design measurably death-spiraled under load) into a +> leadership-scoped commit index (k-th largest durable mark); `ack=quorum` +> waits are fully async (watch-channel bridge — thread-per-wait exhausted +> the blocking pool); timeouts are retryable 503s naming laggards; every +> cluster write returns `x-tidal-seq`. The gate also surfaced and fixed an +> m11p2 flaw: follower blob applies paid one fsync per item — now batched +> through shared group commits (22× item-seed speedup). Idempotency +> question DECIDED: no new machinery; at-least-once + dedup guidance in +> runbook §8. Evidence: 100/100 kill-points zero acked loss; 3,600 quorum +> writes/s within SLO locally = 79% of p1's leader-ack figure (Ref-A run +> still blocked on k3s access). Details: +> [planning/milestone-11/phase-3.md](planning/milestone-11/phase-3.md). + ### m11p4 — Failure detection, election, fencing (size: XL) — closes ROADMAP gap **G5** **Goal:** "a machine died" is a non-event, not a runbook page. diff --git a/docs/runbooks/cluster.md b/docs/runbooks/cluster.md index 73f22f6..0a19fb3 100644 --- a/docs/runbooks/cluster.md +++ b/docs/runbooks/cluster.md @@ -30,22 +30,24 @@ write-durability contract. > orchestration. > > **Honest remaining limits (both modes):** -> * **Writes are leader-durable, NOT quorum-acked.** A `204` means the leader -> durably applied the write (storage + WAL fsync). The follower ship is -> best-effort; no follower acknowledgement is asserted. A quorum-ack write -> contract is **post-M8 follow-up** (see [§8](#8-write-durability-contract-read-this-before-trusting-a-204)). +> * **Quorum durability is opt-in.** The default `204` is leader-durable +> (storage + WAL fsync; follower ship off the request path). Since m11p3, +> `ack=quorum` — topology default or per-request `x-tidal-ack` header — +> gates success on a **majority of the replica set durably holding the +> write**, surviving permanent leader loss (see +> [§8](#8-write-durability-contract-the-ack-knob-and-its-cost-m11p3)). > * **Leadership is operator-driven, not automatic.** There is **no automatic > failure detector and no automatic leader election.** `/cluster/promote` moves > leadership and fans the new view out to peers; a node that misses the fan-out > self-corrects on its next forwarded write / status poll. "Survive a machine -> dying" is an operator runbook step (detect → promote), not an automatic -> failover. +> dying" is an operator runbook step (detect → promote the max-applied +> survivor), not an automatic failover (elections are m11p4). > > **For a production deployment today**, run a **single `tidal-server standalone`** > node backed by host-level redundancy and disk durability (see > [kubernetes.md](kubernetes.md) and [server-deployment.md](../guides/server-deployment.md)), > and reach for multi-process cluster mode for read-scale / multi-region -> experiments — not as a quorum-HA story. +> deployments whose writes need `ack=quorum`'s failover-survivable contract. > > **Both modes refuse to start** unless you explicitly opt in with either the > `--experimental-cluster` flag or the `TIDAL_ALLOW_EXPERIMENTAL_CLUSTER=1` @@ -255,6 +257,8 @@ Fields: | `replication.batch_max_events` | unused | optional | Max relay events coalesced into one shipped batch (1–256, the WAL wire-format ceiling). Default 256. | | `replication.window` | unused | optional | In-flight batches per peer (1–64). 1 = strictly in-order shipping; higher pipelines across the peer RTT (out-of-order arrivals park gap-aware on the receiver). Default 4. | | `replication.retry_ms` | unused | optional | Backoff (ms) before a transiently-failed batch ship retries. Default 100. | +| `replication.ack` | unused | optional | Deployment-default write acknowledgment: `leader` (default) or `quorum` (majority-durable — see [§8](#8-write-durability-contract-the-ack-knob-and-its-cost-m11p3)). Per-request override: the `x-tidal-ack` header. | +| `replication.quorum_timeout_ms` | unused | optional | Budget an `ack=quorum` write waits for the commit index before the retryable 503 naming the laggards. Default 2000. | | `wal.batch_size` | optional | optional | Events per WAL group-commit fsync (1–256). Default 100. | | `wal.batch_timeout_ms` | optional | optional | Max ms a partial group-commit batch waits before flushing. Default 10. Tune against the measured `tidaldb_cluster_wal_fsync_us` on the deployment's volume. | @@ -360,11 +364,16 @@ curl -X POST "$BASE/embeddings" \ # → 204 No Content ``` -The `201`/`204` asserts **leader durability**: the record is fsynced into the -stream every follower receives (live push, or pull-based catch-up after -downtime — see [§6 heal](#6-cluster-management-api)). A write to a non-leader -forwards to the leader transparently. A down/partitioned peer needs no -backfill bookkeeping: it converges from the log when it returns. +The `201`/`204` asserts **leader durability** and carries **`x-tidal-seq`** +(the record's seqno in the replicated log); `x-tidal-ack: quorum` upgrades it +to a majority-durable ack (see +[§8](#8-write-durability-contract-the-ack-knob-and-its-cost-m11p3)). The +record is fsynced into the stream every follower receives (live push, or +pull-based catch-up after downtime — see +[§6 heal](#6-cluster-management-api)). A write to a non-leader forwards to +the leader transparently (the ack header and seq header travel through the +forward). A down/partitioned peer needs no backfill bookkeeping: it +converges from the log when it returns. Embeddings: tidalDB does **not** generate vectors — the caller brings them. The write L2-normalizes and inserts into the HNSW index. Dimensions are **strict**: @@ -385,7 +394,10 @@ curl -X POST "$BASE/signals" \ ``` The signal name must be declared in the schema (an undeclared name returns **400** -naming it). This route records a **global** signal on the leader and ships it to +naming it). The 204 carries **`x-tidal-seq`** (the write's replicated-log seqno), +and `x-tidal-ack: quorum` upgrades it to a majority-durable ack — see +[§8](#8-write-durability-contract-the-ack-knob-and-its-cost-m11p3). This route +records a **global** signal on the leader and ships it to followers over the replicated WAL stream; it does not personalize (see the personalization note in [§3](#3-topology-yaml)). On a non-leader gateway it forwards to the leader transparently and still returns `204`. @@ -466,7 +478,10 @@ curl "$BASE/cluster/status" | jq ``` `relay_log_len` is the leader's high-water-mark (`last_seq`); each region's -`lag_events` is `relay_log_len − applied_events` (saturating). A region the +`lag_events` is `relay_log_len − applied_events` (saturating). Since m11p3 +`/cluster/status/local` also reports the node's `ack` default and, on the +leader, `commit_index` — the highest seqno a majority of the replica set +durably holds (`last_seq − commit_index` is the quorum lag). A region the gateway **cannot reach** within the per-peer budget is reported honestly as `reachable: false`, `partitioned: true`, `applied_events: 0`, and worst-case lag (`lag_events == relay_log_len`). A non-zero, *growing* lag on a reachable region is @@ -613,54 +628,93 @@ shards' results. The merge dedups replicated copies of an entity (keeping the best-scoring copy), reconciles `total_candidates` so replicated shards are not counted multiple times, and re-enforces `max_per_creator` across the merged set. -## 8. Write-durability contract (read this before trusting a 204) +## 8. Write-durability contract: the ack knob and its cost (m11p3) -A `204 No Content` from `/signals` (and the data writes' success codes) means -**the write is durably applied on the leader** — storage updated plus **WAL -fsync** — and nothing more. +Every replicated write (`/signals`, `/items`, `/embeddings`) runs under one of +two acknowledgment modes. Pick the deployment default with the topology's +`replication.ack`; any caller overrides per request with the **`x-tidal-ack` +header** (`leader` or `quorum`; anything else is a 400). -Since m11p2 there is **one replicated log**: the leader's WAL. Every -replicated mutation — signals (kind-0 batches) AND items/embeddings (kind-1/2 -blob records, journaled BEFORE storage) — rides it, and what ships to peers -is the WAL's own fsynced batches, byte-identical on disk and on the wire. +| | `ack=leader` (default) | `ack=quorum` | +|---|---|---| +| Success means | Durable on the **leader** (storage + WAL group-commit fsync) | Durable on a **majority of the replica set** (leader + `floor(n/2)` followers, each storage-applied + own-WAL-fsynced) | +| Survives | Any follower failure; leader restart (WAL replay) | **Any single node's permanent loss, including the leader's** — promote the max-applied survivor and every acked write is there (the 167-kill-point ledger gate proves it) | +| Does NOT survive | Permanent leader loss before followers caught up (the un-shipped tail dies with it) | Simultaneous majority loss | +| Latency cost | One group-commit fsync (~ms-scale; the macOS F_FULLFSYNC tail is the local floor) | + one ship RTT + the follower's group-commit fsync, **pipelined**: acks are batch-level, so concurrent writers share the round trip the same way they share fsyncs | +| Failure mode | 5xx only for local faults | Additionally a **retryable 503** when the quorum budget (`replication.quorum_timeout_ms`, default 2000) expires — the JSON body names the `laggards`, the `commit_index`, and `needed`/`confirmed` counts | +| Availability | Unaffected by follower outages | **Blocks when a majority is unreachable.** In a 2-region cluster quorum = leader + THE follower: one follower outage stops all quorum writes (by design — that is what the contract says). 3+ regions tolerate `floor((n-1)/2)` follower outages | -- The success code returns at **leader group-commit fsync**. Follower - shipping happens entirely off the request path on per-peer sender threads - that push the WAL flush feed's tail (`replication.batch_max_events` per - run, `replication.window` runs in flight per peer). Concurrent writers - share group-commit fsyncs instead of serializing one solo fsync each — - this is what lifted the replicated ceiling from ~90/s to thousands/s - (m11p1). -- Peers only ever receive **fsynced batches** by construction: nothing enters - the ship feed before its fsync completes, so an event a follower holds but - the leader could lose is impossible. -- If a write's fsync **fails**, that write errors (the WAL writer notifies - every waiting caller and keeps serving; the failed seqno range is reused on - retry) — nothing unfsynced can ship. Repeated fsync failures are a dying - volume: restart the node or promote a survivor. -- The follower ship is **best-effort**. A ship that fails is parked and retried - by its sender every `replication.retry_ms` (first failure and every 50th log - at WARN with the running count; recovery logs at INFO). Data that rotates - out of the in-memory ship tail during a long outage is **follower-pulled** - via the `StreamSegments` catch-up stream over the durable segments. The - success code does **NOT** assert quorum and does **NOT** assert any - follower acknowledged the write (quorum acks are m11p3). -- In **multi-process** mode, if the leader's process crashes after the 204 but - before followers caught up, the leader's WAL still has the write (it survives - restart from disk); the followers reconcile on the next ship/heal. The other - regions' processes keep serving — this is real process isolation, but it is - **not** an automatic failover (an operator promotes a survivor — see - [§9](#9-failover-drill)). -- In **single-process** mode there is only one process, so a crash is total - downtime, not a failover. +Mechanics, in one paragraph: there is **one replicated log** (the leader's +WAL — m11p2); peers only ever receive **fsynced batches** by construction. +Since m11p3 every follower **pushes its durably-applied frontier** back to +the leader once per apply round (`ReportApplied` — batch-level, decoupled +from ship acks, flowing even when the follower converges by catch-up pull or +the leader is quiet); ship acks additionally carry the same frontier as an +instant floor hint. The leader folds both into per-peer durable marks; the +**commit index** is the k-th largest mark (k = `floor(n/2)`), and +`ack=quorum` responses gate on it passing the write's seqno — awaited +asynchronously, so quorum waiters never hold threads or starve completions. +The index is leadership-scoped: promote resets it to the new stream +baseline, and a demoted leader fails its in-flight quorum waiters (it must +never claim quorum for a stream it no longer owns). -A **quorum-ack write contract** (a 204 that asserts N followers acknowledged) is -explicitly **post-M8 follow-up** work, tracked in -[docs/planning/ROADMAP.md](../planning/ROADMAP.md) M8 Known Gaps. It is **not** part -of m8p10. +Every cluster write's success response carries **`x-tidal-seq`** — the +write's seqno in the replicated log (relayed through gateway forwards). +Persist it if you need an exact durability cursor: `commit_index >= seq` on +`/cluster/status/local` is "this write is majority-durable", regardless of +which mode acked it. (The rare dedup-suppressed signal write — an identical +event within the WAL's ~60s content-hash window is already durably logged, +so no new record exists to track — carries **`x-tidal-deduplicated: 1`** +instead, relayed through forwards like the seq header.) -In short: **204 = leader durability, not cluster durability.** Design your client -retries accordingly (the writes are idempotent on `entity_id` + signal). +**Retry semantics under `ack=quorum` — read this twice.** A quorum-timeout +503 means *not confirmed in budget*, not *not written*: the write is in the +leader's log and usually commits moments later. Retries are therefore +**at-least-once**: + +- `/items` and `/embeddings` retries are **always safe** — idempotent + upserts keyed by `entity_id`. +- `/signals` retries can **double-count** the signal's weight when the + original did commit (the server stamps each request's timestamp, so the + WAL's content-hash dedup window cannot identify a client retry). The + distortion is one extra decaying signal per retried timeout — bounded by + your retry rate (`tidaldb_cluster_quorum_timeouts_total` is exactly that + budget). Accounting that cannot tolerate it should route through session + writes (which carry idempotency keys) or dedup client-side on its own key. +- The laggard names in the 503 are your runbook pointer: a persistent + laggard is a down/partitioned region — heal it ([§6](#6-cluster-management-api)) + or accept leader-ack for the duration (`x-tidal-ack: leader`). + +**Rolling upgrades into m11p3 — upgrade the leader first.** A pre-m11p3 +leader neither serves the `ReportApplied` RPC nor recognizes `x-tidal-ack`: +it silently applies **leader-ack semantics to a `quorum` request** — a +durability downgrade the caller cannot see. Upgrade order: + +1. Promote leadership off the leader node if needed, upgrade it, promote it + back (or simply upgrade the standing leader per [§9](#9-failover-drill)'s + restart procedure). From this moment `ack=quorum` is honored: the commit + index rides the m11p2 ship-ack floor hints from not-yet-upgraded + followers (correct, just laggier). +2. Upgrade followers one at a time. Each upgraded follower starts pushing + `ReportApplied` and quorum freshness returns to batch-level. (An upgraded + follower reporting to a still-old leader is harmless — the report is + refused and logged, replication and heal are unaffected.) + +Until step 1 completes, treat the cluster as `ack=leader`-only — do not +point `ack=quorum` traffic at it expecting majority durability. + +Other facts unchanged from m11p1/p2: the success code never waits on +shipping for `ack=leader` (sender threads push the WAL flush feed's tail off +the request path; `replication.batch_max_events`/`window` tune it); a failed +fsync errors that write and nothing unfsynced can ship; long-outage data is +follower-pulled via `StreamSegments`; a leader crash is **not** an automatic +failover (an operator promotes a survivor — see [§9](#9-failover-drill), and +under `ack=quorum` **promote the survivor with the highest `applied_events`** +— that rule is what makes the zero-acked-loss guarantee hold). + +In short: **`ack=leader` = leader durability. `ack=quorum` = failover-survivable +durability, priced at one pipelined replication round trip and majority +availability.** ## 9. Failover drill (multi-process) @@ -673,7 +727,12 @@ suite (`cluster_runbook.rs::runbook_s9_failover_drill`) executes it: (`?region=eu-west`) to confirm it is serving and roughly caught up. 3. **Promote.** `POST /cluster/promote { "region": "eu-west" }`. Confirm the `{ ok, leader, acked, failed }` response. If the OLD leader is dead, expect it - in `failed` — that is fine. + in `failed` — that is fine. **Under `ack=quorum`, promote the survivor with + the highest `applied_events`** (compare `/cluster/status/local` across + survivors): a quorum ack guarantees the write is on at least one follower's + contiguous frontier, so the max-applied survivor holds every acked write — + promoting any other node may discard acked data (m11p4's elections encode + this rule; until then it is the operator's). 4. **Verify.** `GET /cluster/status` now reports `eu-west` as leader. Send a write (`POST /signals`) to the new leader and confirm `relay_log_len` advances and the other regions' `applied_events` follow within a heartbeat. diff --git a/tidal-net/proto/wal_shipping.proto b/tidal-net/proto/wal_shipping.proto index 7589ea8..5e0b37b 100644 --- a/tidal-net/proto/wal_shipping.proto +++ b/tidal-net/proto/wal_shipping.proto @@ -62,6 +62,25 @@ message HeartbeatResponse { bool acknowledged = 1; } +// A follower's self-report of its durable frontier (m11p3). +// +// Pushed by the receiver once per apply round — fully decoupled from ship +// acks, so the leader's quorum commit index stays fresh even when its +// outbound ships stall (gap-parked follower, quiet leader, pull catch-up). +message AppliedReport { + // The reporting node's shard id. + uint32 reporter_shard = 1; + // The stream's source shard (the leader being reported to). + uint32 source_shard = 2; + // The reporter's contiguous durably-applied seqno for that stream. + uint64 applied_seqno = 3; +} + +// Applied-report acknowledgement. +message AppliedReportAck { + bool acknowledged = 1; +} + // WAL segment shipping service between tidalDB shards. service WalShipping { // Ship a single WAL segment to a peer shard (unary). @@ -72,4 +91,7 @@ service WalShipping { // Periodic health check for the ControlPlane. rpc Heartbeat(HeartbeatRequest) returns (HeartbeatResponse); + + // Follower -> leader durable-frontier report (m11p3 quorum acks). + rpc ReportApplied(AppliedReport) returns (AppliedReportAck); } diff --git a/tidal-net/src/client.rs b/tidal-net/src/client.rs index 85c1de3..1712d36 100644 --- a/tidal-net/src/client.rs +++ b/tidal-net/src/client.rs @@ -179,6 +179,39 @@ impl PeerPool { } } + /// Push this node's durable applied frontier to the stream's source peer + /// (m11p3 `ReportApplied`). Best-effort: respects an open circuit breaker + /// (skips, never trips it — reports are advisory and self-correcting). + /// + /// # Errors + /// + /// Unknown peer, open breaker, or transport failure — callers log at + /// debug and rely on the next advanced apply round. + pub async fn report_applied( + &self, + to: ShardId, + reporter: ShardId, + applied: u64, + ) -> Result<(), GrpcTransportError> { + let Some(peer) = self.peers.get(&to) else { + return Err(GrpcTransportError::PeerUnreachable(to)); + }; + if peer.circuit_breaker.check().is_err() { + return Err(GrpcTransportError::CircuitOpen(to)); + } + let mut client = peer.client.clone(); + let request = tonic::Request::new(crate::proto::AppliedReport { + reporter_shard: u32::from(reporter.0), + source_shard: u32::from(to.0), + applied_seqno: applied, + }); + client + .report_applied(request) + .await + .map(|_| ()) + .map_err(|status| GrpcTransportError::Grpc(Box::new(status))) + } + /// Open a `StreamSegments` catch-up stream from `shard` starting at /// `from_seqno` (m11p2: follower-pulled catch-up over the leader's /// durable WAL). diff --git a/tidal-net/src/server.rs b/tidal-net/src/server.rs index 7b5ceea..e5a9813 100644 --- a/tidal-net/src/server.rs +++ b/tidal-net/src/server.rs @@ -1,6 +1,7 @@ //! gRPC server implementing the `WalShipping` service. -use std::sync::Arc; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; use tidaldb::replication::{shard::ShardId, transport::WalSegmentPayload}; use tokio::sync::mpsc; @@ -9,14 +10,35 @@ use tonic::{Request, Response, Status}; use crate::{ config::GrpcTransportConfig, proto::{ - HeartbeatRequest, HeartbeatResponse, ShipSegmentRequest, ShipSegmentResponse, - StreamRequest, WalSegmentId, + AppliedReport, AppliedReportAck, HeartbeatRequest, HeartbeatResponse, ShipSegmentRequest, + ShipSegmentResponse, StreamRequest, WalSegmentId, wal_shipping_server::{WalShipping, WalShippingServer}, }, sources::ServingSources, tls, }; +/// Shared per-peer applied-hint map (transport + service): the monotonic max +/// of every ship-ack hint AND every `ReportApplied` push for a peer. Both +/// inputs are durable-true (a follower's frontier only advances after its +/// own storage apply + WAL fsync). +pub(crate) type PeerAppliedMap = Arc>>; + +/// Fold a durable mark into the shared hint map (monotonic max; 0 ignored). +pub(crate) fn fold_peer_applied(map: &PeerAppliedMap, peer: ShardId, applied: u64) { + if applied == 0 { + return; + } + let mut hints = map + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let entry = hints.entry(peer).or_insert(0); + if applied > *entry { + *entry = applied; + } + drop(hints); +} + /// Per-chunk caps for the `StreamSegments` catch-up path: small enough to /// stay far below the codec/payload ceilings, large enough that a 100k-item /// catch-up is a few hundred messages, not a few hundred thousand. @@ -29,7 +51,11 @@ pub struct WalShippingService { max_payload_bytes: usize, /// Node-side read views (m11p2): applied-seqno for ack piggybacking and /// WAL read-back for the catch-up stream. Absent = pre-m11p2 behavior. + /// m11p3 adds the `applied_sink` hook for follower frontier reports. sources: ServingSources, + /// Per-peer durable marks shared with the transport (m11p3): folded from + /// `ReportApplied` pushes here and from ship acks on the client side. + peer_applied: PeerAppliedMap, } impl WalShippingService { @@ -39,11 +65,13 @@ impl WalShippingService { inbound_tx: mpsc::Sender, max_payload_bytes: usize, sources: ServingSources, + peer_applied: PeerAppliedMap, ) -> Self { Self { inbound_tx, max_payload_bytes, sources, + peer_applied, } } @@ -83,9 +111,15 @@ impl WalShipping for WalShippingService { let payload = WalSegmentPayload::try_from(req) .map_err(|e| Status::invalid_argument(e.to_string()))?; - // The ack's applied hint: read BEFORE enqueue (the segment cannot have - // been applied yet) — a monotonic floor of the receiver's progress, so - // the leader's fold of it is always safe. + // The ack's applied hint: read BEFORE enqueue (the segment cannot + // have been applied yet) — a monotonic floor of this node's durable + // progress, always safe for the leader to fold. The ack deliberately + // does NOT wait for this segment's apply: holding acks couples the + // leader's ship cadence to apply latency, and one gap-parked + // follower then throttles its own feed into a death spiral + // (measured: total quorum collapse at 1k rps). Durable-frontier + // freshness flows the OTHER way instead — the receiver pushes + // `ReportApplied` once per apply round. let applied_seqno = self.applied_hint(source_shard); // Try to send on the inbound channel. On full, yield once and retry. @@ -263,6 +297,37 @@ impl WalShipping for WalShippingService { // is the remaining work. Ok(Response::new(HeartbeatResponse { acknowledged: true })) } + + async fn report_applied( + &self, + request: Request, + ) -> Result, Status> { + let report = request.into_inner(); + let (Ok(reporter), Ok(source)) = ( + u16::try_from(report.reporter_shard), + u16::try_from(report.source_shard), + ) else { + return Err(Status::invalid_argument("shard ids must fit u16")); + }; + let reporter = ShardId(reporter); + // A report is addressed to the stream's SOURCE; when this node's + // serving identity is known (segment source wired), refuse a report + // for some other node's stream — folding a mark into the wrong + // quorum would be silent corruption. + if let Some(segments) = &self.sources.segments + && segments.source_shard() != ShardId(source) + { + return Err(Status::not_found(format!( + "this node serves shard {} (report addressed to shard {source})", + segments.source_shard().0 + ))); + } + fold_peer_applied(&self.peer_applied, reporter, report.applied_seqno); + if let Some(sink) = self.sources.applied_sink.get() { + sink.peer_applied(reporter, report.applied_seqno); + } + Ok(Response::new(AppliedReportAck { acknowledged: true })) + } } /// Start the gRPC server on the given address. @@ -279,11 +344,13 @@ pub fn start_server( config: &GrpcTransportConfig, inbound_tx: mpsc::Sender, sources: ServingSources, + peer_applied: PeerAppliedMap, ) -> Result< tokio::task::JoinHandle>, crate::error::GrpcTransportError, > { - let service = WalShippingService::new(inbound_tx, config.max_payload_bytes, sources); + let service = + WalShippingService::new(inbound_tx, config.max_payload_bytes, sources, peer_applied); let addr = config.listen_addr; let mut server_builder = tonic::transport::Server::builder(); @@ -372,7 +439,12 @@ mod tests { runtime.block_on(async { let max = 1024usize; let (tx, mut rx) = mpsc::channel(4); - let service = WalShippingService::new(tx, max, ServingSources::default()); + let service = WalShippingService::new( + tx, + max, + ServingSources::default(), + Arc::new(Mutex::new(HashMap::new())), + ); // Exactly max: accepted, and forwarded onto the inbound channel. let resp = service @@ -403,10 +475,12 @@ mod tests { }); } - /// m11p2: an ack must piggyback the wired applied source's seqno for the - /// request's source shard. + /// The ship ack piggybacks the applied source's CURRENT floor for the + /// request's source shard — instantly, never waiting on the segment's + /// own apply (m11p3 keeps acks decoupled from apply latency by design; + /// durable freshness flows via `ReportApplied` instead). #[test] - fn ship_segment_ack_carries_applied_seqno() { + fn ship_segment_ack_carries_applied_floor() { struct FixedApplied; impl crate::sources::AppliedSource for FixedApplied { fn applied_seqno(&self, source_shard: ShardId) -> u64 { @@ -422,8 +496,10 @@ mod tests { let sources = ServingSources { applied: Some(Arc::new(FixedApplied)), segments: None, + applied_sink: Arc::default(), }; - let service = WalShippingService::new(tx, 1024, sources); + let service = + WalShippingService::new(tx, 1024, sources, Arc::new(Mutex::new(HashMap::new()))); let mut req = make_request(8); req.id.as_mut().unwrap().shard_id = 2; let resp = service @@ -432,7 +508,86 @@ mod tests { .unwrap() .into_inner(); assert!(resp.accepted); - assert_eq!(resp.applied_seqno, 42, "ack must carry shard 2's applied"); + assert_eq!(resp.applied_seqno, 42, "ack must carry shard 2's floor"); + }); + } + + /// m11p3: a follower's `ReportApplied` folds its durable mark into the + /// shared hint map AND the late-bound applied sink (the quorum input); + /// stale reports never regress; a report addressed to another node's + /// stream is refused. + #[test] + fn report_applied_folds_marks_and_feeds_the_sink() { + struct RecordingSink(Mutex>); + impl crate::sources::AppliedSink for RecordingSink { + fn peer_applied(&self, peer: ShardId, applied: u64) { + self.0.lock().unwrap().push((peer, applied)); + } + } + struct FixedSegments; + impl crate::sources::SegmentSource for FixedSegments { + fn source_shard(&self) -> ShardId { + ShardId(0) + } + fn stream_baseline(&self) -> u64 { + 0 + } + fn flushed_seq(&self) -> u64 { + 0 + } + fn collect_from( + &self, + _from_seq: u64, + _max_events: u64, + _max_bytes: usize, + ) -> Result, String> { + Ok(vec![]) + } + } + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let sink = Arc::new(RecordingSink(Mutex::new(Vec::new()))); + let sources = ServingSources { + applied: None, + segments: Some(Arc::new(FixedSegments)), + applied_sink: Arc::default(), + }; + sources.set_applied_sink(Arc::clone(&sink) as Arc); + let map: PeerAppliedMap = Arc::new(Mutex::new(HashMap::new())); + let (tx, _rx) = mpsc::channel(4); + let service = WalShippingService::new(tx, 1024, sources, Arc::clone(&map)); + + let report = |reporter, source, applied| AppliedReport { + reporter_shard: reporter, + source_shard: source, + applied_seqno: applied, + }; + service + .report_applied(Request::new(report(2, 0, 9))) + .await + .expect("report for this node's stream is accepted"); + // A stale (lower) report folds into the sink but must not + // regress the hint map. + service + .report_applied(Request::new(report(2, 0, 5))) + .await + .unwrap(); + assert_eq!(map.lock().unwrap().get(&ShardId(2)), Some(&9)); + assert_eq!( + sink.0.lock().unwrap().as_slice(), + &[(ShardId(2), 9), (ShardId(2), 5)], + "the sink receives every report verbatim (it owns monotonicity)" + ); + + // Wrong stream: refused loudly. + let err = service + .report_applied(Request::new(report(2, 7, 11))) + .await + .expect_err("a report addressed to another node's stream is a routing bug"); + assert_eq!(err.code(), tonic::Code::NotFound); }); } @@ -490,8 +645,10 @@ mod tests { let sources = ServingSources { applied: None, segments: Some(Arc::new(FakeSegments)), + applied_sink: Arc::default(), }; - let service = WalShippingService::new(tx, 1024, sources); + let service = + WalShippingService::new(tx, 1024, sources, Arc::new(Mutex::new(HashMap::new()))); // Request from seqno 1: the baseline (10) clamps the start to 11. let resp = service diff --git a/tidal-net/src/sources.rs b/tidal-net/src/sources.rs index 4fde8c5..01f3c28 100644 --- a/tidal-net/src/sources.rs +++ b/tidal-net/src/sources.rs @@ -37,6 +37,19 @@ pub trait AppliedSource: Send + Sync + 'static { fn applied_seqno(&self, source_shard: ShardId) -> u64; } +/// Leader-side consumer of follower durable-frontier reports (m11p3). +/// +/// The `ReportApplied` handler invokes it with the reporter's shard and its +/// contiguous durably-applied seqno for THIS node's stream — the input that +/// advances the quorum commit index. Wired late (the embedding application +/// builds its ship queue after the transport), hence the `OnceLock` cell on +/// [`ServingSources`]. +pub trait AppliedSink: Send + Sync + 'static { + /// Fold a follower's durable mark (monotonic; stale or unknown-peer + /// reports must be ignored by the implementation). + fn peer_applied(&self, peer: ShardId, applied: u64); +} + /// Read-back over the node's durable WAL for the catch-up stream. /// /// Implementations are synchronous (they read segment files); the service @@ -75,6 +88,19 @@ pub struct ServingSources { pub applied: Option>, /// WAL read-back for `StreamSegments` (`None` = catch-up unimplemented). pub segments: Option>, + /// Consumer of follower durable-frontier reports (m11p3). A `OnceLock` + /// because the embedding application can only build it AFTER the + /// transport exists (the ship queue takes the transport): fill it via + /// [`ServingSources::set_applied_sink`] once available. Reports arriving + /// before it is set fold into the transport's hint map only. + pub applied_sink: Arc>>, +} + +impl ServingSources { + /// Late-bind the applied-report sink (idempotent; first set wins). + pub fn set_applied_sink(&self, sink: Arc) { + let _ = self.applied_sink.set(sink); + } } impl std::fmt::Debug for ServingSources { @@ -82,6 +108,7 @@ impl std::fmt::Debug for ServingSources { f.debug_struct("ServingSources") .field("applied", &self.applied.is_some()) .field("segments", &self.segments.is_some()) + .field("applied_sink", &self.applied_sink.get().is_some()) .finish() } } diff --git a/tidal-net/src/transport.rs b/tidal-net/src/transport.rs index b13e036..53f5bdf 100644 --- a/tidal-net/src/transport.rs +++ b/tidal-net/src/transport.rs @@ -7,7 +7,7 @@ //! inside a tokio context. use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, sync::{ Arc, Mutex, atomic::{AtomicBool, Ordering}, @@ -68,9 +68,19 @@ pub struct GrpcTransport { /// identical for both (m11p2). inbound_tx: mpsc::Sender, pool: Arc, - /// Per-peer applied-seqno hints from `ShipSegmentResponse` acks (m11p2), - /// read by [`Transport::peer_applied_hint`]. Monotonic max. - peer_applied: Mutex>, + /// Per-peer durable marks, shared with the gRPC service (m11p3): the + /// monotonic max of ship-ack hints (client side) and `ReportApplied` + /// pushes (server side). Read by [`Transport::peer_applied_hint`]. + peer_applied: crate::server::PeerAppliedMap, + /// Last frontier reported per source shard by [`Transport::notify_applied`] + /// (dedup: unchanged rounds send nothing). + last_reported: Mutex>, + /// Source shards whose last `ReportApplied` push failed: the FIRST + /// failure of a streak logs at WARN (a silently stalling frontier report + /// surfaces as unexplained quorum 503s at 3am), repeats stay at debug, + /// and recovery logs at INFO. Shared with the fire-and-forget report + /// tasks. (`Arc`: the spawned task outlives the `&self` borrow.) + report_failing: Arc>>, /// Per-shard catch-up pull state: single-flight + rate limit. catchup: Mutex>, server_handle: tokio::task::JoinHandle>, @@ -188,8 +198,10 @@ impl GrpcTransport { // Build server and client pool inside the runtime context. // TLS setup in tonic requires a tokio reactor to be available. let server_tx = inbound_tx.clone(); + let peer_applied: crate::server::PeerAppliedMap = Arc::new(Mutex::new(HashMap::new())); + let server_map = Arc::clone(&peer_applied); let (server_handle, pool) = runtime.block_on(async { - let handle = server::start_server(&config, server_tx, sources)?; + let handle = server::start_server(&config, server_tx, sources, server_map)?; let pool = PeerPool::new(&config)?; Ok::<_, GrpcTransportError>((handle, pool)) })?; @@ -200,7 +212,9 @@ impl GrpcTransport { inbound_rx: Mutex::new(inbound_rx), inbound_tx, pool: Arc::new(pool), - peer_applied: Mutex::new(HashMap::new()), + peer_applied, + last_reported: Mutex::new(HashMap::new()), + report_failing: Arc::new(Mutex::new(HashSet::new())), catchup: Mutex::new(HashMap::new()), server_handle, shutdown: Arc::new(ShutdownSignal::new()), @@ -316,17 +330,7 @@ impl Transport for GrpcTransport { .runtime() .block_on(self.pool.send_to(to, payload)) .map_err(TransportError::from)?; - if applied > 0 { - let mut hints = self - .peer_applied - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let entry = hints.entry(to).or_insert(0); - if applied > *entry { - *entry = applied; - } - drop(hints); - } + crate::server::fold_peer_applied(&self.peer_applied, to, applied); Ok(()) } @@ -339,6 +343,78 @@ impl Transport for GrpcTransport { .unwrap_or(0) } + fn notify_applied(&self, source_shard: ShardId, applied: u64) { + // Push this node's durable frontier to the stream's source (m11p3 + // quorum acks): fire-and-forget, once per ADVANCED apply round (the + // dedup map drops unchanged rounds), fully decoupled from ship acks + // — so the leader's commit index stays fresh even when its outbound + // ships stall (gap-parked follower, pull-based catch-up, quiet + // leader). A lost report self-corrects on the next advanced round; + // marks are monotonic on the receiving side. + if self.shutdown.is_requested() || source_shard == self.config.local_shard { + return; + } + let mut last = self + .last_reported + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let entry = last.entry(source_shard).or_insert(0); + if applied <= *entry { + return; + } + *entry = applied; + drop(last); + let pool = Arc::clone(&self.pool); + let reporter = self.config.local_shard; + let failing = Arc::clone(&self.report_failing); + self.runtime().spawn(async move { + match pool.report_applied(source_shard, reporter, applied).await { + Ok(()) => { + let recovered = failing + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&source_shard); + if recovered { + tracing::info!( + source = source_shard.0, + applied, + "applied-frontier reports recovered" + ); + } + } + Err(e) => { + // Best-effort by design: the next advanced round + // re-reports, and ship-ack hints keep flowing regardless. + // But a STREAK of failures must be visible at default log + // levels — a silently stalling frontier report shows up + // as unexplained quorum 503s on the leader. + let first_of_streak = failing + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(source_shard); + if first_of_streak { + tracing::warn!( + source = source_shard.0, + applied, + error = %e, + "applied-frontier report failed; the source's quorum \ + freshness degrades to ship-ack hints until this \ + recovers (will retry on next advance)" + ); + } else { + tracing::debug!( + source = source_shard.0, + applied, + error = %e, + "applied-frontier report still failing (will retry \ + on next advance)" + ); + } + } + } + }); + } + fn request_catchup(&self, from_shard: ShardId, from_seqno: u64) { if self.shutdown.is_requested() { return; diff --git a/tidal-server/src/cluster/forward.rs b/tidal-server/src/cluster/forward.rs index 5cc4654..26f7adb 100644 --- a/tidal-server/src/cluster/forward.rs +++ b/tidal-server/src/cluster/forward.rs @@ -45,6 +45,26 @@ pub const INTERNAL_MARKER: &str = "x-tidal-internal"; /// The marker value siblings set. pub const INTERNAL_MARKER_VALUE: &str = "1"; +/// Request header overriding the deployment's write-acknowledgment mode +/// (m11p3): `leader` or `quorum`. Forwarded verbatim with the write so the +/// leader honors the CALLER's choice, not the gateway's default. +pub const ACK_HEADER: &str = "x-tidal-ack"; + +/// Response header carrying a cluster write's assigned replicated-log seqno +/// (m11p3). Relayed verbatim on forwarded writes. +pub const SEQ_HEADER: &str = "x-tidal-seq"; + +/// Response header (value `1`) marking a 2xx write whose record was +/// dedup-suppressed by the WAL content-hash window (m11p3): an identical +/// record is already durable, no new log entry was created, so the response +/// carries this instead of [`SEQ_HEADER`]. Relayed verbatim on forwarded +/// writes so a gateway client can tell "deduplicated" from "no seqno +/// surface" without consulting the runbook. +pub const DEDUP_HEADER: &str = "x-tidal-deduplicated"; + +/// The marker value set on [`DEDUP_HEADER`]. +pub const DEDUP_HEADER_VALUE: &str = "1"; + /// Connect timeout for every forwarded/broadcast/aggregation request. A peer /// whose TCP connect does not complete in 1s is treated as unreachable. pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(1); @@ -118,6 +138,12 @@ pub fn forwarded_auth(headers: &HeaderMap) -> Option { /// The outcome of forwarding one request to a single peer: the peer's status /// code and raw JSON body, ready to hand straight back to the original client. pub struct ForwardedResponse { + /// The peer's `x-tidal-seq` response header (a cluster write's assigned + /// replicated-log seqno, m11p3), relayed verbatim when present. + pub seq: Option, + /// Whether the peer marked the write dedup-suppressed + /// ([`DEDUP_HEADER`]), relayed verbatim. + pub deduplicated: bool, /// The peer's HTTP status, relayed verbatim. pub status: StatusCode, /// The peer's response body parsed as JSON (or a synthesized object if the @@ -145,9 +171,22 @@ pub async fn forward_json( body: &B, auth: Option<&str>, internal: bool, +) -> Result { + forward_json_with_headers(client, url, body, auth, internal, &[]).await +} + +/// [`forward_json`] with extra request headers passed through verbatim +/// (m11p3: the caller's `x-tidal-ack` override must reach the leader). +pub async fn forward_json_with_headers( + client: &reqwest::Client, + url: &str, + body: &B, + auth: Option<&str>, + internal: bool, + passthrough: &[(&'static str, String)], ) -> Result { // Per-request override of the client-level REQUEST_TIMEOUT: a forward's - // response covers the peer's real work (WAL fsync, its own broadcast, a + // response covers the peer's real work (WAL fsync, a quorum wait, a // CRDT merge), not just a hop. See FORWARD_REQUEST_TIMEOUT. let mut req = client.post(url).timeout(FORWARD_REQUEST_TIMEOUT).json(body); if let Some(auth) = auth { @@ -156,8 +195,23 @@ pub async fn forward_json( if internal { req = req.header(INTERNAL_MARKER, INTERNAL_MARKER_VALUE); } + for (name, value) in passthrough { + req = req.header(*name, value); + } let resp = req.send().await.map_err(|e| e.to_string())?; let status = resp.status(); + // Capture the seq/dedup headers BEFORE consuming the body: a forwarded + // write's caller relays them so the client sees the leader's verdict. + let seq = resp + .headers() + .get(SEQ_HEADER) + .and_then(|v| v.to_str().ok()) + .map(str::to_owned); + let deduplicated = resp + .headers() + .get(DEDUP_HEADER) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v == DEDUP_HEADER_VALUE); // Relay the body verbatim where possible; an empty/non-JSON body becomes a // null so the relay always yields a JSON value the caller can wrap. let bytes = resp.bytes().await.map_err(|e| e.to_string())?; @@ -166,7 +220,12 @@ pub async fn forward_json( } else { serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null) }; - Ok(ForwardedResponse { status, body }) + Ok(ForwardedResponse { + seq, + deduplicated, + status, + body, + }) } /// Fan a JSON body out to every `(name, url)` peer concurrently with the diff --git a/tidal-server/src/cluster/node.rs b/tidal-server/src/cluster/node.rs index 24e8dca..de4b860 100644 --- a/tidal-server/src/cluster/node.rs +++ b/tidal-server/src/cluster/node.rs @@ -63,7 +63,7 @@ use tidaldb::{ }, query::{retrieve::Retrieve, search::Search}, replication::{ - ShipQueue, Transport, WalFeedSource, + CommitIndex, ShipQueue, Transport, WalFeedSource, shard::{RegionId, ShardId}, }, schema::{EntityId, Schema, Timestamp}, @@ -74,7 +74,10 @@ use tower_http::timeout::TimeoutLayer; use utoipa::ToSchema; use super::{ - forward::{self, broadcast_marked, forward_json, forwarded_auth, is_internal, peer_url}, + forward::{ + self, broadcast_marked, forward_json, forward_json_with_headers, forwarded_auth, + is_internal, peer_url, + }, routes::{ClusterAppError, ScatterGatherInfo, ShardedFeedResponse, ShardedSearchResponse}, topology::{TopologySpec, shard_of_region}, transport::{GRPC_READY_TIMEOUT, grpc_server_ready, resolve_grpc_addr}, @@ -91,6 +94,44 @@ use crate::{ }, }; +/// Write-acknowledgment mode for replicated cluster writes (m11p3). +/// +/// `Leader` (the default) succeeds at leader group-commit fsync — the m0-m11p2 +/// contract. `Quorum` additionally blocks until a majority of the replica set +/// (leader + peers) durably holds the write (the commit index passes its +/// seqno), trading tail latency for failover-survivable durability. Set the +/// deployment default with `replication.ack` in the topology; override per +/// request with the `x-tidal-ack` header. +/// +/// Adding a mode fans out beyond this enum: `parse`/`as_str` here, the +/// topology validation table (`topology.rs`), the stress CLI's `--ack`, and +/// the runbook §8 table — grep `x-tidal-ack` for the full set. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AckMode { + /// Success = durable on the leader (its WAL group-commit fsync). + Leader, + /// Success = durable on a majority of the replica set. + Quorum, +} + +impl AckMode { + /// Parse a topology/header value. + fn parse(value: &str) -> Option { + match value { + "leader" => Some(Self::Leader), + "quorum" => Some(Self::Quorum), + _ => None, + } + } + + const fn as_str(self) -> &'static str { + match self { + Self::Leader => "leader", + Self::Quorum => "quorum", + } + } +} + /// Persisted stream-baseline filename inside the node's data dir. /// /// The baseline is the WAL seqno at which THIS node's outbound stream started @@ -100,6 +141,13 @@ use crate::{ /// double-apply it on followers. const STREAM_BASELINE_FILE: &str = "stream_baseline"; +/// Cap on one commit-watch bridge condvar wait: the longest the bridge +/// thread can go without re-checking its stop flag, i.e. the worst-case +/// shutdown latency the bridge adds. Deliberately a constant, not a +/// topology knob — it is an internal poll bound, invisible to clients +/// (commit-index CHANGES wake the bridge immediately regardless). +const COMMIT_BRIDGE_WAKE_INTERVAL: Duration = Duration::from_secs(1); + /// A multi-process cluster node owning exactly one region. pub struct RegionClusterState { /// This process's region id (index into the topology declaration order). @@ -167,6 +215,23 @@ pub struct RegionClusterState { /// (`/signals` staging, the cluster admin verbs). Shared by every write /// request. write_pool: ClusterWritePool, + /// Quorum commit index over the ship queue's peers (m11p3); leadership- + /// gated alongside the queue. `ack=quorum` writes await it through + /// `commit_watch` (async — never a parked thread per waiter). + commit: Arc, + /// Async mirror of `commit`: `(epoch, commit_index, active)`, published + /// by one dedicated bridge thread. Handlers `await` changes on a clone — + /// a thread-per-wait design exhausts the blocking pool under open-loop + /// load and starves the completions that advance the index (measured: + /// total quorum collapse at 1k rps). + commit_watch: tokio::sync::watch::Receiver<(u64, u64, bool)>, + /// Stops the commit-watch bridge thread on shutdown. + commit_bridge_stop: Arc, + /// Deployment-default write acknowledgment mode (`replication.ack`). + ack_default: AckMode, + /// Budget an `ack=quorum` write waits for the commit index before the + /// retryable 503 (`replication.quorum_timeout_ms`, default 2s). + quorum_timeout: Duration, /// Flipped on shutdown so `/health` reports not-ready while draining. shutting_down: AtomicBool, } @@ -300,7 +365,11 @@ impl RegionClusterState { baseline: Arc::clone(&stream_baseline), feed: Arc::clone(&ship_feed), })), + // Late-bound below (m11p3): the commit index only exists once + // the ship queue is spawned, which needs this transport first. + applied_sink: Arc::default(), }; + let applied_sink_cell = Arc::clone(&sources.applied_sink); let listen_addr = resolve_grpc_addr(my_grpc_spec.as_deref(), region_name)?; let transport = GrpcTransport::new_with_sources( GrpcTransportConfig { @@ -373,6 +442,53 @@ impl RegionClusterState { transport.request_catchup(leader_shard, applied + 1); } + let commit = ship_queue.commit_index(); + // Wire follower frontier reports (gRPC `ReportApplied`) into the + // quorum commit index: the transport's server folds each report's + // durable mark through this sink. Decoupled from ship acks by + // design — see `Transport::notify_applied`. + let _ = applied_sink_cell.set(Arc::new(CommitIndexSink { + commit: Arc::clone(&commit), + }) as Arc); + let ack_default = topology + .replication + .ack + .as_deref() + .and_then(AckMode::parse) + .unwrap_or(AckMode::Leader); + let quorum_timeout = + Duration::from_millis(topology.replication.quorum_timeout_ms.unwrap_or(2_000)); + + // The async commit bridge: ONE thread blocks on the index's condvar + // and republishes every change into a watch channel that any number + // of request handlers await for free. + let (commit_tx, commit_watch) = tokio::sync::watch::channel(commit.snapshot()); + let commit_bridge_stop = Arc::new(AtomicBool::new(false)); + { + let commit = Arc::clone(&commit); + let stop = Arc::clone(&commit_bridge_stop); + std::thread::Builder::new() + .name("tidal-commit-watch".into()) + .spawn(move || { + let (mut epoch, mut idx, _) = commit.snapshot(); + while !stop.load(Ordering::Acquire) { + // The wake cap bounds shutdown latency; an unchanged + // index republishes nothing (send_if_modified). + let next = commit.wait_change(epoch, idx, COMMIT_BRIDGE_WAKE_INTERVAL); + (epoch, idx) = (next.0, next.1); + commit_tx.send_if_modified(|cur| { + if *cur == next { + false + } else { + *cur = next; + true + } + }); + } + }) + .expect("spawn commit-watch bridge thread"); + } + Ok(Self { region, region_name: region_name.to_string(), @@ -393,6 +509,11 @@ impl RegionClusterState { broadcast_peer_timeout: topology.broadcast_peer_timeout(), blocking_client, write_pool, + commit, + commit_watch, + commit_bridge_stop, + ack_default, + quorum_timeout, shutting_down: AtomicBool::new(false), }) } @@ -414,6 +535,7 @@ impl RegionClusterState { /// signal the segment receiver to exit. Idempotent. pub fn shutdown(&mut self) { self.set_shutting_down(); + self.commit_bridge_stop.store(true, Ordering::Release); // Join the ship-queue senders FIRST so no batch ship races the // transport/db teardown below (their threads hold their own Arcs, but // an in-flight send_segment against a half-down transport would only @@ -654,6 +776,42 @@ impl RegionClusterState { // ── Write path ────────────────────────────────────────────────────────── + /// Resolve a write's acknowledgment mode: the `x-tidal-ack` request + /// header when present (the caller's explicit choice), else the + /// deployment default (`replication.ack`). + fn ack_mode_for(&self, headers: &HeaderMap) -> Result { + let Some(value) = headers.get(forward::ACK_HEADER) else { + return Ok(self.ack_default); + }; + let value = value + .to_str() + .map_err(|_| ServerError::BadRequest("x-tidal-ack must be ASCII".into()))?; + AckMode::parse(value).ok_or_else(|| { + ServerError::BadRequest(format!( + "x-tidal-ack must be \"leader\" or \"quorum\", got {value:?}" + )) + }) + } + + /// The retryable quorum-timeout 503 for `seq`, built from the index's + /// current marks (names the laggard regions). + fn quorum_timeout_error(&self, seq: u64) -> ServerError { + self.cluster_metrics.incr_quorum_timeouts(); + let marks = self.commit.peer_marks(); + let confirmed = marks.iter().filter(|&&(_, mark)| mark >= seq).count(); + ServerError::QuorumTimeout { + seq, + needed: self.commit.needed_peers(), + confirmed, + committed: self.commit.committed(), + laggards: marks + .iter() + .filter(|&&(_, mark)| mark < seq) + .map(|&(shard, _)| self.region_name_of(RegionId(shard.0)).to_string()) + .collect(), + } + } + /// Stage a signal write: submit it to the leader WAL (microseconds — one /// bounded-channel send; the WAL writer assigns the stream seqno at /// flush, m11p2). @@ -694,29 +852,51 @@ impl RegionClusterState { /// event its durable log carries. The dual-stream window a mid-write /// demotion opens is the same one the old eager path had; term fencing /// (m11p4) is what closes it. - fn complete_signal_write(&self, staged: StagedSignal) -> Result<()> { + /// Returns the write's assigned WAL seqno (`0` = dedup-suppressed; an + /// identical record is already durable, so there is nothing new to gate + /// quorum on). + fn complete_signal_write(&self, staged: StagedSignal) -> Result { let db = self.db()?; - staged.wait(db).map_err(ServerError::Tidal)?; - self.cluster_metrics - .set_relay_frontiers(self.ship_feed.flushed_seq(), self.ship_feed.flushed_seq()); - Ok(()) + let seq = staged.wait(db).map_err(ServerError::Tidal)?; + self.set_frontier_gauges(); + Ok(seq) + } + + /// Publish the stream-frontier gauges: `relay_last_seq` = the WAL flushed + /// frontier, `relay_durable_seq` = the quorum commit index (m11p3 — the + /// gap between them is the cluster's quorum lag). With zero peers the + /// leader alone is the majority, so the gauges coincide. + fn set_frontier_gauges(&self) { + let flushed = self.ship_feed.flushed_seq(); + let commit = if self.commit.needed_peers() == 0 { + flushed + } else { + self.commit.committed() + }; + self.cluster_metrics.set_relay_frontiers(flushed, commit); } /// Apply an item write on the LEADER: the engine journals it as a kind-1 /// WAL record first (the one replicated log — this is what replicates it /// and serves catch-up), then upserts storage (m11p2). + /// Returns the kind-1 record's WAL seqno (`None` only when blob + /// journaling is off, which cluster mode never is). fn apply_item_local( db: &TidalDb, entity: EntityId, metadata: &HashMap, - ) -> Result<()> { + ) -> Result> { db.write_item_with_metadata(entity, metadata) .map_err(ServerError::Tidal) } /// Apply an embedding write on the LEADER (kind-2 WAL record first; see /// [`apply_item_local`]). - fn apply_embedding_local(db: &TidalDb, entity: EntityId, values: &[f32]) -> Result<()> { + fn apply_embedding_local( + db: &TidalDb, + entity: EntityId, + values: &[f32], + ) -> Result> { db.write_item_embedding(entity, values) .map_err(ServerError::Tidal) } @@ -888,6 +1068,16 @@ impl RegionClusterState { .map(|r| self.region_name_of(*r).to_string()) .collect(); + let commit_index = if is_leader { + if self.commit.needed_peers() == 0 { + last_seq + } else { + self.commit.committed() + } + } else { + 0 + }; + Ok(LocalStatusResponse { region: self.region_name.clone(), is_leader, @@ -896,6 +1086,8 @@ impl RegionClusterState { applied_events, lag_events, partitioned, + commit_index, + ack: self.ack_default.as_str().to_string(), reachable: true, }) } @@ -1073,8 +1265,9 @@ impl StagedWriteTicket { } } - /// Complete the write on the current (blocking-pool) thread. - fn complete(mut self) -> Result<()> { + /// Complete the write on the current (blocking-pool) thread. Returns the + /// write's WAL seqno (`0` = dedup-suppressed). + fn complete(mut self) -> Result { // `staged` is always `Some` until consumed here or in Drop; this is the // only consuming method and it takes `self` by value. let staged = self @@ -1380,6 +1573,12 @@ pub struct LocalStatusResponse { lag_events: u64, /// Peers this leader is currently partitioned from (ship-skipped). partitioned: Vec, + /// The quorum commit index (m11p3): the highest seqno a majority of the + /// replica set durably holds. Only meaningful when `is_leader`; + /// `last_seq - commit_index` is the cluster's quorum lag. + commit_index: u64, + /// This node's write acknowledgment default (`leader` or `quorum`). + ack: String, /// Always true (this node is serving its own status request). reachable: bool, } @@ -1886,12 +2085,19 @@ pub async fn create_item( if !state.is_leader() { return Err(ClusterAppError(state.not_leader())); } + let ack = state.ack_mode_for(&headers).map_err(ClusterAppError)?; let db = state.db_arc().map_err(ClusterAppError)?; let entity = EntityId::new(req.entity_id); let metadata = req.metadata.clone(); - offload_region_read(move || RegionClusterState::apply_item_local(&db, entity, &metadata)) - .await?; - Ok(StatusCode::CREATED.into_response()) + let seq = + offload_region_read(move || RegionClusterState::apply_item_local(&db, entity, &metadata)) + .await?; + if ack == AckMode::Quorum + && let Some(seq) = seq + { + await_quorum(&state, seq).await?; + } + Ok(with_seq_header(StatusCode::CREATED, seq)) } /// Write an item embedding on the cluster (m11p2: kind-2 WAL record — same @@ -1920,12 +2126,20 @@ pub async fn write_embedding( if !state.is_leader() { return Err(ClusterAppError(state.not_leader())); } + let ack = state.ack_mode_for(&headers).map_err(ClusterAppError)?; let db = state.db_arc().map_err(ClusterAppError)?; let entity = EntityId::new(req.entity_id); let values = req.values.clone(); - offload_region_read(move || RegionClusterState::apply_embedding_local(&db, entity, &values)) - .await?; - Ok(StatusCode::NO_CONTENT.into_response()) + let seq = offload_region_read(move || { + RegionClusterState::apply_embedding_local(&db, entity, &values) + }) + .await?; + if ack == AckMode::Quorum + && let Some(seq) = seq + { + await_quorum(&state, seq).await?; + } + Ok(with_seq_header(StatusCode::NO_CONTENT, seq)) } /// Record a signal on the cluster. @@ -1935,6 +2149,11 @@ pub async fn write_embedding( /// fsync) and best-effort eager-ship to siblings over gRPC. The signal is the /// replicated stream, so there is NO HTTP broadcast — followers receive it via /// the WAL relay. A 204 asserts leader durability only. +/// +/// The 204 carries `x-tidal-seq` (the write's replicated-log seqno) — or +/// `x-tidal-deduplicated: 1` on the rare write suppressed by the WAL's +/// content-hash dedup window (an identical record is already durable; no new +/// log entry exists to name). #[utoipa::path( post, path = "/signals", @@ -1982,6 +2201,7 @@ pub async fn write_signal( // Drop completes the write on a detached thread — an orphaned staged // seqno would otherwise stall the durable frontier (and all shipping) // forever. + let ack = state.ack_mode_for(&headers).map_err(ClusterAppError)?; let state_for_job = Arc::clone(&state); let ticket = state .write_pool @@ -1991,8 +2211,15 @@ pub async fn write_signal( }) .await .map_err(ClusterAppError)?; - offload_region_read(move || ticket.complete()).await?; - Ok(StatusCode::NO_CONTENT.into_response()) + let seq = offload_region_read(move || ticket.complete()).await?; + // Quorum gate (m11p3): the write is leader-durable; now block until a + // majority of the replica set durably holds it. The dedup sentinel (0) + // skips the gate — an identical record is already durable, and ITS ack + // covered quorum (this request created no new log entry to gate on). + if ack == AckMode::Quorum && seq > 0 { + await_quorum(&state, seq).await?; + } + Ok(with_seq_header(StatusCode::NO_CONTENT, Some(seq))) } /// `POST /hardnegs` request body. @@ -2050,6 +2277,127 @@ pub async fn write_hardneg( /// marker so the leader applies the write locally and does NOT re-forward. A /// leader that cannot be reached degrades to a 503 JSON body naming the leader /// and the connect error (never a hang, never a silent drop). +/// Await the commit index covering `seq` (m11p3 `ack=quorum`): a majority of +/// the replica set has durably applied the write. Returns the commit index +/// that satisfied the wait. +/// +/// Fully async — each waiter holds a watch receiver, never a thread. The +/// write is already durable on THIS leader; a timeout means the durability +/// claim could not be confirmed in budget — the retryable 503 names the +/// laggards, and the write MAY still commit (at-least-once on retry; see +/// runbook §8). A leadership change mid-wait (epoch bump / deactivation) +/// fails with `NotLeader`: a demoted leader must never claim quorum. +async fn await_quorum( + state: &Arc, + seq: u64, +) -> std::result::Result { + if state.commit.needed_peers() == 0 { + // Single-replica: the leader alone is the majority — but only while + // it still leads (the demoted-leader invariant has no replica-count + // exception; mirrors `CommitIndex::wait_for`). + let (_, _, active) = state.commit.snapshot(); + if !active { + return Err(ClusterAppError(state.not_leader())); + } + return Ok(seq); + } + let mut watch = state.commit_watch.clone(); + let deadline = tokio::time::Instant::now() + state.quorum_timeout; + let entry_epoch = { + let (epoch, _, active) = *watch.borrow_and_update(); + if !active { + return Err(ClusterAppError(state.not_leader())); + } + epoch + }; + loop { + { + let (epoch, commit, active) = *watch.borrow_and_update(); + if epoch != entry_epoch || !active { + return Err(ClusterAppError(state.not_leader())); + } + if commit >= seq { + return Ok(commit); + } + } + match tokio::time::timeout_at(deadline, watch.changed()).await { + Ok(Ok(())) => {} + Ok(Err(_)) => { + // The bridge sender is gone. Expected on a clean shutdown; + // anything else means the 'tidal-commit-watch' thread died — + // say so loudly, or its quorum 503s get chased as a network + // problem at 3am. + if !state.is_shutting_down() { + tracing::warn!( + seq, + "commit-watch bridge disconnected outside shutdown; \ + check for a panic in the 'tidal-commit-watch' thread" + ); + } + return Err(ClusterAppError(ServerError::Unavailable( + "server shutting down".into(), + ))); + } + Err(_) => { + // Deadline expired — but re-read the INDEX (not the watch + // mirror: the index is fresher by the bridge's republish + // latency) once before erroring: the commit can pass `seq` + // between the timer firing and this arm running, and a 503 + // for a write that IS quorum-committed would be a false + // negative the caller then retries at-least-once. The same + // final read keeps the error's laggard list as fresh as a + // snapshot can be. + let (epoch, commit, active) = state.commit.snapshot(); + if epoch == entry_epoch && active && commit >= seq { + return Ok(commit); + } + return Err(ClusterAppError(state.quorum_timeout_error(seq))); + } + } + } +} + +/// Folds follower `ReportApplied` durable marks into the quorum commit +/// index (m11p3). The index itself enforces monotonicity, unknown-peer +/// rejection, and leadership epochs. +struct CommitIndexSink { + commit: Arc, +} + +impl tidal_net::sources::AppliedSink for CommitIndexSink { + fn peer_applied(&self, peer: ShardId, applied: u64) { + self.commit.update_peer(peer, applied); + } +} + +/// Build a write-success response carrying the `x-tidal-seq` header (the +/// write's replicated-log seqno, m11p3). The dedup sentinel (`Some(0)` — an +/// identical record is already durable, no new log entry exists) carries +/// `x-tidal-deduplicated: 1` instead, so a caller tracking durability +/// cursors can tell "suppressed as duplicate" from "no seqno surface" +/// (`None` — not journaled, e.g. outside cluster mode). +fn with_seq_header(status: StatusCode, seq: Option) -> Response { + let mut resp = status.into_response(); + match seq { + Some(seq) if seq > 0 => { + if let Ok(value) = axum::http::HeaderValue::from_str(&seq.to_string()) { + resp.headers_mut().insert( + axum::http::HeaderName::from_static(forward::SEQ_HEADER), + value, + ); + } + } + Some(_) => { + resp.headers_mut().insert( + axum::http::HeaderName::from_static(forward::DEDUP_HEADER), + axum::http::HeaderValue::from_static(forward::DEDUP_HEADER_VALUE), + ); + } + None => {} + } + resp +} + async fn forward_write( state: &Arc, path: &str, @@ -2063,8 +2411,43 @@ async fn forward_write( }; let url = peer_url(&leader_http, path); let auth = forwarded_auth(headers); - match forward_json(&state.client, &url, body, auth.as_deref(), true).await { - Ok(resp) => Ok((resp.status, Json(resp.body)).into_response()), + // The caller's ack-mode override travels WITH the write (m11p3): the + // leader honors the caller's choice, not this gateway's default. + let passthrough: Vec<(&'static str, String)> = headers + .get(forward::ACK_HEADER) + .and_then(|v| v.to_str().ok()) + .map(|v| vec![(forward::ACK_HEADER, v.to_owned())]) + .unwrap_or_default(); + match forward_json_with_headers( + &state.client, + &url, + body, + auth.as_deref(), + true, + &passthrough, + ) + .await + { + Ok(resp) => { + // Relay the leader's seq/dedup headers so the original caller + // sees the write's replicated-log verdict through the forward. + let mut response = (resp.status, Json(resp.body)).into_response(); + if let Some(seq) = resp.seq + && let Ok(value) = axum::http::HeaderValue::from_str(&seq) + { + response.headers_mut().insert( + axum::http::HeaderName::from_static(forward::SEQ_HEADER), + value, + ); + } + if resp.deduplicated { + response.headers_mut().insert( + axum::http::HeaderName::from_static(forward::DEDUP_HEADER), + axum::http::HeaderValue::from_static(forward::DEDUP_HEADER_VALUE), + ); + } + Ok(response) + } Err(e) => { // Leader unreachable: the typed 503 names the leader, its address, // and the connect error (single body shape via ClusterAppError). @@ -2555,7 +2938,7 @@ pub async fn sharded_create_item( req.entity_id, "/sharded/items", &req, - move || RegionClusterState::apply_item_local(&db, entity, &metadata), + move || RegionClusterState::apply_item_local(&db, entity, &metadata).map(|_seq| ()), StatusCode::CREATED, ) .await @@ -2589,7 +2972,7 @@ pub async fn sharded_write_embedding( req.entity_id, "/sharded/embeddings", &req, - move || RegionClusterState::apply_embedding_local(&db, entity, &values), + move || RegionClusterState::apply_embedding_local(&db, entity, &values).map(|_seq| ()), StatusCode::NO_CONTENT, ) .await diff --git a/tidal-server/src/cluster/routes.rs b/tidal-server/src/cluster/routes.rs index 88218d9..21e0b81 100644 --- a/tidal-server/src/cluster/routes.rs +++ b/tidal-server/src/cluster/routes.rs @@ -906,6 +906,21 @@ impl IntoResponse for ClusterAppError { "region": region, "cause": cause, }), + ServerError::QuorumTimeout { + seq, + needed, + confirmed, + committed, + laggards, + } => serde_json::json!({ + "error": self.0.to_string(), + "retryable": true, + "seq": seq, + "needed": needed, + "confirmed": confirmed, + "commit_index": committed, + "laggards": laggards, + }), _ => serde_json::json!({ "error": self.0.to_string() }), }; (status, Json(body)).into_response() diff --git a/tidal-server/src/cluster/topology.rs b/tidal-server/src/cluster/topology.rs index 65b18c4..ac49dc8 100644 --- a/tidal-server/src/cluster/topology.rs +++ b/tidal-server/src/cluster/topology.rs @@ -68,6 +68,17 @@ pub struct ReplicationSpec { /// Milliseconds before a transiently-failed batch ship is retried. #[serde(default)] pub retry_ms: Option, + /// Deployment-default write acknowledgment mode (m11p3): `leader` + /// (default — success at leader group-commit fsync) or `quorum` (success + /// once a majority of the replica set durably holds the write). Callers + /// override per request with the `x-tidal-ack` header. + #[serde(default)] + pub ack: Option, + /// Milliseconds an `ack=quorum` write waits for the commit index before + /// returning a retryable 503 naming the laggards. Default 2000 (the + /// cross-region replication SLO). + #[serde(default)] + pub quorum_timeout_ms: Option, } /// WAL group-commit tuning (the optional `wal:` YAML block). @@ -220,6 +231,18 @@ fn validate_spec_values(spec: &TopologySpec) -> Result<()> { "replication.retry_ms must be >= 1 (omit it for the 100ms default)".into(), )); } + if let Some(ack) = spec.replication.ack.as_deref() + && !matches!(ack, "leader" | "quorum") + { + return Err(ServerError::SchemaConfig(format!( + "replication.ack must be \"leader\" or \"quorum\", got {ack:?}" + ))); + } + if spec.replication.quorum_timeout_ms == Some(0) { + return Err(ServerError::SchemaConfig( + "replication.quorum_timeout_ms must be >= 1 (omit it for the 2000ms default)".into(), + )); + } if let Some(n) = spec.wal.batch_size && !(1..=256).contains(&n) { diff --git a/tidal-server/src/error.rs b/tidal-server/src/error.rs index 0c79270..b09ae73 100644 --- a/tidal-server/src/error.rs +++ b/tidal-server/src/error.rs @@ -67,6 +67,22 @@ pub enum ServerError { /// 503 naming the region and the transport cause. #[error("region '{region}' unreachable: {cause}")] RegionUnreachable { region: String, cause: String }, + /// An `ack=quorum` write was durable on the leader but a majority of the + /// replica set did not confirm durability within the quorum budget + /// (m11p3). Maps to a retryable 503 naming the laggards. The write is in + /// the leader's log and MAY still commit — retries are at-least-once + /// (see runbook §8 for the dedup guidance). + #[error( + "quorum not reached for seqno {seq} within budget: {confirmed} of {needed} required \ + follower confirmations (commit index {committed}); laggards: {laggards:?}" + )] + QuorumTimeout { + seq: u64, + needed: usize, + confirmed: usize, + committed: u64, + laggards: Vec, + }, } impl ServerError { diff --git a/tidal-server/src/router.rs b/tidal-server/src/router.rs index be77955..2cad066 100644 --- a/tidal-server/src/router.rs +++ b/tidal-server/src/router.rs @@ -434,7 +434,8 @@ pub const fn status_from_error(err: &ServerError) -> StatusCode { ServerError::Unavailable(_) | ServerError::NotLeader { .. } | ServerError::LeaderUnreachable { .. } - | ServerError::RegionUnreachable { .. } => StatusCode::SERVICE_UNAVAILABLE, + | ServerError::RegionUnreachable { .. } + | ServerError::QuorumTimeout { .. } => StatusCode::SERVICE_UNAVAILABLE, ServerError::Tidal(tidal_err) => match tidal_err { tidaldb::TidalError::NotFound { .. } => StatusCode::NOT_FOUND, tidaldb::TidalError::Schema(_) | tidaldb::TidalError::InvalidInput(_) => { diff --git a/tidal-server/src/scatter_gather.rs b/tidal-server/src/scatter_gather.rs index 47bc40d..b13af36 100644 --- a/tidal-server/src/scatter_gather.rs +++ b/tidal-server/src/scatter_gather.rs @@ -327,6 +327,7 @@ pub fn sharded_write_item( .node(shard) .db .write_item_with_metadata(entity_id, metadata) + .map(|_seq| ()) .map_err(ServerError::from) } @@ -346,6 +347,7 @@ pub fn sharded_write_embedding( .node(shard) .db .write_item_embedding(entity_id, embedding) + .map(|_seq| ()) .map_err(ServerError::from) } diff --git a/tidal-server/src/state.rs b/tidal-server/src/state.rs index 938c787..ddc8517 100644 --- a/tidal-server/src/state.rs +++ b/tidal-server/src/state.rs @@ -73,6 +73,7 @@ impl ServerState { ) -> Result<()> { self.db .write_item_with_metadata(entity_id, metadata) + .map(|_seq| ()) .map_err(ServerError::from) } @@ -84,6 +85,7 @@ impl ServerState { pub fn write_embedding(&self, entity_id: EntityId, embedding: &[f32]) -> Result<()> { self.db .write_item_embedding(entity_id, embedding) + .map(|_seq| ()) .map_err(ServerError::from) } diff --git a/tidal-server/tests/cluster_chaos.rs b/tidal-server/tests/cluster_chaos.rs index 9e17f51..ed8c2b8 100644 --- a/tidal-server/tests/cluster_chaos.rs +++ b/tidal-server/tests/cluster_chaos.rs @@ -723,9 +723,8 @@ fn mp_items_ride_the_log_and_catchup_stream() { assert_feed_parity("leader vs follower (items via log)", &leader_feed, &feed); } println!( - "[log] {} items (one via a follower gateway) + signals converged on every node \ - through the one replicated log", - via_follower + "[log] {via_follower} items (one via a follower gateway) + signals converged on \ + every node through the one replicated log" ); // ── Downtime → restart → boot-time catch-up stream (no heal verb) ───────── diff --git a/tidal-server/tests/cluster_quorum.rs b/tidal-server/tests/cluster_quorum.rs new file mode 100644 index 0000000..c06122e --- /dev/null +++ b/tidal-server/tests/cluster_quorum.rs @@ -0,0 +1,419 @@ +//! Tier-3 quorum-ack suite (m11p3, REAL multi-process cluster). +//! +//! Two pillars: +//! +//! 1. **Quorum semantics under partition** — `ack=quorum` writes succeed +//! while a majority is reachable, fail fast (retryable 503 naming the +//! laggards) when it is not, never disturb `ack=leader` traffic, and +//! recover after heal. +//! 2. **The ledger checker (the m11p3 exit gate)** — SIGKILL the leader +//! under concurrent `ack=quorum` load, across many distinct kill points, +//! and prove ZERO acknowledged-write loss: every write the client saw a +//! 2xx + `x-tidal-seq` for is present on the promoted survivor. +//! +//! The proof is two-layered per kill point: +//! - **Frontier**: a quorum ack for seqno S means some follower's +//! CONTIGUOUS applied frontier reached S (durably — m11p3 acks are +//! post-apply). The operator rule "promote the max-applied survivor" +//! therefore guarantees the promoted node holds EVERY acked seqno: +//! `max(acked seq) <= max(survivor applied_events)` is asserted before +//! the promote. +//! - **Content**: every acked ITEM is found via `/search` on the new +//! leader (the frontier can't lie about data it doesn't have, but this +//! catches a frontier that lies about data it has). +//! +//! Kill-point count: `TIDAL_QUORUM_KILLPOINTS` (default 8 for CI; the +//! exit-gate run is 100 — see docs/planning/milestone-11/phase-3.md for +//! the recorded run). +//! +//! Run: `cargo test -p tidal-server --features cluster-e2e --test cluster_quorum -- --nocapture` + +#![cfg(feature = "cluster-e2e")] +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::cast_possible_truncation, + clippy::cast_precision_loss, + clippy::too_many_lines +)] + +mod support; + +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; +use std::time::{Duration, Instant}; + +use support::{ + multiproc::{BREAKER_RESET, ClusterOptions, MultiProcCluster, convergence_budget}, + partition::proxied_rewrite, +}; + +const LEADER: usize = 0; + +/// CI-default kill points; the exit-gate run sets `TIDAL_QUORUM_KILLPOINTS=100`. +fn killpoints() -> usize { + std::env::var("TIDAL_QUORUM_KILLPOINTS") + .ok() + .and_then(|v| v.parse().ok()) + .filter(|&n| n > 0) + .unwrap_or(8) +} + +/// A unique all-alpha search token for an entity id (digits 0-9 → letters +/// a-j), so `/search?query=` is an exact item-presence probe under +/// the default tokenizer. +fn item_token(entity_id: u64) -> String { + let mut token = String::from("kpq"); + for d in entity_id.to_string().bytes() { + token.push(char::from(b'a' + (d - b'0'))); + } + token +} + +/// POST with the `x-tidal-ack` header through a dedicated client. Returns +/// `Some(seq)` only for a 2xx carrying `x-tidal-seq` — the ledger's +/// definition of "acknowledged". +fn post_acked( + client: &reqwest::blocking::Client, + base: &str, + path: &str, + ack: &str, + body: &serde_json::Value, +) -> Option { + let resp = client + .post(format!("{base}{path}")) + .header("x-tidal-ack", ack) + .json(body) + .send() + .ok()?; + if !resp.status().is_success() { + return None; + } + resp.headers() + .get("x-tidal-seq")? + .to_str() + .ok()? + .parse() + .ok() +} + +/// m11p3 quorum semantics over a REAL 3-process cluster with real TCP +/// partitions: +/// +/// - healthy: `ack=quorum` 204/201 + seq header; the leader's `commit_index` +/// tracks the writes; +/// - ONE follower severed: quorum (2 of 3) still commits via the other; +/// - BOTH followers severed: quorum 503s fast naming the laggards while +/// `ack=leader` writes keep succeeding (the knob's cost is the caller's +/// choice, never the deployment's); +/// - healed: quorum commits again. +#[test] +fn mp_quorum_writes_gate_and_recover_under_partition() { + let (rewrite, proxies) = proxied_rewrite(&["eu-west", "ap-south"]); + let cluster = MultiProcCluster::start_with(ClusterOptions::new(3).with_rewrite(rewrite)); + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(8)) + .build() + .unwrap(); + let leader_base = cluster.node(LEADER); + + // ── Healthy majority: quorum writes commit ────────────────────────────── + let seq = post_acked( + &client, + &leader_base, + "/items", + "quorum", + &serde_json::json!({ "entity_id": 1, "metadata": { "title": "quorum one" } }), + ) + .expect("healthy-cluster quorum item write must ack"); + let view_seq = post_acked( + &client, + &leader_base, + "/signals", + "quorum", + &serde_json::json!({ "entity_id": 1, "signal": "view", "weight": 1.0 }), + ) + .expect("healthy-cluster quorum signal write must ack"); + assert!(view_seq > seq, "one log: signal follows the item"); + let status = cluster.local_status(LEADER).unwrap(); + assert!( + status["commit_index"].as_u64().unwrap() >= view_seq, + "a quorum ack is at or below the commit index: {status}" + ); + println!("[quorum] healthy: item seq={seq}, view seq={view_seq} committed"); + + // ── One follower down: 2-of-3 majority still commits ──────────────────── + proxies.region("ap-south").sever_grpc(); + let seq = post_acked( + &client, + &leader_base, + "/signals", + "quorum", + &serde_json::json!({ "entity_id": 1, "signal": "view", "weight": 2.0 }), + ) + .expect("quorum must survive a single-follower outage (majority intact)"); + println!("[quorum] ap-south severed: quorum still commits (seq={seq})"); + + // ── Both followers down: quorum 503s naming the laggards ──────────────── + proxies.region("eu-west").sever_grpc(); + let resp = client + .post(format!("{leader_base}/signals")) + .header("x-tidal-ack", "quorum") + .json(&serde_json::json!({ "entity_id": 1, "signal": "view", "weight": 3.0 })) + .send() + .unwrap(); + assert_eq!( + resp.status().as_u16(), + 503, + "no majority reachable: the quorum write must fail fast" + ); + let body: serde_json::Value = resp.json().unwrap(); + assert_eq!(body["retryable"].as_bool(), Some(true)); + let laggards: Vec<&str> = body["laggards"] + .as_array() + .unwrap() + .iter() + .map(|v| v.as_str().unwrap()) + .collect(); + assert!( + laggards.contains(&"eu-west") || laggards.contains(&"ap-south"), + "the 503 names the lagging followers: {body}" + ); + // The leader-ack contract is untouched by the followers' outage. + let resp = client + .post(format!("{leader_base}/signals")) + .header("x-tidal-ack", "leader") + .json(&serde_json::json!({ "entity_id": 1, "signal": "view", "weight": 4.0 })) + .send() + .unwrap(); + assert_eq!( + resp.status().as_u16(), + 204, + "ack=leader writes keep succeeding through a follower outage" + ); + println!("[quorum] both severed: quorum 503 named {laggards:?}; leader-ack still 204"); + + // ── Heal: quorum recovers (drive through the breaker window) ──────────── + proxies.region("eu-west").heal_all(); + proxies.region("ap-south").heal_all(); + for region in ["eu-west", "ap-south"] { + let resp = client + .post(format!("{leader_base}/cluster/heal")) + .json(&serde_json::json!({ "region": region })) + .send() + .unwrap(); + assert_eq!(resp.status().as_u16(), 200); + } + let deadline = Instant::now() + BREAKER_RESET + convergence_budget(); + let mut healed_seq = None; + while healed_seq.is_none() { + assert!( + Instant::now() <= deadline, + "healed quorum writes must commit within the breaker+convergence budget" + ); + healed_seq = post_acked( + &client, + &leader_base, + "/signals", + "quorum", + &serde_json::json!({ "entity_id": 1, "signal": "view", "weight": 5.0 }), + ); + if healed_seq.is_none() { + // Re-issue the heal: the breaker can swallow the first post-heal + // ships (the documented runbook loop). + for region in ["eu-west", "ap-south"] { + let _ = client + .post(format!("{leader_base}/cluster/heal")) + .json(&serde_json::json!({ "region": region })) + .send(); + } + std::thread::sleep(Duration::from_millis(250)); + } + } + println!( + "[quorum] healed: quorum commits again (seq={})", + healed_seq.unwrap() + ); +} + +/// THE m11p3 EXIT GATE: SIGKILL the leader under `ack=quorum` load at many +/// distinct kill points; the ledger checker proves zero acknowledged loss on +/// the promoted (max-applied) survivor every time. See the module docs for +/// the two-layer proof. +#[test] +fn mp_quorum_ledger_zero_acked_loss_across_killpoints() { + let rounds = killpoints(); + println!("[ledger] running {rounds} leader-kill points (TIDAL_QUORUM_KILLPOINTS to widen)"); + + for round in 0..rounds { + let mut cluster = MultiProcCluster::start(3); + let leader_base = cluster.node(LEADER); + let gateway_bases = [ + cluster.node(LEADER), + cluster.node(1), // forwarded quorum writes through a follower + ]; + + // ── Concurrent quorum writers, ledger = client-observed acks ─────── + let stop = Arc::new(AtomicBool::new(false)); + let id_base = 1_000 * (round as u64 + 1); + let mut writers = Vec::new(); + for (w, base) in gateway_bases.iter().enumerate() { + let base = base.clone(); + let stop = Arc::clone(&stop); + writers.push(std::thread::spawn(move || { + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(4)) + .build() + .unwrap(); + // (entity_id, item_seq, view_seq-if-acked) + let mut acked: Vec<(u64, u64, Option)> = Vec::new(); + let mut n = 0u64; + while !stop.load(Ordering::Acquire) { + let entity_id = id_base + (w as u64) * 500 + n; + n += 1; + let Some(item_seq) = post_acked( + &client, + &base, + "/items", + "quorum", + &serde_json::json!({ + "entity_id": entity_id, + "metadata": { "title": item_token(entity_id) } + }), + ) else { + // Not acknowledged (leader dying/dead, forward failed, + // or quorum timeout): by contract it owes us nothing. + continue; + }; + let view_seq = post_acked( + &client, + &base, + "/signals", + "quorum", + &serde_json::json!({ + "entity_id": entity_id, "signal": "view", "weight": 1.0 + }), + ); + acked.push((entity_id, item_seq, view_seq)); + } + acked + })); + } + + // Pseudo-random kill point: spread across boot-warm, mid-burst, and + // saturated states deterministically (reproducible per round). + let kill_after = Duration::from_millis(120 + (round as u64 * 97) % 480); + std::thread::sleep(kill_after); + cluster.kill_hard(LEADER); + stop.store(true, Ordering::Release); + let _ = client_drain(&leader_base); // flush any half-open socket + let mut ledger: Vec<(u64, u64, Option)> = Vec::new(); + for w in writers { + ledger.extend(w.join().expect("writer thread")); + } + let max_acked_seq = ledger + .iter() + .map(|(_, item_seq, view_seq)| view_seq.unwrap_or(*item_seq).max(*item_seq)) + .max() + .unwrap_or(0); + + // ── Operator rule: promote the max-applied survivor ──────────────── + let survivors = [1usize, 2usize]; + let applied: Vec<(usize, u64)> = survivors + .iter() + .map(|&idx| { + let status = cluster + .local_status(idx) + .expect("survivor must serve status"); + (idx, status["applied_events"].as_u64().unwrap()) + }) + .collect(); + let (chosen, chosen_applied) = applied + .iter() + .copied() + .max_by_key(|&(_, a)| a) + .expect("two survivors"); + + // ── INVARIANT A (frontier): no acked seqno above the chosen + // survivor's contiguous durable frontier ─────────────────────────── + assert!( + max_acked_seq <= chosen_applied, + "round {round}: ACKNOWLEDGED LOSS — max acked seq {max_acked_seq} exceeds the \ + max-applied survivor's frontier {chosen_applied} (applied: {applied:?}, \ + {} acked writes)", + ledger.len() + ); + + let new_leader = cluster.region_name(chosen).to_string(); + let resp = cluster.post( + chosen, + "/cluster/promote", + &serde_json::json!({ "region": new_leader }), + ); + assert_eq!(resp.status().as_u16(), 200, "round {round}: promote"); + cluster.wait_leader_agreed(&new_leader, Duration::from_secs(10)); + + // ── INVARIANT B (content): every acked item is on the new leader ─── + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(4)) + .build() + .unwrap(); + // The text index auto-commits every 2s (engine default), so the FIRST + // probe polls past the commit interval; data presence is what is + // asserted, not commit timing. + let new_leader_base = cluster.node(chosen); + let search_deadline = Instant::now() + Duration::from_secs(10); + for (entity_id, item_seq, _) in &ledger { + let token = item_token(*entity_id); + // Definitely assigned: the loop body's first statement writes it + // before any break can be reached. + let mut last: serde_json::Value; + let present = loop { + last = client + .get(format!("{new_leader_base}/search?query={token}&limit=5")) + .send() + .unwrap() + .json() + .unwrap(); + let hit = last["items"] + .as_array() + .unwrap_or(&Vec::new()) + .iter() + .any(|it| it["entity_id"].as_u64() == Some(*entity_id)); + if hit { + break true; + } + if Instant::now() > search_deadline { + break false; + } + std::thread::sleep(Duration::from_millis(200)); + }; + assert!( + present, + "round {round}: ACKNOWLEDGED LOSS — item {entity_id} (seq {item_seq}, \ + acked at quorum) is missing on promoted leader {new_leader}: {last}" + ); + } + println!( + "[ledger] round {round}: kill@{kill_after:?} → {} acked writes (max seq \ + {max_acked_seq}) all present on {new_leader} (applied {chosen_applied})", + ledger.len() + ); + drop(cluster); + } +} + +/// Issue one throwaway GET so a dead leader's half-open client sockets are +/// observed closed before the ledger math (keeps the teardown deterministic +/// on macOS, where a killed process's sockets can linger in the client pool). +fn client_drain(base: &str) -> Option<()> { + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_millis(300)) + .build() + .ok()?; + let _ = client.get(format!("{base}/health/live")).send(); + Some(()) +} diff --git a/tidal-server/tests/cluster_region.rs b/tidal-server/tests/cluster_region.rs index 7351f10..3d1f71e 100644 --- a/tidal-server/tests/cluster_region.rs +++ b/tidal-server/tests/cluster_region.rs @@ -921,3 +921,261 @@ fn region_node_lag_honest_across_promote() { rt.shutdown_timeout(Duration::from_secs(2)); } + +/// m11p3 `ack=quorum` end to end over real gRPC, 2-node shape (majority = the +/// leader plus THE follower): +/// +/// 1. With the follower live, a quorum write 204s and carries `x-tidal-seq`. +/// 2. With the follower partitioned (ship-skipped), a quorum write returns +/// the retryable 503 naming the follower as the laggard — while an +/// `x-tidal-ack: leader` override on the SAME cluster still 204s (the +/// leader-ack contract is untouched by a follower outage). +/// 3. After heal, quorum writes 204 again and the leader's `commit_index` +/// catches its `last_seq`. +#[test] +fn region_node_quorum_write_gates_on_follower_durability() { + let pair = Pair::new(); + // The same spec both nodes parse, with the quorum deployment default and + // a short budget so the partitioned case fails fast (TopologySpec is not + // Clone; build it per node). + let quorum_topology = || { + let mut t = pair.topology(); + t.replication.ack = Some("quorum".into()); + t.replication.quorum_timeout_ms = Some(400); + t + }; + + let leader_dir = region_dir(); + let leader = build_region(quorum_topology(), &pair.leader_name, &leader_dir); + let follower_dir = region_dir(); + let follower = build_region(quorum_topology(), &pair.follower_name, &follower_dir); + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + serve( + &rt, + build_region_router(Arc::new(leader), None), + pair.leader_http, + ); + serve( + &rt, + build_region_router(Arc::new(follower), None), + pair.follower_http, + ); + + let client = reqwest::blocking::Client::new(); + let leader_base = format!("http://{}", pair.leader_http); + + // ── 1. Live follower: quorum write succeeds with a seq header ────────── + let resp = client + .post(format!("{leader_base}/signals")) + .json(&serde_json::json!({ "entity_id": 1, "signal": "view", "weight": 1.0 })) + .send() + .unwrap(); + assert_eq!( + resp.status().as_u16(), + 204, + "quorum write with a live follower must succeed" + ); + let seq: u64 = resp + .headers() + .get("x-tidal-seq") + .expect("a quorum-acked write carries its log seqno") + .to_str() + .unwrap() + .parse() + .unwrap(); + assert!(seq > 0, "the assigned seqno is a real stream position"); + + // The leader's status shows the commit index covering the write. + let status = poll_status(&client, &leader_base, |_, _| true); + assert!( + status["commit_index"].as_u64().unwrap() >= seq, + "a 204'd quorum write is at or below the commit index: {status}" + ); + assert_eq!(status["ack"].as_str(), Some("quorum")); + + // ── 2. Partitioned follower: quorum 503 names the laggard ────────────── + let resp = client + .post(format!("{leader_base}/cluster/partition")) + .json(&serde_json::json!({ "region": pair.follower_name })) + .send() + .unwrap(); + assert_eq!(resp.status().as_u16(), 200); + + let resp = client + .post(format!("{leader_base}/signals")) + .json(&serde_json::json!({ "entity_id": 2, "signal": "view", "weight": 1.0 })) + .send() + .unwrap(); + assert_eq!( + resp.status().as_u16(), + 503, + "a quorum write cannot commit while THE follower is partitioned (n=2)" + ); + let body: serde_json::Value = resp.json().unwrap(); + assert_eq!(body["retryable"].as_bool(), Some(true)); + assert_eq!( + body["laggards"], + serde_json::json!([pair.follower_name]), + "the 503 names the laggard: {body}" + ); + assert_eq!(body["needed"].as_u64(), Some(1)); + + // The caller's per-request override still gets leader-ack semantics. + let resp = client + .post(format!("{leader_base}/signals")) + .header("x-tidal-ack", "leader") + .json(&serde_json::json!({ "entity_id": 3, "signal": "view", "weight": 1.0 })) + .send() + .unwrap(); + assert_eq!( + resp.status().as_u16(), + 204, + "x-tidal-ack: leader bypasses the quorum gate during the outage" + ); + + // An unknown ack mode is a 400, not a silent default. + let resp = client + .post(format!("{leader_base}/signals")) + .header("x-tidal-ack", "everyone") + .json(&serde_json::json!({ "entity_id": 4, "signal": "view", "weight": 1.0 })) + .send() + .unwrap(); + assert_eq!(resp.status().as_u16(), 400, "invalid x-tidal-ack is a 400"); + + // ── 3. Heal: quorum writes commit again ──────────────────────────────── + let resp = client + .post(format!("{leader_base}/cluster/heal")) + .json(&serde_json::json!({ "region": pair.follower_name })) + .send() + .unwrap(); + assert_eq!(resp.status().as_u16(), 200); + + // Retry the quorum write until the healed pipeline commits one (the heal + // resume + the durable ack fold may need a retry tick). + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let resp = client + .post(format!("{leader_base}/signals")) + .json(&serde_json::json!({ "entity_id": 5, "signal": "view", "weight": 1.0 })) + .send() + .unwrap(); + if resp.status().as_u16() == 204 { + break; + } + assert!( + Instant::now() <= deadline, + "healed quorum writes must commit within 5s (last: {})", + resp.status() + ); + std::thread::sleep(Duration::from_millis(50)); + } + let status = poll_status(&client, &leader_base, |_, _| true); + let last_seq = status["last_seq"].as_u64().unwrap(); + let commit = status["commit_index"].as_u64().unwrap(); + assert!( + commit >= last_seq.saturating_sub(1), + "post-heal the commit index tracks the flushed frontier: {status}" + ); + + rt.shutdown_timeout(Duration::from_secs(2)); +} + +/// m11p3: a quorum write THROUGH a follower gateway — the `x-tidal-ack` +/// override travels with the forward, the leader gates on quorum, and the +/// `x-tidal-seq` response header relays back to the original caller. Also +/// covers items + embeddings (kind-1/2 records gate on the same commit index). +#[test] +fn region_node_quorum_forward_and_blob_writes() { + let pair = Pair::new(); + + let leader_dir = region_dir(); + let leader = build_region(pair.topology(), &pair.leader_name, &leader_dir); + let follower_dir = region_dir(); + let follower = build_region(pair.topology(), &pair.follower_name, &follower_dir); + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + serve( + &rt, + build_region_router(Arc::new(leader), None), + pair.leader_http, + ); + serve( + &rt, + build_region_router(Arc::new(follower), None), + pair.follower_http, + ); + + let client = reqwest::blocking::Client::new(); + let leader_base = format!("http://{}", pair.leader_http); + let follower_base = format!("http://{}", pair.follower_http); + + // Quorum item via the FOLLOWER gateway (topology default is leader-ack; + // the header overrides through the forward). + let resp = client + .post(format!("{follower_base}/items")) + .header("x-tidal-ack", "quorum") + .json(&serde_json::json!({ + "entity_id": 7, + "metadata": { "title": "quorum item via follower" } + })) + .send() + .unwrap(); + assert_eq!( + resp.status().as_u16(), + 201, + "a forwarded quorum item write must succeed" + ); + let item_seq: u64 = resp + .headers() + .get("x-tidal-seq") + .expect("the leader's seq header relays through the forward") + .to_str() + .unwrap() + .parse() + .unwrap(); + assert!(item_seq > 0); + + // Quorum embedding straight at the leader. + let resp = client + .post(format!("{leader_base}/embeddings")) + .header("x-tidal-ack", "quorum") + .json(&serde_json::json!({ "entity_id": 7, "values": [0.1, 0.2, 0.3, 0.4] })) + .send() + .unwrap(); + assert_eq!(resp.status().as_u16(), 204); + let emb_seq: u64 = resp + .headers() + .get("x-tidal-seq") + .expect("embedding writes carry their seq too") + .to_str() + .unwrap() + .parse() + .unwrap(); + assert!( + emb_seq > item_seq, + "one log: the embedding's seqno follows the item's ({item_seq} -> {emb_seq})" + ); + + // The quorum-acked writes are durable on the follower BY CONTRACT — + // its applied frontier already covers them (no convergence poll needed, + // that is the whole point of ack=quorum). + let follower_status: serde_json::Value = client + .get(format!("{follower_base}/cluster/status/local")) + .send() + .unwrap() + .json() + .unwrap(); + assert!( + follower_status["applied_events"].as_u64().unwrap() >= emb_seq, + "a 2-node quorum ack means THE follower durably applied it: {follower_status}" + ); + + rt.shutdown_timeout(Duration::from_secs(2)); +} diff --git a/tidal-stress/src/client.rs b/tidal-stress/src/client.rs index 721b97b..4ab3295 100644 --- a/tidal-stress/src/client.rs +++ b/tidal-stress/src/client.rs @@ -22,12 +22,19 @@ use crate::workload::{HttpMethod, Plan}; pub struct HttpClient { inner: reqwest::Client, api_key: Option, + /// `x-tidal-ack` value sent with every write (m11p3: `leader`/`quorum`); + /// `None` = the deployment's topology default. + ack: Option, } impl HttpClient { /// `request_timeout` should sit ABOVE the server's 30s request timeout so the /// server's own 408 surfaces as a Timeout class rather than a client abort. - pub fn new(request_timeout: Duration, api_key: Option) -> Result { + pub fn new( + request_timeout: Duration, + api_key: Option, + ack: Option, + ) -> Result { let inner = reqwest::Client::builder() .timeout(request_timeout) .connect_timeout(Duration::from_secs(5)) @@ -39,7 +46,11 @@ impl HttpClient { .tcp_nodelay(true) .build() .map_err(StressError::Client)?; - Ok(Self { inner, api_key }) + Ok(Self { + inner, + api_key, + ack, + }) } fn apply_auth(&self, rb: reqwest::RequestBuilder) -> reqwest::RequestBuilder { @@ -55,7 +66,10 @@ impl HttpClient { let rb = match plan.method { HttpMethod::Get => self.inner.get(&plan.url), HttpMethod::Post => { - let rb = self.inner.post(&plan.url); + let mut rb = self.inner.post(&plan.url); + if let Some(ack) = &self.ack { + rb = rb.header("x-tidal-ack", ack); + } match &plan.body { Some(b) => rb.json(b), None => rb, diff --git a/tidal-stress/src/main.rs b/tidal-stress/src/main.rs index 101ef08..cd56eaa 100644 --- a/tidal-stress/src/main.rs +++ b/tidal-stress/src/main.rs @@ -49,6 +49,11 @@ struct Cli { #[arg(long, env = "TIDAL_API_KEY")] api_key: Option, + /// Write acknowledgment mode sent as `x-tidal-ack` on every write + /// (m11p3: leader|quorum). Omitted = the cluster's topology default. + #[arg(long)] + ack: Option, + /// Ramp: a preset (smoke|quick|peach-100k|max) or `rps:secs,rps:secs,...`. #[arg(long, default_value = "peach-100k")] ramp: String, @@ -163,9 +168,17 @@ async fn run() -> Result<()> { }; let mix = parse_mix(&cli.mix)?; let stages = parse_ramp(&cli.ramp, cli.stage_secs)?; + if let Some(ack) = cli.ack.as_deref() + && !matches!(ack, "leader" | "quorum") + { + return Err(StressError::Target(format!( + "--ack must be leader|quorum, got {ack:?}" + ))); + } let client = Arc::new(HttpClient::new( Duration::from_secs(cli.request_timeout_secs), cli.api_key.clone(), + cli.ack.clone(), )?); // The leader URL anchors seeding (items must broadcast from the leader so diff --git a/tidal/src/db/items.rs b/tidal/src/db/items.rs index 098bd6d..be4bce0 100644 --- a/tidal/src/db/items.rs +++ b/tidal/src/db/items.rs @@ -18,7 +18,9 @@ impl TidalDb { /// /// A no-op outside cluster mode (`replicate_blobs` false) and in /// ephemeral mode (no WAL): single-node items keep their fjall-only - /// durability, paying zero extra fsyncs. + /// durability, paying zero extra fsyncs. Returns the record's assigned + /// WAL seqno when journaled (`None` on the no-op paths) — blobs skip the + /// dedup window, so the seqno is never the dedup sentinel. /// /// `record` is built lazily so non-cluster writes never pay the /// serialization. @@ -28,9 +30,9 @@ impl TidalDb { /// `TidalError::Durability` when the WAL append cannot be staged or its /// fsync fails — the caller must NOT apply to storage in that case (a /// storage-applied, never-logged item would silently never replicate). - fn wal_blob_first(&self, record: impl FnOnce() -> BlobRecord) -> crate::Result<()> { + fn wal_blob_first(&self, record: impl FnOnce() -> BlobRecord) -> crate::Result> { if !self.replicate_blobs { - return Ok(()); + return Ok(None); } let sender = { let wal = self @@ -42,20 +44,22 @@ impl TidalDb { let Some(sender) = sender else { // Ephemeral cluster nodes have no WAL to log to (and no segments // to serve catch-up from); their items are memory-only anyway. - return Ok(()); + return Ok(None); }; - let pending = sender.append_blob_staged(record()).map_err(|e| { - TidalError::Durability(crate::schema::DurabilityError { - message: format!("WAL blob append staging failed: {e}"), - }) - })?; + let pending = sender + .append_blob_staged(std::sync::Arc::new(record())) + .map_err(|e| { + TidalError::Durability(crate::schema::DurabilityError { + message: format!("WAL blob append staging failed: {e}"), + }) + })?; let seq = pending.wait().map_err(|e| { TidalError::Durability(crate::schema::DurabilityError { message: format!("WAL blob append failed: {e}"), }) })?; super::wal_bridge::bump_last_seq_atomic(&self.last_wal_seq, seq); - Ok(()) + Ok(Some(seq)) } /// Write (or overwrite) item metadata and update in-memory indexes. /// @@ -96,7 +100,7 @@ impl TidalDb { &self, id: EntityId, metadata: &HashMap, - ) -> crate::Result<()> { + ) -> crate::Result> { self.require_writeable("write_item_with_metadata")?; Self::validate_item_write(id, metadata)?; let stored_metadata = Self::metadata_with_created_at(id, metadata); @@ -108,14 +112,15 @@ impl TidalDb { // followers receive. The record carries the STORED map (with the // materialized `created_at`), so every replica persists and indexes // the identical metadata. - self.wal_blob_first(|| { + let seq = self.wal_blob_first(|| { BlobRecord::ItemMetadata(ItemMetadataRecord { entity_id: id.as_u64(), metadata_bytes: super::metadata::serialize_metadata(&stored_metadata), }) })?; - self.apply_item_metadata_indexed(id, &stored_metadata) + self.apply_item_metadata_indexed(id, &stored_metadata)?; + Ok(seq) } /// Apply item metadata locally WITHOUT a WAL append: the replay path for @@ -138,6 +143,132 @@ impl TidalDb { self.apply_item_metadata_indexed(id, &stored_metadata) } + /// Apply one replicated apply-round's blob records as a BATCH (m11p3): + /// validate every record first (an invalid record halts the round before + /// anything is journaled), then stage EVERY record's WAL append before + /// waiting — the whole round shares the writer's group-commit fsyncs — + /// then upsert storage in record order. The follower-side counterpart of + /// the leader's `wal_blob_first` discipline: a record-at-a-time apply + /// paid one solo fsync per item, capping item apply at the fsync floor. + /// + /// Takes the records by value: they move into `Arc`s shared with the WAL + /// writer by refcount, so the apply path never deep-clones a + /// metadata/embedding buffer. + /// + /// # Errors + /// + /// Validation, durability, or storage errors; the segment receiver HALTS + /// on any of them (blobs are idempotent upserts, so redelivery after a + /// mid-batch halt re-applies safely). On a durability error every append + /// staged by this call has still been WAITED — no staged blob is left + /// unresolved behind the halt; the first error is returned. + pub(crate) fn apply_replicated_blobs(&self, records: Vec) -> crate::Result<()> { + /// One validated record, parsed exactly once in Phase 1 and applied + /// in Phase 3 (item metadata is deserialized here, never re-parsed). + enum BlobApply<'a> { + Item { + id: EntityId, + metadata: HashMap, + }, + Embedding { + id: EntityId, + values: &'a [f32], + }, + } + + self.require_writeable("apply_replicated_blobs")?; + let records: Vec> = + records.into_iter().map(std::sync::Arc::new).collect(); + + // Phase 1 — validate ALL records before journaling ANY, capturing + // each record's parsed form for the Phase 3 upserts. + let mut applies: Vec> = Vec::with_capacity(records.len()); + for record in &records { + match &**record { + BlobRecord::ItemMetadata(r) => { + let id = EntityId::new(r.entity_id); + let metadata = deserialize_metadata(&r.metadata_bytes); + Self::validate_item_write(id, &metadata)?; + applies.push(BlobApply::Item { id, metadata }); + } + BlobRecord::Embedding(r) => { + Self::validate_embedding_for_log(&r.values)?; + applies.push(BlobApply::Embedding { + id: EntityId::new(r.entity_id), + values: &r.values, + }); + } + } + } + + // Phase 2 — WAL-first, batched: stage every append, then wait every + // fsync (the writer coalesces the staged blobs into shared group + // syncs). Skipped outside cluster mode / without a WAL, exactly like + // `wal_blob_first`. + if self.replicate_blobs { + let sender = { + let wal = self + .wal + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + wal.as_ref().map(crate::wal::WalHandle::sender) + }; + if let Some(sender) = sender { + let mut pending = Vec::with_capacity(records.len()); + let mut first_err: Option = None; + for record in &records { + match sender.append_blob_staged(std::sync::Arc::clone(record)) { + Ok(p) => pending.push(p), + Err(e) => { + // Stop staging, but fall through to wait what IS + // staged: every append this call enqueued must be + // resolved before the round returns, or a + // mid-batch failure leaves blobs durably queued + // behind a halted receiver with nobody to observe + // their outcome. + first_err = + Some(TidalError::Durability(crate::schema::DurabilityError { + message: format!("WAL blob append staging failed: {e}"), + })); + break; + } + } + } + for p in pending { + match p.wait() { + Ok(seq) => { + super::wal_bridge::bump_last_seq_atomic(&self.last_wal_seq, seq); + } + Err(e) => { + if first_err.is_none() { + first_err = + Some(TidalError::Durability(crate::schema::DurabilityError { + message: format!("WAL blob append failed: {e}"), + })); + } + } + } + } + if let Some(e) = first_err { + return Err(e); + } + } + } + + // Phase 3 — storage upserts, in record order (parsed in Phase 1). + for apply in applies { + match apply { + BlobApply::Item { id, metadata } => { + self.apply_item_metadata_local(id, &metadata)?; + } + BlobApply::Embedding { id, values } => { + self.apply_item_embedding_local(id, values)?; + } + } + } + Ok(()) + } + /// Up-front validation shared by the write and replay paths: the u32 /// item-universe guard and the metadata size limits. Runs BEFORE the WAL /// append so an invalid write is rejected without ever entering the @@ -421,7 +552,11 @@ impl TidalDb { /// - `TidalError::Internal` if no storage backend is wired or lock is poisoned. /// - `TidalError::Storage` on storage engine failure. /// - `TidalError::Internal` if the embedding has zero norm. - pub fn write_item_embedding(&self, id: EntityId, embedding: &[f32]) -> crate::Result<()> { + pub fn write_item_embedding( + &self, + id: EntityId, + embedding: &[f32], + ) -> crate::Result> { self.require_writeable("write_item_embedding")?; Self::validate_embedding_for_log(embedding)?; @@ -429,14 +564,15 @@ impl TidalDb { // The record carries the CALLER's raw values: every replica runs the // identical deterministic L2-normalization in `write_entity_embedding`, // so persisted bytes converge without shipping normalized floats. - self.wal_blob_first(|| { + let seq = self.wal_blob_first(|| { BlobRecord::Embedding(EmbeddingRecord { entity_id: id.as_u64(), values: embedding.to_vec(), }) })?; - self.apply_item_embedding_local(id, embedding) + self.apply_item_embedding_local(id, embedding)?; + Ok(seq) } /// Apply an item embedding locally WITHOUT a WAL append: the replay path diff --git a/tidal/src/db/metrics/cluster.rs b/tidal/src/db/metrics/cluster.rs index e09049f..17fef0b 100644 --- a/tidal/src/db/metrics/cluster.rs +++ b/tidal/src/db/metrics/cluster.rs @@ -77,8 +77,14 @@ pub struct ClusterMetrics { write_pool_rejections_total: AtomicU64, /// The relay's last committed seqno (leader stream high-water mark). relay_last_seq: AtomicU64, - /// The relay's contiguous leader-durable frontier. + /// The quorum commit index (m11p3) — the highest seqno a majority of the + /// replica set durably holds. Pre-m11p3 this gauge carried the leader's + /// own durable frontier; the name is kept for dashboard continuity, and + /// `relay_last_seq - relay_durable_seq` is now the cluster's quorum lag. relay_durable_seq: AtomicU64, + /// Total `ack=quorum` writes that timed out awaiting the commit index + /// (each returned a retryable 503 naming the laggards). + quorum_timeouts_total: AtomicU64, } impl ClusterMetrics { @@ -94,6 +100,7 @@ impl ClusterMetrics { write_pool_rejections_total: AtomicU64::new(0), relay_last_seq: AtomicU64::new(0), relay_durable_seq: AtomicU64::new(0), + quorum_timeouts_total: AtomicU64::new(0), } } @@ -172,12 +179,18 @@ impl ClusterMetrics { self.group_commit_events.observe(batch_events as u64); } - /// Update the relay stream gauges (committed + durable frontiers). + /// Update the stream gauges: the flushed high-water mark and the quorum + /// commit index (m11p3). pub fn set_relay_frontiers(&self, last_seq: u64, durable_seq: u64) { self.relay_last_seq.store(last_seq, Ordering::Relaxed); self.relay_durable_seq.store(durable_seq, Ordering::Relaxed); } + /// Count an `ack=quorum` write that timed out awaiting the commit index. + pub fn incr_quorum_timeouts(&self) { + self.quorum_timeouts_total.fetch_add(1, Ordering::Relaxed); + } + /// Report the server write pool's current queue depth. pub fn set_write_pool_depth(&self, depth: u64) { self.write_pool_depth.store(depth, Ordering::Relaxed); @@ -235,10 +248,17 @@ impl ClusterMetrics { super::write_metric_line( out, "tidaldb_cluster_relay_durable_seq", - "Leader relay contiguous durable frontier (fsynced prefix)", + "Quorum commit index: highest seqno a majority of the replica set durably holds (m11p3)", "gauge", self.relay_durable_seq.load(Ordering::Relaxed) as f64, ); + super::write_metric_line( + out, + "tidaldb_cluster_quorum_timeouts_total", + "ack=quorum writes that timed out awaiting the commit index (retryable 503s)", + "counter", + self.quorum_timeouts_total.load(Ordering::Relaxed) as f64, + ); // Per-peer series, labeled by peer shard id + this node's partition. // Snapshot the cells under the read lock, then render lock-free (the diff --git a/tidal/src/db/replication_ops.rs b/tidal/src/db/replication_ops.rs index 08de836..6745ffb 100644 --- a/tidal/src/db/replication_ops.rs +++ b/tidal/src/db/replication_ops.rs @@ -523,17 +523,11 @@ impl WeakBlobApplier { } impl crate::replication::receiver::ReplicatedBlobApplier for WeakBlobApplier { - fn apply_item_metadata(&self, entity_id: u64, metadata_bytes: &[u8]) -> crate::Result<()> { - let db = self.upgrade()?; - let metadata = crate::db::metadata::deserialize_metadata(metadata_bytes); + fn apply_blobs(&self, records: Vec) -> crate::Result<()> { // The FULL write path, not the local-apply variant: the follower - // re-journals the record in its OWN WAL (WAL-first) so follower - // recovery — and a later promotion's outbound stream — carry it. - db.write_item_with_metadata(crate::schema::EntityId::new(entity_id), &metadata) - } - - fn apply_embedding(&self, entity_id: u64, values: &[f32]) -> crate::Result<()> { - let db = self.upgrade()?; - db.write_item_embedding(crate::schema::EntityId::new(entity_id), values) + // re-journals every record in its OWN WAL (WAL-first, staged as one + // batch so the round shares group-commit fsyncs — m11p3) so follower + // recovery — and a later promotion's outbound stream — carry them. + self.upgrade()?.apply_replicated_blobs(records) } } diff --git a/tidal/src/db/signals.rs b/tidal/src/db/signals.rs index 6ed4caf..5ef79f0 100644 --- a/tidal/src/db/signals.rs +++ b/tidal/src/db/signals.rs @@ -31,13 +31,18 @@ impl StagedSignal { /// in-memory aggregate (identical end state to a completed /// [`TidalDb::signal`] call, including the write-latency metrics). /// + /// Returns the event's assigned WAL seqno — the replicated-stream + /// position quorum acks gate on (m11p3). `0` = suppressed by the dedup + /// window (an identical record is already durable; its quorum status is + /// that record's, so callers must not gate on the sentinel). + /// /// # Errors /// /// Returns `TidalError::Durability` if the WAL flush failed. The in-memory /// aggregate is untouched on error. Replication-relay callers must treat /// this as fatal for their stream (the staged event's seqno is already /// woven in) — see `SignalRelay`'s poison semantics. - pub fn wait(self, db: &TidalDb) -> crate::Result<()> { + pub fn wait(self, db: &TidalDb) -> crate::Result { let result = db.ledger()?.complete_staged(self.staged); #[cfg(feature = "metrics")] @@ -77,7 +82,7 @@ impl TidalDb { id: EntityId, metadata: &HashMap, ) -> crate::Result<()> { - self.write_item_with_metadata(id, metadata) + self.write_item_with_metadata(id, metadata).map(|_seq| ()) } /// Persist item metadata directly to the items storage backend with no diff --git a/tidal/src/replication/commit.rs b/tidal/src/replication/commit.rs new file mode 100644 index 0000000..e2e9efa --- /dev/null +++ b/tidal/src/replication/commit.rs @@ -0,0 +1,481 @@ +//! Quorum commit index over per-peer durable marks (m11p3). +//! +//! The leader tracks, per peer, the highest seqno the peer has **durably +//! applied** — its own storage upserts done and its own WAL fsync complete. +//! Three durable-true inputs feed the marks: the follower's pushed frontier +//! reports (`ReportApplied`, once per apply round — fresh even when ships +//! stall), the floor hints piggybacked on ship acks +//! ([`peer_applied_hint`]), and heal resumes (the follower's own status +//! report). The **commit index** is the highest seqno durably held by a +//! majority of the replica set (leader + peers): with `n` total replicas, a +//! write at seqno `S` is committed once `floor(n/2)` peers report a durable +//! mark `>= S` (the leader itself is the remaining member of the majority — +//! its own group-commit fsync precedes shipping by construction). +//! +//! `ack=quorum` writes wait via [`CommitIndex::wait_for`] (sync) or an +//! async watch-channel bridge over [`snapshot`]/[`wait_change`] until the +//! commit index passes their seqno, their deadline expires (naming the +//! laggards), or leadership moves. The index is leadership-scoped: +//! [`activate`] resets it to the promote baseline and [`deactivate`] fails +//! every waiter — a demoted leader must never report quorum for a stream it +//! no longer owns. +//! +//! Marks advance batch-level and pipelined: one report can advance the +//! commit index across thousands of seqnos, releasing every waiter at or +//! below it. +//! +//! [`activate`]: CommitIndex::activate +//! [`deactivate`]: CommitIndex::deactivate +//! [`snapshot`]: CommitIndex::snapshot +//! [`wait_change`]: CommitIndex::wait_change +//! [`peer_applied_hint`]: crate::replication::Transport::peer_applied_hint + +use std::collections::HashMap; +use std::sync::{Condvar, Mutex, PoisonError}; +use std::time::Instant; + +use super::ShardId; + +/// Why a [`CommitIndex::wait_for`] did not observe commitment. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum QuorumWaitError { + /// Leadership moved (deactivate/re-activate) while waiting, or the queue + /// was never active. The write is durable on the old leader only; the + /// caller must not report quorum. + Demoted, + /// The deadline expired before `floor(n/2)` peers reported durable marks + /// at or past the awaited seqno. + Timeout { + /// The commit index at expiry (how far the quorum actually got). + committed: u64, + /// Peers whose durable mark was still below the awaited seqno. + laggards: Vec, + }, +} + +impl std::fmt::Display for QuorumWaitError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Demoted => write!(f, "leadership moved while awaiting quorum"), + Self::Timeout { + committed, + laggards, + } => write!( + f, + "quorum not reached before deadline (committed={committed}, laggards={laggards:?})" + ), + } + } +} + +impl std::error::Error for QuorumWaitError {} + +struct CommitInner { + /// Leadership gate: waiters on an inactive index fail with `Demoted`. + active: bool, + /// Bumped on every activate/deactivate so in-flight waiters detect a + /// leadership change that happened mid-wait (even activate→deactivate→ + /// activate cycles that end "active"). + epoch: u64, + /// Per-peer durable marks, monotonic within an epoch. + peers: HashMap, + /// Cached commit index (recomputed on every mark advance). + commit: u64, + /// Reusable buffer for the k-th-largest selection in + /// [`CommitIndex::compute_commit`] — mark folds run on every ship ack + /// and frontier report, so the recompute must not allocate per call. + scratch: Vec, +} + +/// Leader-side quorum commit index. See the module docs. +/// +/// **Leadership-scoped**: the index is only meaningful for the leadership +/// term that [`activate`](Self::activate)d it. Demotion +/// ([`deactivate`](Self::deactivate)) fails every waiter, and the epoch +/// counter unmasks an activate→deactivate→activate cycle that ends "active" +/// — a waiter never resumes into a term it did not start in. +pub struct CommitIndex { + inner: Mutex, + cv: Condvar, + /// Peers needed at-or-past a seqno for majority: `floor((peers+1)/2)`. + needed: usize, +} + +impl std::fmt::Debug for CommitIndex { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let inner = self.lock(); + f.debug_struct("CommitIndex") + .field("active", &inner.active) + .field("epoch", &inner.epoch) + .field("commit", &inner.commit) + .field("needed", &self.needed) + .field("peers", &inner.peers) + .finish_non_exhaustive() + } +} + +impl CommitIndex { + /// Build an index over `peers` (the replica set EXCLUDING the leader), + /// initially inactive with all marks at zero. + #[must_use] + pub fn new(peers: &[ShardId], active: bool) -> Self { + Self { + inner: Mutex::new(CommitInner { + active, + epoch: 0, + peers: peers.iter().map(|&p| (p, 0)).collect(), + commit: 0, + scratch: Vec::with_capacity(peers.len()), + }), + cv: Condvar::new(), + // Majority of n = peers+1 total replicas is floor(n/2)+1 nodes; + // the leader is always one of them, so floor(n/2) = ceil(p/2) + // peers remain. + needed: peers.len().div_ceil(2), + } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, CommitInner> { + self.inner.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Peers required at-or-past a seqno (beyond the leader) for majority. + #[must_use] + pub const fn needed_peers(&self) -> usize { + self.needed + } + + /// The current commit index (0 when no quorum information exists yet). + /// With zero peers the index is meaningless — callers gate on a zero + /// [`needed_peers`](Self::needed_peers) and treat leader-durable as + /// committed. + #[must_use] + pub fn committed(&self) -> u64 { + self.lock().commit + } + + /// Per-peer durable marks (for status surfaces), sorted by shard id. + #[must_use] + pub fn peer_marks(&self) -> Vec<(ShardId, u64)> { + let inner = self.lock(); + let mut marks: Vec<(ShardId, u64)> = inner.peers.iter().map(|(&p, &m)| (p, m)).collect(); + drop(inner); + marks.sort_unstable_by_key(|(p, _)| p.0); + marks + } + + /// Activate for a new leadership term starting at `baseline` (the + /// promote-time flushed frontier). Marks reset to the baseline — peers + /// jump their frontier there via the promote fan-out / catch-up + /// announcements, and nothing above it has been shipped yet. + pub fn activate(&self, baseline: u64) { + let mut inner = self.lock(); + inner.active = true; + inner.epoch += 1; + for mark in inner.peers.values_mut() { + *mark = baseline; + } + inner.commit = baseline; + drop(inner); + self.cv.notify_all(); + } + + /// Deactivate (leadership moved or the queue is shutting down): every + /// current and future waiter fails with [`QuorumWaitError::Demoted`]. + pub fn deactivate(&self) { + let mut inner = self.lock(); + inner.active = false; + inner.epoch += 1; + drop(inner); + self.cv.notify_all(); + } + + /// Fold a peer's reported durable mark (monotonic; 0 = "unknown" and is + /// ignored). Advances the commit index and wakes waiters when the k-th + /// largest mark moves. + pub fn update_peer(&self, peer: ShardId, durable: u64) { + if durable == 0 { + return; + } + let mut inner = self.lock(); + let Some(mark) = inner.peers.get_mut(&peer) else { + return; + }; + if durable <= *mark { + return; + } + *mark = durable; + let commit = self.compute_commit(&mut inner); + if commit > inner.commit { + inner.commit = commit; + drop(inner); + self.cv.notify_all(); + } + } + + /// k-th largest peer mark (k = `needed`); 0 when there are no peers + /// (callers must special-case a zero `needed` — see + /// [`committed`](Self::committed)). Selection (not a full sort) over the + /// inner scratch buffer: this runs on every ship ack and frontier + /// report, so it must not allocate or do superlinear work per fold. + fn compute_commit(&self, inner: &mut CommitInner) -> u64 { + if self.needed == 0 { + return inner.commit; + } + let mut scratch = std::mem::take(&mut inner.scratch); + scratch.clear(); + scratch.extend(inner.peers.values().copied()); + let commit = if scratch.len() < self.needed { + 0 + } else { + // k-th largest = index `needed - 1` in descending order. + *scratch + .select_nth_unstable_by(self.needed - 1, |a, b| b.cmp(a)) + .1 + }; + inner.scratch = scratch; + commit + } + + /// One coherent reading of the index: `(epoch, commit, active)`. + /// + /// Async front-ends bridge this through a watch channel (one dedicated + /// publisher thread in [`wait_change`](Self::wait_change)'s loop) so + /// request handlers can await commitment WITHOUT parking a thread each — + /// thread-per-wait exhausts a runtime's blocking pool under open-loop + /// load and starves the very completions that advance the index. + #[must_use] + pub fn snapshot(&self) -> (u64, u64, bool) { + let inner = self.lock(); + (inner.epoch, inner.commit, inner.active) + } + + /// Block until the index differs from the seen `(epoch, commit)` (a mark + /// advance, activation, deactivation) or `timeout` elapses; returns the + /// current snapshot either way. The publisher-thread half of the async + /// bridge (see [`snapshot`](Self::snapshot)). + #[must_use] + pub fn wait_change( + &self, + seen_epoch: u64, + seen_commit: u64, + timeout: std::time::Duration, + ) -> (u64, u64, bool) { + let deadline = Instant::now() + timeout; + let mut inner = self.lock(); + loop { + if inner.epoch != seen_epoch || inner.commit != seen_commit { + return (inner.epoch, inner.commit, inner.active); + } + let now = Instant::now(); + if now >= deadline { + return (inner.epoch, inner.commit, inner.active); + } + let (guard, _timed_out) = self + .cv + .wait_timeout(inner, deadline - now) + .unwrap_or_else(PoisonError::into_inner); + inner = guard; + } + } + + /// Block until the commit index reaches `seq`, the deadline passes, or + /// leadership moves. Returns the commit index that satisfied the wait. + /// + /// With zero peers (single-replica deployment) the write's own leader + /// fsync IS the majority: returns immediately — but only while the + /// index is active. The demoted-leader invariant has no replica-count + /// exception, so the active check happens under the lock BEFORE the + /// zero-peer fast path. + /// + /// # Errors + /// + /// [`QuorumWaitError::Demoted`] if the index is inactive or leadership + /// changes mid-wait; [`QuorumWaitError::Timeout`] (naming the laggard + /// peers) if the deadline expires first. + pub fn wait_for(&self, seq: u64, deadline: Instant) -> Result { + let mut inner = self.lock(); + if !inner.active { + return Err(QuorumWaitError::Demoted); + } + if self.needed == 0 { + return Ok(seq); + } + let epoch = inner.epoch; + loop { + if inner.epoch != epoch || !inner.active { + return Err(QuorumWaitError::Demoted); + } + if inner.commit >= seq { + return Ok(inner.commit); + } + let now = Instant::now(); + if now >= deadline { + let mut laggards: Vec = inner + .peers + .iter() + .filter(|&(_, &mark)| mark < seq) + .map(|(&peer, _)| peer) + .collect(); + laggards.sort_unstable_by_key(|p| p.0); + return Err(QuorumWaitError::Timeout { + committed: inner.commit, + laggards, + }); + } + let (guard, _timed_out) = self + .cv + .wait_timeout(inner, deadline - now) + .unwrap_or_else(PoisonError::into_inner); + inner = guard; + } + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] // test assertions on known-good fixtures +mod tests { + use super::*; + use std::sync::Arc; + use std::time::Duration; + + fn shards(ids: &[u16]) -> Vec { + ids.iter().map(|&i| ShardId(i)).collect() + } + + fn deadline_in(ms: u64) -> Instant { + Instant::now() + Duration::from_millis(ms) + } + + #[test] + fn majority_math_matches_replica_counts() { + // n=2 replicas (1 peer): majority 2 ⇒ THE peer must hold the write. + assert_eq!(CommitIndex::new(&shards(&[1]), true).needed_peers(), 1); + // n=3 (2 peers): majority 2 ⇒ 1 peer beyond the leader. + assert_eq!(CommitIndex::new(&shards(&[1, 2]), true).needed_peers(), 1); + // n=4 (3 peers): majority 3 ⇒ 2 peers. + assert_eq!( + CommitIndex::new(&shards(&[1, 2, 3]), true).needed_peers(), + 2 + ); + // n=5 (4 peers): majority 3 ⇒ 2 peers. + assert_eq!( + CommitIndex::new(&shards(&[1, 2, 3, 4]), true).needed_peers(), + 2 + ); + // Single replica: the leader alone is the majority. + assert_eq!(CommitIndex::new(&[], true).needed_peers(), 0); + } + + #[test] + fn commit_is_kth_largest_durable_mark() { + let idx = CommitIndex::new(&shards(&[1, 2, 3, 4]), true); // needs 2 + idx.update_peer(ShardId(1), 10); + assert_eq!(idx.committed(), 0, "one peer at 10 is not a majority"); + idx.update_peer(ShardId(2), 7); + assert_eq!(idx.committed(), 7, "2nd largest mark commits"); + idx.update_peer(ShardId(3), 12); + assert_eq!(idx.committed(), 10); + // Stale and unknown reports never regress the index. + idx.update_peer(ShardId(3), 5); + idx.update_peer(ShardId(1), 0); + assert_eq!(idx.committed(), 10); + // An unknown peer is ignored entirely. + idx.update_peer(ShardId(99), 1_000); + assert_eq!(idx.committed(), 10); + } + + #[test] + fn wait_resolves_when_quorum_arrives() { + let idx = Arc::new(CommitIndex::new(&shards(&[1, 2]), true)); // needs 1 + let waiter = { + let idx = Arc::clone(&idx); + std::thread::spawn(move || idx.wait_for(5, deadline_in(2_000))) + }; + std::thread::sleep(Duration::from_millis(20)); + idx.update_peer(ShardId(2), 8); + assert_eq!(waiter.join().unwrap(), Ok(8)); + } + + #[test] + fn wait_timeout_names_the_laggards() { + let idx = CommitIndex::new(&shards(&[1, 2]), true); + idx.update_peer(ShardId(1), 9); + let err = idx.wait_for(10, deadline_in(30)).unwrap_err(); + assert_eq!( + err, + QuorumWaitError::Timeout { + committed: 9, + laggards: shards(&[1, 2]), + } + ); + // Already-committed seqs resolve instantly regardless of deadline. + assert_eq!(idx.wait_for(9, deadline_in(0)), Ok(9)); + } + + #[test] + fn deactivate_fails_current_and_future_waiters() { + let idx = Arc::new(CommitIndex::new(&shards(&[1]), true)); + let waiter = { + let idx = Arc::clone(&idx); + std::thread::spawn(move || idx.wait_for(1, deadline_in(5_000))) + }; + std::thread::sleep(Duration::from_millis(20)); + idx.deactivate(); + assert_eq!(waiter.join().unwrap(), Err(QuorumWaitError::Demoted)); + assert_eq!( + idx.wait_for(1, deadline_in(10)), + Err(QuorumWaitError::Demoted) + ); + } + + #[test] + fn reactivation_mid_wait_is_a_demotion_not_a_resume() { + // activate→deactivate→activate while a waiter sleeps: the waiter's + // term is gone even though the index ends "active" — epoch detects it. + let idx = Arc::new(CommitIndex::new(&shards(&[1]), true)); + let waiter = { + let idx = Arc::clone(&idx); + std::thread::spawn(move || idx.wait_for(3, deadline_in(5_000))) + }; + std::thread::sleep(Duration::from_millis(20)); + idx.deactivate(); + idx.activate(10); + assert_eq!(waiter.join().unwrap(), Err(QuorumWaitError::Demoted)); + } + + #[test] + fn activate_resets_marks_to_the_baseline() { + let idx = CommitIndex::new(&shards(&[1, 2]), true); + idx.update_peer(ShardId(1), 50); + assert_eq!(idx.committed(), 50); + idx.activate(20); + assert_eq!(idx.committed(), 20, "stale pre-term marks must not leak"); + // Quorum past the baseline still requires a fresh report. + assert!(matches!( + idx.wait_for(21, deadline_in(20)), + Err(QuorumWaitError::Timeout { committed: 20, .. }) + )); + assert_eq!(idx.wait_for(20, deadline_in(0)), Ok(20)); + } + + #[test] + fn zero_peers_commits_at_leader_durability() { + let idx = CommitIndex::new(&[], true); + assert_eq!(idx.wait_for(123, deadline_in(0)), Ok(123)); + } + + #[test] + fn zero_peers_still_respects_demotion() { + // The demoted-leader invariant has no replica-count exception: a + // deactivated single-replica index must never claim quorum. + let idx = CommitIndex::new(&[], true); + idx.deactivate(); + assert_eq!( + idx.wait_for(123, deadline_in(0)), + Err(QuorumWaitError::Demoted) + ); + idx.activate(0); + assert_eq!(idx.wait_for(123, deadline_in(0)), Ok(123)); + } +} diff --git a/tidal/src/replication/mod.rs b/tidal/src/replication/mod.rs index 15434dd..07e1ad5 100644 --- a/tidal/src/replication/mod.rs +++ b/tidal/src/replication/mod.rs @@ -3,6 +3,7 @@ //! The `replication` module is empty in single-node deployments -- //! all types default to `shard_id=0`, `region_id=0`, and routing is a no-op. +pub mod commit; pub mod control; pub mod crdt; pub mod idempotency; @@ -22,6 +23,7 @@ pub mod tenant; pub mod transport; pub mod upgrade; +pub use commit::{CommitIndex, QuorumWaitError}; pub use control::{ClusterHealth, ControlPlane, RegionHealth, ShardStats}; pub use crdt::{Hlc, HlcTimestamp}; pub use idempotency::{IdempotencyKey, IdempotencyStore}; diff --git a/tidal/src/replication/receiver.rs b/tidal/src/replication/receiver.rs index 3a5a1a6..eda7381 100644 --- a/tidal/src/replication/receiver.rs +++ b/tidal/src/replication/receiver.rs @@ -59,20 +59,20 @@ use crate::{ /// blob in its own WAL (so follower recovery rebuilds it) and then applies it /// to storage as an idempotent upsert. pub trait ReplicatedBlobApplier: Send + Sync { - /// Apply a replicated item-metadata record. + /// Apply one apply-round's replicated blob records, in order, as a + /// BATCH: the implementation stages every record's WAL append before + /// waiting, so a whole round shares group-commit fsyncs (m11p3 — a + /// record-at-a-time apply pays one solo fsync per item, capping item + /// apply throughput at the fsync floor, ~100/s on macOS). Records are + /// handed over by value so the engine can share them with its WAL + /// writer by refcount instead of deep-cloning every payload. /// /// # Errors /// /// Any engine error; the receiver HALTS on it (a follower must never - /// silently acknowledge an item it failed to durably record). - fn apply_item_metadata(&self, entity_id: u64, metadata_bytes: &[u8]) -> crate::Result<()>; - - /// Apply a replicated item-embedding record. - /// - /// # Errors - /// - /// Any engine error; the receiver HALTS on it. - fn apply_embedding(&self, entity_id: u64, values: &[f32]) -> crate::Result<()>; + /// silently acknowledge an item it failed to durably record). Blobs are + /// idempotent upserts, so redelivery after a mid-batch halt is safe. + fn apply_blobs(&self, records: Vec) -> crate::Result<()>; } /// Handle to a running segment receiver thread. @@ -179,6 +179,12 @@ pub fn spawn_receiver( let thread = std::thread::Builder::new() .name("tidaldb-segment-receiver".into()) .spawn(move || -> Result<(), WalError> { + // Per-source-shard floor of the last frontier handed to + // `notify_applied`: an apply round that did not advance a shard's + // frontier re-notifies nothing (the transport dedups too, but + // skipping here saves its lock acquisition per quiet round). + let mut last_notified: std::collections::HashMap = + std::collections::HashMap::new(); loop { let Some(first) = transport.recv_segment() else { // Clean shutdown: the transport closed / shutdown was @@ -259,6 +265,19 @@ pub fn spawn_receiver( // the default (in-process) implementation is a no-op. for (shard, max_last) in shard_maxima { let applied = replication_state.applied_seqno(shard).unwrap_or(0); + // Durable-ack release (m11p3): everything this round + // applied is durably folded (storage upserts + this + // node's own WAL fsync precede the frontier advance), so + // ship acks parked on seqnos <= applied may now answer + // with a true durable mark. Only an ADVANCED frontier is + // worth a notification. + if applied > 0 { + let notified = last_notified.entry(shard).or_insert(0); + if applied > *notified { + *notified = applied; + transport.notify_applied(shard, applied); + } + } if applied < max_last { transport.request_catchup(shard, applied + 1); } @@ -359,7 +378,7 @@ fn apply_drained( prepared.push(seg); } } - commit_segments(&prepared, ledger, state, blob_applier)?; + commit_segments(&mut prepared, ledger, state, blob_applier)?; } Ok(()) } @@ -503,7 +522,7 @@ pub fn apply_segment( match prepared { // No blob applier on this legacy/test call shape: a blob batch // reaching it is a wiring error and halts loudly in commit_segments. - Some(seg) => commit_segments(std::slice::from_ref(&seg), ledger, state, None), + Some(mut seg) => commit_segments(std::slice::from_mut(&mut seg), ledger, state, None), None => Ok(()), } } @@ -714,7 +733,7 @@ fn prepare_segment( /// fold every prepared segment's events WAL-first through ONE staged /// group-commit batch, then advance each segment's range in order. fn commit_segments( - prepared: &[PreparedSegment], + prepared: &mut [PreparedSegment], ledger: &SignalLedger, state: &ReplicationState, blob_applier: Option<&dyn ReplicatedBlobApplier>, @@ -729,37 +748,33 @@ fn commit_segments( // ── Blob (item-metadata / embedding) records apply FIRST ── // - // The applier routes each record through the engine's WAL-first item - // write path (the follower re-journals it, then upserts storage). Blobs - // are idempotent, so a halt AFTER blob apply but BEFORE the signal fold - // redelivers safely; folding signals first would double-count them when - // a later blob failure forces redelivery of the same range. - for seg in prepared { - for blob in &seg.blobs { - let Some(applier) = blob_applier else { - return Err(WalError::Io(std::io::Error::other(format!( - "received a kind-{} blob batch (entity {}) but no blob \ - applier is wired on this receiver; halting rather than \ - silently dropping a replicated item", - blob.kind(), - blob.entity_id() - )))); - }; - let result = match blob { - BlobRecord::ItemMetadata(record) => { - applier.apply_item_metadata(record.entity_id, &record.metadata_bytes) - } - BlobRecord::Embedding(record) => { - applier.apply_embedding(record.entity_id, &record.values) - } - }; - if let Err(e) = result { - return Err(WalError::Io(std::io::Error::other(format!( - "replicated kind-{} blob apply failed for entity {}: {e}", - blob.kind(), - blob.entity_id() - )))); - } + // The applier routes the whole group's records through the engine's + // WAL-first item write path as ONE batch (stage all, wait all — group- + // commit fsyncs; m11p3), then upserts storage. Blobs are idempotent, so + // a halt AFTER blob apply but BEFORE the signal fold redelivers safely; + // folding signals first would double-count them when a later blob + // failure forces redelivery of the same range. The records are DRAINED + // out of the prepared segments (ownership moves into the applier — no + // deep clone); on failure the receiver halts and redelivery rebuilds + // them from the segment bytes, so nothing here needs them back. + let group_blobs: Vec = prepared + .iter_mut() + .flat_map(|seg| seg.blobs.drain(..)) + .collect(); + if !group_blobs.is_empty() { + let Some(applier) = blob_applier else { + return Err(WalError::Io(std::io::Error::other(format!( + "received {} blob record(s) but no blob applier is wired on \ + this receiver; halting rather than silently dropping \ + replicated items", + group_blobs.len() + )))); + }; + let blob_count = group_blobs.len(); + if let Err(e) = applier.apply_blobs(group_blobs) { + return Err(WalError::Io(std::io::Error::other(format!( + "replicated blob batch apply failed ({blob_count} records): {e}" + )))); } } @@ -1744,18 +1759,19 @@ mod tests { applied: StdMutex>, } impl ReplicatedBlobApplier for RecordingApplier { - fn apply_item_metadata(&self, entity_id: u64, bytes: &[u8]) -> crate::Result<()> { - self.applied - .lock() - .unwrap() - .push(format!("item:{entity_id}:{}", bytes.len())); - Ok(()) - } - fn apply_embedding(&self, entity_id: u64, values: &[f32]) -> crate::Result<()> { - self.applied - .lock() - .unwrap() - .push(format!("emb:{entity_id}:{}", values.len())); + fn apply_blobs(&self, records: Vec) -> crate::Result<()> { + let lines: Vec = records + .iter() + .map(|record| match record { + BlobRecord::ItemMetadata(r) => { + format!("item:{}:{}", r.entity_id, r.metadata_bytes.len()) + } + BlobRecord::Embedding(r) => { + format!("emb:{}:{}", r.entity_id, r.values.len()) + } + }) + .collect(); + self.applied.lock().unwrap().extend(lines); Ok(()) } } diff --git a/tidal/src/replication/relay.rs b/tidal/src/replication/relay.rs index b078ccc..4e315bb 100644 --- a/tidal/src/replication/relay.rs +++ b/tidal/src/replication/relay.rs @@ -445,7 +445,7 @@ impl SignalRelay { pub fn complete_write(&self, db: &TidalDb, write: StagedRelayWrite) -> crate::Result { let StagedRelayWrite { seqno, staged } = write; match staged.wait(db) { - Ok(()) => { + Ok(_wal_seq) => { self.mark_durable(seqno); Ok(seqno) } diff --git a/tidal/src/replication/ship.rs b/tidal/src/replication/ship.rs index d7a0c81..a856468 100644 --- a/tidal/src/replication/ship.rs +++ b/tidal/src/replication/ship.rs @@ -67,6 +67,7 @@ use std::{ }; use super::{ + commit::CommitIndex, relay::{SignalRelay, encode_run, range_payload}, shard::ShardId, transport::{Transport, TransportError}, @@ -338,6 +339,9 @@ struct ShipShared { shutdown: AtomicBool, /// Leadership gate: only an active queue dispatches (see module docs). active: AtomicBool, + /// Quorum commit index over the peers' durable marks (m11p3). Follows + /// the queue's leadership gate: activated/deactivated with it. + commit: Arc, #[cfg(feature = "metrics")] metrics: Option>, } @@ -408,6 +412,11 @@ impl ShipQueue { }) .collect(); + let commit = Arc::new(CommitIndex::new( + &peer_cells.iter().map(|c| c.peer).collect::>(), + active, + )); + let shared = Arc::new(ShipShared { source, transport, @@ -415,6 +424,7 @@ impl ShipQueue { config, shutdown: AtomicBool::new(false), active: AtomicBool::new(active), + commit, #[cfg(feature = "metrics")] metrics, }); @@ -462,6 +472,14 @@ impl ShipQueue { self.shared.active.load(Ordering::Acquire) } + /// The quorum commit index over this queue's peers (m11p3). Follows the + /// queue's leadership gate; `ack=quorum` writes block on its + /// [`wait_for`](CommitIndex::wait_for). + #[must_use] + pub fn commit_index(&self) -> Arc { + Arc::clone(&self.shared.commit) + } + /// Activate dispatch with a fresh stream baseline: every peer's cursor /// jumps to `baseline + 1`, parked retries clear, and acked frontiers /// reset to the baseline. Called when this node becomes leader (promote): @@ -479,6 +497,7 @@ impl ShipQueue { } cell.acked_atomic.store(baseline, Ordering::Release); } + self.shared.commit.activate(baseline); self.shared.active.store(true, Ordering::Release); self.shared.wake_all(); tracing::info!(baseline, "ship queue activated (leadership)"); @@ -488,6 +507,7 @@ impl ShipQueue { /// complete; parked retries clear (the new leader's stream supersedes). pub fn deactivate(&self) { self.shared.active.store(false, Ordering::Release); + self.shared.commit.deactivate(); for cell in &self.shared.peers { cell.lock().retry.clear(); cell.cv.notify_all(); @@ -525,6 +545,9 @@ impl ShipQueue { } cell.acked_atomic.store(state.acked, Ordering::Release); drop(state); + // The resume seqno is the follower's own durable report (fetched + // from its status), so it counts toward quorum. + self.shared.commit.update_peer(peer, applied); cell.cv.notify_all(); } } @@ -547,6 +570,9 @@ impl ShipQueue { /// Signal every sender to exit and join them. Idempotent. pub fn shutdown(&mut self) { self.shared.shutdown.store(true, Ordering::Release); + // Quorum waiters must not sleep out their deadline against a queue + // that will never ack again. + self.shared.commit.deactivate(); self.shared.wake_all(); for handle in self.threads.drain(..) { if handle.join().is_err() { @@ -721,6 +747,11 @@ fn pause_on_source_fault( /// Logs recovery (at INFO) when this success ends a failure streak. fn record_success(shared: &ShipShared, cell: &PeerShip, run: &ClaimedRun) { let reported = shared.transport.peer_applied_hint(cell.peer); + // Quorum fold (m11p3): the hint is the follower's durably-applied mark + // (the durable ship ack), the ONLY input that may advance the commit + // index. Transport acceptance below advances `acked` for retry pruning + // but never counts toward quorum. + shared.commit.update_peer(cell.peer, reported); let recovered_after = { let mut state = cell.lock(); state.completed.insert(run.first, run.last); diff --git a/tidal/src/replication/transport.rs b/tidal/src/replication/transport.rs index 60ed67c..473f215 100644 --- a/tidal/src/replication/transport.rs +++ b/tidal/src/replication/transport.rs @@ -163,14 +163,29 @@ pub trait Transport: Send + Sync + 'static { /// node's stream), or `0` when unknown — a monotonic HINT, not a /// guarantee. /// - /// The gRPC transport learns it from every `ShipSegmentResponse` (m11p2: - /// the ack carries the follower's applied seqno); the ship queue folds it - /// into its acked frontier so retries of data the follower already holds - /// are pruned and heal needs no separate status fetch. + /// The gRPC transport learns it from every `ShipSegmentResponse`. Since + /// m11p3 the ship ack is sent AFTER the follower durably applies the + /// shipped segment (storage upserts done + its own WAL fsynced), so the + /// hint is a **durable** mark: the ship queue folds it into its acked + /// frontier (retry pruning) AND the quorum [`CommitIndex`] (ack=quorum + /// gating). On a follower that stalls past the ack wait budget the ack + /// degrades to the m11p2 pre-enqueue floor — still monotonic-safe, just + /// laggier, which is exactly what a quorum timeout should observe. + /// + /// [`CommitIndex`]: crate::replication::CommitIndex fn peer_applied_hint(&self, _peer: ShardId) -> u64 { 0 } + /// Notify the transport that this node's applied frontier for + /// `source_shard`'s stream advanced to `applied` (called by the segment + /// receiver once per apply round, AFTER durable apply). + /// + /// The gRPC transport uses it to release in-flight `ShipSegment` acks + /// waiting for their segment's durable apply (m11p3 durable acks). The + /// default is a no-op: in-process transports ack synchronously. + fn notify_applied(&self, _source_shard: ShardId, _applied: u64) {} + /// The shard identity of this transport endpoint. fn local_shard(&self) -> ShardId; } @@ -206,6 +221,13 @@ impl Transport for std::sync::Arc { (**self).peer_applied_hint(peer) } + fn notify_applied(&self, source_shard: ShardId, applied: u64) { + // Forward explicitly: the trait default no-op would silently turn + // every durable ship ack into a wait-budget timeout for type-erased + // callers. + (**self).notify_applied(source_shard, applied); + } + fn local_shard(&self) -> ShardId { (**self).local_shard() } diff --git a/tidal/src/signals/ledger/core.rs b/tidal/src/signals/ledger/core.rs index 0509844..b2ea14e 100644 --- a/tidal/src/signals/ledger/core.rs +++ b/tidal/src/signals/ledger/core.rs @@ -168,10 +168,13 @@ impl SignalLedger { /// relay callers must treat a completion failure as fatal for their stream /// (see `SignalRelay`'s poison semantics). /// + /// Returns the event's assigned WAL seqno (`0` = suppressed by the + /// dedup window — durably represented by an identical prior record). + /// /// # Errors /// /// Returns `TidalError::Durability` if the WAL flush failed. - pub fn complete_staged(&self, staged: StagedLedgerApply) -> crate::Result<()> { + pub fn complete_staged(&self, staged: StagedLedgerApply) -> crate::Result { let StagedLedgerApply { pending, type_id, @@ -179,7 +182,7 @@ impl SignalLedger { weight, ts_ns, } = staged; - let _seq = pending.wait()?; + let seq = pending.wait()?; #[cfg(any(test, feature = "test-utils"))] crate::testing::crash_injector::check_crash_point( @@ -197,7 +200,7 @@ impl SignalLedger { crate::testing::CrashPoint::WalPostAggregate, ); - Ok(()) + Ok(seq) } /// Record a signal event carrying an explicit [`SignalScope`]. diff --git a/tidal/src/wal/mod.rs b/tidal/src/wal/mod.rs index edd469a..8453636 100644 --- a/tidal/src/wal/mod.rs +++ b/tidal/src/wal/mod.rs @@ -31,7 +31,7 @@ pub mod segment; pub mod session_journal; pub mod writer; -use std::{fs, path::PathBuf}; +use std::{fs, path::PathBuf, sync::Arc}; pub use config::WalConfig; use crossbeam::channel::{Sender, bounded}; @@ -216,13 +216,16 @@ impl WalSender { /// replicated log). /// /// Blobs skip the dedup window (idempotent upserts), so the resolved - /// seqno is never the dedup sentinel `0`. + /// seqno is never the dedup sentinel `0`. The record rides in an `Arc` + /// so batch stagers (the follower's replicated blob apply) share one + /// buffer with the writer thread instead of deep-cloning every + /// metadata/embedding payload per staged append. /// /// # Errors /// /// Returns `WalError::SendFailed` if the writer thread has exited. A /// staging error means the record was NOT enqueued (safe to retry). - pub fn append_blob_staged(&self, record: BlobRecord) -> Result { + pub fn append_blob_staged(&self, record: Arc) -> Result { let (reply_tx, reply_rx) = bounded(1); self.tx .send(WalCommand::AppendBlob { @@ -673,18 +676,22 @@ mod tests { let s1 = handle.append(make_event(1)).expect("signal 1"); let sender = handle.sender(); let item_seq = sender - .append_blob_staged(BlobRecord::ItemMetadata(ItemMetadataRecord { - entity_id: 7, - metadata_bytes: vec![1, 2, 3], - })) + .append_blob_staged(std::sync::Arc::new(BlobRecord::ItemMetadata( + ItemMetadataRecord { + entity_id: 7, + metadata_bytes: vec![1, 2, 3], + }, + ))) .expect("stage item blob") .wait() .expect("item blob durable"); let emb_seq = sender - .append_blob_staged(BlobRecord::Embedding(EmbeddingRecord { - entity_id: 7, - values: vec![0.25, -0.5], - })) + .append_blob_staged(std::sync::Arc::new(BlobRecord::Embedding( + EmbeddingRecord { + entity_id: 7, + values: vec![0.25, -0.5], + }, + ))) .expect("stage embedding blob") .wait() .expect("embedding blob durable"); diff --git a/tidal/src/wal/writer.rs b/tidal/src/wal/writer.rs index 92b5a6d..fb9b5f2 100644 --- a/tidal/src/wal/writer.rs +++ b/tidal/src/wal/writer.rs @@ -36,9 +36,12 @@ type QueuedAppend = ( /// Same reply contract as [`QueuedAppend`] — every queued blob eventually /// resolves its reply with its assigned seqno or the flush error. Blobs skip /// the dedup window (they are idempotent upserts; a duplicate apply is -/// harmless) and never return the dedup sentinel. +/// harmless) and never return the dedup sentinel. The record rides in an +/// `Arc` so batch stagers (the follower's replicated blob apply) share one +/// buffer with the writer by refcount instead of deep-cloning every +/// metadata/embedding payload across the channel. type QueuedBlob = ( - BlobRecord, + Arc, crossbeam::channel::Sender>, ); @@ -80,7 +83,7 @@ pub enum WalCommand { /// replicated log). The reply receives the assigned seqno once the blob /// batch is durably fsynced. AppendBlob { - record: BlobRecord, + record: Arc, reply: crossbeam::channel::Sender>, }, /// Delete segments whose first sequence number is less than `before_seq`. @@ -328,93 +331,116 @@ fn sync_segment_observed( Ok(()) } -/// Encode, write, and fsync ONE blob (kind-1/2) record as its own -/// single-seqno batch, notifying the caller of the outcome. -/// -/// The blob counterpart of [`flush_batch`], sharing the same contract: the -/// reply channel is always resolved (success or error) before this returns, -/// and a flush failure leaves `seq` unconsumed (the caller's retry reuses -/// it). Blobs skip the dedup window — they are idempotent upserts keyed by -/// entity id, so a duplicate apply is harmless and suppression would only -/// complicate replay. -/// -/// # Errors -/// -/// Returns the underlying [`WalError`] from encode/rotate/write/sync; the -/// reply channel has already been notified with an equivalent error. -fn flush_blob( +/// Encode and write ONE blob (kind-1/2) record as its own single-seqno +/// batch — WITHOUT syncing. The caller groups consecutive blob writes under +/// one fsync ([`flush_pending_blobs`]); rotation is still safe mid-group +/// because [`SegmentWriter::rotate`] syncs the outgoing segment first. +fn write_blob( segment: &mut SegmentWriter, config: &WriterConfig, seq: u64, record: &BlobRecord, - reply: &crossbeam::channel::Sender>, -) -> Result { - let batch_ts = crate::schema::Timestamp::now().as_nanos(); - let write_result = (|| -> Result>, WalError> { - #[cfg(test)] - if take_flush_failure() { - return Err(WalError::Io(std::io::Error::other( - "injected flush failure", - ))); - } - let encoded = record.encode(seq, batch_ts, config.shard_id, config.region_id)?; - if segment.needs_rotation() { - segment.rotate(seq)?; - } - segment.write_batch_bytes(&encoded)?; - sync_segment_observed(segment, config, 1)?; - Ok(Arc::new(encoded)) - })(); - - match write_result { - Ok(encoded) => { - tracing::debug!( - seq, - kind = record.kind(), - entity = record.entity_id(), - "wal: blob batch appended" - ); - // Feed BEFORE the caller ack — same ordering contract as - // flush_batch (durable implies shippable). - if let Some(feed) = &config.ship_feed { - feed.push(FlushedBatch { - bytes: encoded, - first_seq: seq, - last_seq: seq, - event_count: 1, - }); - } - let _ = reply.send(Ok(seq)); - Ok(seq + 1) - } - Err(err) => { - let err_msg = err.to_string(); - let _ = reply.send(Err(WalError::Io(std::io::Error::other(err_msg)))); - Err(err) - } +) -> Result>, WalError> { + #[cfg(test)] + if take_flush_failure() { + return Err(WalError::Io(std::io::Error::other( + "injected flush failure", + ))); } + let batch_ts = crate::schema::Timestamp::now().as_nanos(); + let encoded = record.encode(seq, batch_ts, config.shard_id, config.region_id)?; + if segment.needs_rotation() { + segment.rotate(seq)?; + } + segment.write_batch_bytes(&encoded)?; + Ok(Arc::new(encoded)) } -/// Flush every queued blob in arrival order, mirroring the steady-state -/// loop's flush-failure resilience: a failed blob flush notifies its caller -/// and leaves `next_seq` unchanged (the seqno is free for a retry), and the -/// writer keeps serving. +/// Flush every queued blob in arrival order under ONE group fsync (m11p3): +/// a sync per blob capped item/embedding apply throughput at the fsync +/// floor (~100/s on macOS `F_FULLFSYNC`) — the follower's batched blob +/// apply stages a whole round of records into one drain window, and this +/// is the half that turns that window into one disk round trip. +/// +/// Failure contract: a failed WRITE aborts the drain — the failed blob and +/// every blob still queued behind it are notified with the error (their +/// seqnos were never consumed; retries re-stage), while the records written +/// BEFORE the failure are intact and still group-sync + ack below. The drain +/// must not keep writing past a failed write: `write_all` may have left +/// partial bytes at that offset, and appending another record after them +/// would bury a torn record MID-segment (replay stops at the first tear, so +/// every later record — though acked — would vanish on recovery) while +/// handing the same seqno to two records. The freed seqno is reused by the +/// NEXT drain — the same write-failure contract the steady-state event path +/// ([`flush_batch`]) has always had; tail-quarantine for that shared +/// residual (rotating away a suspect tail) is a WAL-wide change tracked in +/// the roadmap, not a per-path patch. A failed group SYNC notifies every +/// caller whose write it covered, and nothing unsynced reaches the ship +/// feed. fn flush_pending_blobs( segment: &mut SegmentWriter, config: &WriterConfig, mut next_seq: u64, blobs: Vec, ) -> u64 { - for (record, reply) in blobs { - match flush_blob(segment, config, next_seq, &record, &reply) { - Ok(seq) => next_seq = seq, + type BlobReply = crossbeam::channel::Sender>; + /// One written-but-unsynced blob batch: (seq, encoded, kind, entity, reply). + type WrittenBlob = (u64, Arc>, u8, u64, BlobReply); + let mut written: Vec = Vec::new(); + let mut queue = blobs.into_iter(); + for (record, reply) in queue.by_ref() { + match write_blob(segment, config, next_seq, &record) { + Ok(encoded) => { + written.push((next_seq, encoded, record.kind(), record.entity_id(), reply)); + next_seq += 1; + } Err(e) => { tracing::error!( error = %e, seq = next_seq, kind = record.kind(), - "wal: blob flush failed; caller notified to retry, writer continuing" + "wal: blob write failed; drain aborted, this and queued \ + callers notified to retry, writer continuing" ); + let err_msg = e.to_string(); + let _ = reply.send(Err(WalError::Io(std::io::Error::other(err_msg.clone())))); + for (_, queued_reply) in queue.by_ref() { + let _ = queued_reply + .send(Err(WalError::Io(std::io::Error::other(err_msg.clone())))); + } + break; + } + } + } + if written.is_empty() { + return next_seq; + } + match sync_segment_observed(segment, config, written.len()) { + Ok(()) => { + for (seq, encoded, kind, entity, reply) in written { + tracing::debug!(seq, kind, entity, "wal: blob batch appended"); + // Feed BEFORE the caller ack — same ordering contract as + // flush_batch (durable implies shippable). + if let Some(feed) = &config.ship_feed { + feed.push(FlushedBatch { + bytes: encoded, + first_seq: seq, + last_seq: seq, + event_count: 1, + }); + } + let _ = reply.send(Ok(seq)); + } + } + Err(e) => { + tracing::error!( + error = %e, + blobs = written.len(), + "wal: blob group fsync failed; callers notified, writer continuing" + ); + let err_msg = e.to_string(); + for (_, _, _, _, reply) in written { + let _ = reply.send(Err(WalError::Io(std::io::Error::other(err_msg.clone())))); } } } @@ -776,18 +802,17 @@ pub fn run_writer( } } - // Drained blobs flush through the same contract: healthy → flush in - // order; after any drain failure → notify the remaining callers with the - // same error class instead of dropping their channels. - for (record, reply) in final_blobs { - if let Some(ref e) = drain_err { + // Drained blobs flush through the same contract: healthy → one group + // flush in order (write errors notify their own callers inside); after + // any drain failure → notify the remaining callers with the same error + // class instead of dropping their channels. + if let Some(ref e) = drain_err { + for (_, reply) in final_blobs { let _ = reply.send(Err(WalError::Io(std::io::Error::other(e.to_string())))); - continue; - } - match flush_blob(&mut segment, config, next_seq, &record, &reply) { - Ok(seq) => next_seq = seq, - Err(e) => drain_err = Some(e), } + } else if !final_blobs.is_empty() { + // The post-blob seqno is final here — the writer exits after this drain. + let _ = flush_pending_blobs(&mut segment, config, next_seq, final_blobs); } if let Some(e) = drain_err { return Err(e);