From 95461d3cf804fbc9d72eae835aec4f9db5bc53d5 Mon Sep 17 00:00:00 2001 From: jx12n Date: Thu, 11 Jun 2026 23:30:24 -0600 Subject: [PATCH] feat(m11): Raft leader election over WAL stream (m11p4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kind-3 term markers in the WAL stream, STREAM-relative vote frontiers, heartbeat-only divergence detection + quarantine, and fenced promote. Elections converge in 0.6–1.0s; zero acked-write loss across all kill points. Closes G5 (leaderless recovery) from the v0.9 wave. --- CHANGELOG.md | 67 + docs/ops/stress-test-p4-regression.md | 64 + docs/planning/ROADMAP.md | 6 +- docs/planning/milestone-11/phase-4.md | 427 ++++++ docs/roadmap-to-cluster.md | 40 +- docs/runbooks/cluster.md | 110 +- tidal-net/benches/transport_throughput.rs | 2 + tidal-net/proto/wal_shipping.proto | 87 +- tidal-net/src/client.rs | 91 +- tidal-net/src/convert.rs | 18 + tidal-net/src/error.rs | 12 +- tidal-net/src/lib.rs | 3 +- tidal-net/src/server.rs | 205 ++- tidal-net/src/sources.rs | 114 +- tidal-net/src/transport.rs | 220 +++- tidal-net/tests/catchup_retry.rs | 2 + tidal-net/tests/election_rpc.rs | 419 ++++++ tidal-net/tests/large_payload.rs | 2 + tidal-net/tests/mtls.rs | 6 + tidal-net/tests/multi_node_uat.rs | 8 + tidal-net/tests/reconnection.rs | 2 + tidal-net/tests/transport_contract.rs | 4 + tidal-server/src/cluster/election_driver.rs | 613 +++++++++ tidal-server/src/cluster/mod.rs | 5 +- tidal-server/src/cluster/node.rs | 703 +++++++++- tidal-server/src/cluster/routes.rs | 7 +- tidal-server/src/cluster/topology.rs | 105 ++ tidal-server/src/error.rs | 5 +- tidal-server/src/main.rs | 8 + tidal-server/tests/cluster_election.rs | 530 ++++++++ tidal-server/tests/cluster_grpc.rs | 3 +- tidal-server/tests/cluster_lifecycle.rs | 21 +- tidal-server/tests/cluster_quorum.rs | 44 +- tidal-server/tests/cluster_region.rs | 4 +- tidal-server/tests/cluster_routes.rs | 3 +- tidal-server/tests/support/multiproc.rs | 41 +- tidal-server/tests/support/partition.rs | 21 + tidal/src/db/items.rs | 54 +- tidal/src/db/metrics/cluster.rs | 78 ++ tidal/src/db/mod.rs | 7 + tidal/src/db/replication_ops.rs | 74 ++ tidal/src/replication/commit.rs | 61 +- tidal/src/replication/election.rs | 1319 +++++++++++++++++++ tidal/src/replication/election_store.rs | 277 ++++ tidal/src/replication/in_process.rs | 4 + tidal/src/replication/mod.rs | 6 + tidal/src/replication/receiver.rs | 19 + tidal/src/replication/relay.rs | 31 + tidal/src/replication/ship.rs | 51 +- tidal/src/replication/shipper.rs | 5 +- tidal/src/replication/transport.rs | 11 + tidal/src/wal/diagnostics.rs | 3 +- tidal/src/wal/format/batch.rs | 98 +- tidal/src/wal/format/mod.rs | 14 +- tidal/src/wal/mod.rs | 45 + tidal/src/wal/reader.rs | 12 + tidal/tests/m8p10_reconcile_idempotence.rs | 2 + tidal/tests/m8p2_replication.rs | 18 + tidal/tests/m8p2_replication_durability.rs | 2 + 59 files changed, 6042 insertions(+), 171 deletions(-) create mode 100644 docs/ops/stress-test-p4-regression.md create mode 100644 docs/planning/milestone-11/phase-4.md create mode 100644 tidal-net/tests/election_rpc.rs create mode 100644 tidal-server/src/cluster/election_driver.rs create mode 100644 tidal-server/tests/cluster_election.rs create mode 100644 tidal/src/replication/election.rs create mode 100644 tidal/src/replication/election_store.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index b24b104..bf8a9b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,73 @@ All notable changes to tidalDB will be documented in this file. ### Added +**Automatic failover: failure detection, Raft-style election, term fencing (m11p4) — closes ROADMAP gap G5** +- **"A machine died" is a non-event.** Every multi-process cluster node runs a + purpose-built election-only Raft (pre-vote + vote + check-quorum + fenced + leadership transfer) over the existing `tidal-net` transport: leader + heartbeats every 300ms (config `election.heartbeat_interval_ms`); a follower + that hears no leader for a randomized 1500–3000ms starts a pre-vote; a + majority elects. SIGKILL the leader under `ack=quorum` load → a survivor is + elected and writes resume in well under a second locally, with **zero + acknowledged-write loss across repeated random kill points** (tier-3 gate + `cluster_election.rs::mp_auto_failover_writes_resume_zero_acked_loss`). + No raft crate: the WAL is already the replicated log (m11p2) and the quorum + commit index (m11p3) already proves majority durability — the election adds + only the tiny consensus state `(term, leader)` on top of them. +- **The WAL carries its own election history.** An elected leader's FIRST log + entry is a kind-3 **term-marker record** that replicates like any record, so + every replica's `(lastLogTerm, lastLogIndex)` — Raft's vote restriction — is + derived from one fsync stream and can never disagree across a crash. The + frontier half is compared in the **last joined term's stream numbering** + (a reseeded node's own WAL numbering diverges from the stream's after a + baseline jump; comparing raw local frontiers across nodes would let a + behind node win). Hard state `(current_term, voted_for)` persists in + `data_dir/election_state` (magic + version + checksum; corrupt → refuse to + boot; deleted-with-a-WAL-present → forced follower) and is fsynced BEFORE + any vote reply or leadership claim leaves the node. +- **Term fencing everywhere (kills incident §1.4-1).** Every replication RPC + carries the sender's term: stale-term ships/chunks/heartbeats/frontier + reports are rejected, the quorum commit index folds only reports stamped + with its activation term (race-free, under the index's own lock), and a + restarted ex-leader boots as a FOLLOWER from its durable state — never from + the topology file. A partitioned ex-leader that restarts cannot accept a + single write (`mp_fenced_ex_leader_restart_cannot_write`); a leader that + loses majority contact steps down within `election.leader_lease_ms` + (default 900; validated `lease + heartbeat < election_timeout_min` so a + deposed leader stops before any successor can exist). +- **Divergent suffixes quarantine instead of lying.** A node whose log + extends past what the elected leadership subsumed (leader-acked, + never-quorum-acked writes on a dead leader) detects it at term-join — the + heartbeat carries the leader's election-time log position — and fences + itself from the data plane (status `quarantined: true`, metric + `tidaldb_cluster_divergence_quarantined`, ERROR naming the reseed runbook) + while still voting. Followers apply on receipt, so un-applying is not a + thing: reseed is the honest recovery (m11p5 snapshots automate it). +- **`/cluster/promote` is now a fenced transfer**: with a live leader it + drains (waits for the target to hold the flushed prefix, breaker-immune via + the commit index's marks) then sanctions an immediate election; with a dead + leader the target campaigns. The election refuses a target whose log lags — + the m11p3 "promote the max-applied survivor" operator rule is now enforced + by the protocol. The legacy term-0 fan-out survives only on clusters that + have never elected (mixed-version rollouts; the chaos drills' deliberate + isolated-node override) and is permanently retired per node at its first + joined election. `election.auto_election: false` preserves the full + pre-m11p4 operator posture (no auto elections, no check-quorum step-down). +- **Bounded churn under flapping links**: the pre-vote (no term inflation + without a majority probe) plus the leader-freshness lease absorb short + flaps entirely and bound terms under long ones + (`mp_flapping_links_bounded_churn`). Election observability: + `tidaldb_cluster_election_term/_role/_elections_started_total/ + _leader_changes_total`, and `/cluster/status/local` grows `term`, `role`, + `quarantined`, `prev_log_term/seq`. +- New tier-3 exit-gate suite `cluster_election.rs` (auto-failover ledger, + fencing under partition+restart, bounded churn); `cluster_quorum.rs` and + `cluster_lifecycle.rs` pin `auto_election: false` (they validate the manual + drill, which remains supported); the partition harness gained + bidirectional isolation (`isolate_region`) — severing only a node's inbound + edges leaves its outbound heartbeats/votes flowing, which silently defeats + partition scenarios. + **Catch-up self-healing + WAL segment format versioning (m11p4) — timer-retried pulls, `TSEG` segment header, structured "snapshot required"** - **Failed catch-up pulls retry on a timer.** The pull trigger was event-only: a follower whose `StreamSegments` pull failed (e.g. the leader's gRPC server diff --git a/docs/ops/stress-test-p4-regression.md b/docs/ops/stress-test-p4-regression.md new file mode 100644 index 0000000..94711f8 --- /dev/null +++ b/docs/ops/stress-test-p4-regression.md @@ -0,0 +1,64 @@ +# Stress test regression — p4 (m11p4) (2026-06-11) + +Live 3-StatefulSet k3s cluster, m11p4 image (`sha256:86dd59c4...`). Fresh +deployment over existing PVCs (no reset needed — m11p4 WAL reader accepts m11p3 +segments as implicit v0). All 3 pods Running, lag=0 at test start. + +--- + +## Catch-up self-healing validation + +Validates the m11p4 fix: failed/missed catch-up pulls now retry on a 30s timer +rather than waiting for the next push event. + +**Procedure:** +1. Confirmed baseline lag=0 on all 3 nodes. +2. Killed tidaldb-eu-west-0; wrote 500 events to leader (ack=leader) while pod + was down. Pod restarted via StatefulSet (fast restart). +3. Stopped all writes with eu-west at lag=2,035,769. +4. Polled `/cluster/status` every 10s — zero writes issued. + +**Result:** + +| t+ | eu-west lag | +|---|---| +| 0s (write-stop) | 2,035,769 | +| 10s | **0** | + +eu-west caught up to lag=0 within 10s of pod start with no new writes to the +cluster. Prior to m11p4, this scenario left the follower stuck indefinitely +(the 2026-06-11 p3 rollout incident: both followers stuck at lag=136507 until +the cluster received new writes). + +**Verdict: timer-triggered catch-up CONFIRMED.** + +--- + +## Throughput regression (p3 gate re-check) + +`tidal-stress` open-loop, `--ack quorum`, `--mix writes`, `peach-100k` ramp, +60s/stage, 20k corpus, 100k users, `--stop-on-knee`. + +| Stage | Target rps | Signal ok/s | Error rate | p99 write | Max lag | Pass | +|---|---|---|---|---|---|---| +| 1 | 50 | 50 | 0.00% | 35 ms | 0 | ✓ | +| 2 | 150 | 150 | 0.00% | 35 ms | 0 | ✓ | +| 3 | 400 | 400 | 0.00% | 37 ms | 0 | ✓ | +| 4 | 800 | 800 | 0.00% | 43 ms | 8 | ✓ | +| 5 | 1,500 | 1,499 | 0.04% | 41 ms | 0 | ✓ | +| **6** | **3,000** | **2,975** | **0.82%** | **49 ms** | **27** | **✓ highest** | +| 7 ⚠ | 5,000 | 4,411 | 11.76% | 164 ms | 24 | ✗ knee | + +No 503 quorum timeouts at any stage. + +### Regression verdict + +| Metric | T2-A (m11p3) | Regression (m11p4) | Delta | Result | +|---|---|---|---|---| +| Quorum signal-writes/s | 2,980/s | **2,975/s** | −0.2% | **✓ no regression** | +| Write p99 at stage 6 | 49 ms | **49 ms** | 0 | **✓ no regression** | +| Error rate at stage 6 | 0.67% | **0.82%** | +0.15% | **✓ within gate (<1%)** | +| Capacity knee | stage 7 (5k) | **stage 7 (5k)** | none | **✓ no regression** | +| Lag at stage 6 | 23 events | **27 events** | +4 | **✓ within gate (≤2s)** | + +**p3 gate (≥1,000/s quorum, p99 ≤50ms, error <1%) holds through m11p4.** diff --git a/docs/planning/ROADMAP.md b/docs/planning/ROADMAP.md index ff5f26b..95232f3 100644 --- a/docs/planning/ROADMAP.md +++ b/docs/planning/ROADMAP.md @@ -37,7 +37,7 @@ A single embeddable database can replace the 6-system content ranking stack by t | M8 | Distributed Fabric | Multi-region, multi-tenant replication keeps agent-memory semantics intact | Hosted tidalDB, cloud/edge deployments, shared agent substrate — **✅ COMPLETE**: in-process primitives + multi-node replication over real gRPC + true multi-process cluster mode (one process per region, real process isolation) with full tier-3 UAT (partition injection via TCP-proxy, clock-skew, rolling-upgrade, runbook verification); G1 + G2 resolved. Post-M8 follow-ups: quorum-ack writes, automatic failure detection / leader election | | M9 | Community Sync & Revocation | Local embeddable profiles can opt into community personalization and safely leave/purge contributions | Community personalization, federated taste graphs, shared feeds — ✅ COMPLETE (2026-06-06) | | M10 | Governance & Agent Rights | Community rules and agent-scoped permissions control what signals influence ranking | User-owned AI personalization at scale, policy-compliant agents — ✅ COMPLETE (2026-06-06) | -| M11 | Enterprise-Grade Cluster | The multi-process cluster becomes a system of record: fast unified log, quorum-durable, self-healing, horizontally scalable, secured, observable, chaos-tested | Paying-customer cluster deployments — **IN PROGRESS**: m11p1 (replication perf floor) ✅ + m11p2 (one replicated log) ✅ + m11p3 (quorum-acked writes: `ack=leader\|quorum`, commit index, zero-acked-loss ledger gate — closes G4) ✅ COMPLETE 2026-06-11; p4–p9 planned in [docs/roadmap-to-cluster.md](../roadmap-to-cluster.md) | +| M11 | Enterprise-Grade Cluster | The multi-process cluster becomes a system of record: fast unified log, quorum-durable, self-healing, horizontally scalable, secured, observable, chaos-tested | Paying-customer cluster deployments — **IN PROGRESS**: m11p1 ✅ + m11p2 ✅ + m11p3 ✅ + m11p4 ✅ (automatic failover: Raft-style election + term fencing + divergence quarantine — closes **G5**; v0.9 "Credible HA" wave complete 2026-06-12); p5–p9 planned in [docs/roadmap-to-cluster.md](../roadmap-to-cluster.md) | ### Embeddable → Distributed Path @@ -2906,12 +2906,12 @@ These phases take the proven in-process primitives and deliver actual multi-node > `/signals` path from ~90/s to 4,534/s sustained (3-process localhost, > release build) with lag bounded ≤377 events, and cluster mode gained its > first `/metrics` listener + `tidaldb_cluster_*` series. G4 (quorum acks, -> m11p3) and G5 (election/fencing, m11p4) remain open. +> m11p3) and G5 (election/fencing, m11p4) are both now closed. | Gap | Origin | Severity | Description | |-----|--------|----------|-------------| | G4 | ✅ closed by m11p3 (2026-06-11) | Medium | **Quorum-ack write contract.** ~~A `204` from `/signals` (and the data writes) asserts leader durability only~~ — `ack=quorum` (topology `replication.ack` default or per-request `x-tidal-ack`) now gates success on a majority of the replica set durably holding the write, with retryable-503 timeout semantics and a 100/100-kill-point zero-acked-loss ledger gate. Runbook §8 documents the knob and its cost. | -| G5 | post-M8 | Medium | **Automatic failure detection / leader election.** Leadership is operator-driven: `/cluster/promote` moves the leader and fans the view out to peers; there is no automatic failure detector and no automatic election. "Survive a machine dying" is a runbook step (detect → promote), not an automatic failover. (Runbook §9 documents the operator drill.) | +| G5 | ✅ closed by m11p4 (2026-06-12) | Medium | **Automatic failure detection / leader election.** ~~Leadership is operator-driven; "survive a machine dying" is a runbook step~~ — every node now runs a heartbeat failure detector + Raft-style election (pre-vote, check-quorum, durable terms) with term fencing on every replication RPC: kill the leader and a survivor is elected in under a second locally with zero acknowledged-write loss; a restarted ex-leader boots as a follower from durable state and cannot accept a write (the §1.4-1 split-brain incident closed by construction). `/cluster/promote` became a fenced transfer; `election.auto_election: false` preserves the operator posture. Runbook §9 rewritten. | | G6 | post-M8 | Low | **Embedding validation surfaces as HTTP 500.** A strict-dimension mismatch or zero-norm vector on `/embeddings` is correctly rejected, but the engine wraps the `VectorError` as an internal error, so the HTTP status is 500 rather than 400. The write is safely rejected (no corruption); only the status code is imprecise. (Runbook §5 documents the 500.) | ### Done When (M8 Full) — ✅ all satisfied diff --git a/docs/planning/milestone-11/phase-4.md b/docs/planning/milestone-11/phase-4.md new file mode 100644 index 0000000..cbcffe3 --- /dev/null +++ b/docs/planning/milestone-11/phase-4.md @@ -0,0 +1,427 @@ +# m11p4 — Failure Detection, Election, Fencing (COMPLETE — 2026-06-12) + +Phase spec and exit gate: [docs/roadmap-to-cluster.md §4/m11p4](../../roadmap-to-cluster.md). +Closes ROADMAP gap **G5** ("operator-driven failover; restarted node believes +the static topology — split-brain seed", incident §1.4-1). +Predecessors: [phase-1.md](phase-1.md) (ship queue), [phase-2.md](phase-2.md) +(one replicated log), [phase-3.md](phase-3.md) (quorum machinery — the +prerequisite this phase's election rides on). + +**Already shipped in this phase** (d0a52e4, 2026-06-11): catch-up timer retry +(re-arm-on-skip liveness) + TSEG segment version header + structured +"snapshot required" refusal. This document covers the remaining XL work. + +**Goal:** "a machine died" is a non-event. Kill the leader under load → +writes resume <10 s p99 with zero operator action and zero acknowledged +loss; a restarted or partitioned ex-leader cannot accept a single write; +election churn under flapping links is bounded. + +## Design (as adopted) + +### 0. Open question #2 resolved: purpose-built election, no raft crate + +The roadmap deferred "minimal purpose-built Raft over `tidal-net` vs. a +vetted raft crate" to this phase. Decision: **purpose-built election-only +Raft** (Raft §5.1–§5.4 election subset + pre-vote + check-quorum + leadership +transfer), for one structural reason: **the replicated log already exists.** +m11p2 made the WAL the stream; m11p3 built the quorum commit index over it. +A raft crate (openraft, raft-rs) owns its own log and storage contracts — +adopting one means either running a second log beside the WAL or contorting +the crate to delegate its log to ours. The state that actually needs +consensus is tiny — `(term, leader)` — and the safety-critical machinery +(durable votes, the up-to-date restriction, term fencing) maps directly onto +primitives we already have (`flushed_seq()`, the `stream_baseline` +persistence pattern, the commit-index epoch). Membership stays static this +phase (the topology file); m11p5 makes it dynamic through this phase's +metadata machinery. + +The election state machine is **pure and deterministic** (no clocks, no I/O +— inputs carry `Instant`s, randomness is an injected seed), so the entire +protocol is unit-testable without processes or sleeps. + +### 1. The durable election state (kills restart amnesia) + +A small per-node file, `data_dir/election_state`, written with the exact +`stream_baseline` discipline (write `.tmp`, `sync_file_durable` +[F_FULLFSYNC on macOS], atomic rename, `sync_dir_durable`), carrying a +magic + format version + checksum so corruption is detectable: + +| Field | Type | Meaning | +|---|---|---| +| `current_term` | u64 | Highest term this node has seen. **Strictly monotonic** — a term is adopted only upward; no inbound message can ever lower it. | +| `voted_for` | Option | Vote cast in `current_term` (at most one). | + +The WAL-tail term deliberately does NOT live in this file — see §1b. + +**Persistence is a fence, not bookkeeping**: the file is fsynced BEFORE any +externally visible action that depends on it — before a vote response is +sent, before a higher term is acknowledged, before leadership is assumed. +Votes and term changes are rare (not per-write), so the fsync cost is +irrelevant. All election-state mutations serialize under one lock, and the +boot-time persist completes before the HTTP routes serve a single request +(design-review C6). + +**Boot rule** (the incident §1.4-1 fix), four cases (design-review C1): + +1. File exists and reads clean → the node **always boots as a follower**, + regardless of what the file or the topology says — leadership is learned + from heartbeats or won by election, never assumed from a restart. +2. File exists but is corrupt (bad magic/version/checksum) → **refuse to + boot** with a FATAL error naming the file. A node that cannot prove its + term knowledge must not guess at term 0 — guessing re-opens the §1.4-1 + amnesia hole through the side door of a disk fault. +3. File absent but the data dir is NOT fresh (a WAL exists) → the file was + deleted out from under a node that has run before. Boot **as a + follower** at term 0 with a loud ERROR (never assume topology + leadership), and persist the recreated state immediately. Terms recover + upward from the first heartbeat/ship the node hears. +4. File absent and the data dir is fresh → a genuinely new node: boot with + the topology's term-0 roles exactly as today, persisting `{term: 0}` + before serving. Existing deployments boot byte-for-byte identically on + first start. + +**Term-0 conflicting-leader guard** (design-review C5): a term-0 leader that +receives another node's term-0 *leadership assertion* (heartbeat or ship +stamped `term=0, leader=other`) — possible only through mismatched topology +files, a deployment error — logs ERROR and starts an election. Term ≥ 1 +leadership is unique by votes; the election converges the misdeployment +instead of letting two config-believing leaders run open-loop. + +### 1b. WAL term markers (the up-to-date restriction, crash-atomic) + +Raft's vote restriction compares `(lastLogTerm, lastLogIndex)`. The first +design draft persisted a `last_applied_term` in the election file; the +adversarial review (C7) found the killer: that file and the WAL frontier +can never be made crash-atomic — a crash between the two persists yields a +node whose up-to-date claim lies in the **unsafe** direction (it can win an +election while missing committed entries). + +Fix: the term lives **in the log itself**. On becoming leader of term T +(any T ≥ 1), the leader journals a `TermMarker(T)` record — a new small +WAL record kind beside m11p2's item/embedding blobs — as its first term-T +entry. It replicates to every follower through the normal stream like any +other record; followers fold it by recording the term (no storage effect). +The node's **WAL-tail term** is then the term of the last marker at or +below its durable frontier — recovered from the WAL on boot, advanced on +apply, and *atomic with the frontier by construction* (they are the same +fsync stream). No marker (a pre-election log) reads as term 0. + +The vote comparison becomes `(wal_tail_term, durable_frontier)`, +lexicographic, both read from the same WAL state. A crash at any point +yields a node that understates — never overstates — its log, which is the +safe direction (it may lose an election it deserved; it can never win one +it shouldn't, because every committed entry's quorum contains an +un-crashed holder whose refusal stands). + +Wire compatibility falls out of "markers only exist at term ≥ 1": a p4 +binary on a term-0 cluster never writes one, so a rolling m11p3→m11p4 +upgrade ships zero unknown record kinds; the first election (only possible +once a vote quorum of upgraded binaries exists) is the moment the format +becomes live. Runbook: complete the binary upgrade before relying on +auto-failover. + +### 2. Term model: term 0 is the topology, terms ≥ 1 are elected + +- **Term 0** = the static topology file's leader. A fresh cluster boots + leading/following at term 0 with no election, preserving every existing + boot path and test. +- **Any election or fenced transfer moves the cluster to term ≥ 1.** From + then on the durable state is authoritative and the topology's `leader` + field is dead weight (kept for bootstrap only). +- Proto compatibility falls out: proto3 defaults absent term fields to 0, + so a pre-m11p4 peer's ships/reports read as term-0 traffic. A node still + at term 0 interoperates with them; a node at term ≥ 1 rejects them + (`FAILED_PRECONDITION`) — exactly the fencing semantics, applied to + version skew. + +### 3. Election protocol + +Roles: `Follower → PreCandidate → Candidate → Leader` (and anything → +`Follower` on observing a higher term). + +- **Failure detector**: the leader sends `Heartbeat` to every peer each + `heartbeat_interval` (default 500 ms), now carrying `(term, leader_region, + stream_baseline)`. A follower that hears nothing valid for a randomized + `election_timeout` ∈ [1500 ms, 3000 ms) starts a **pre-vote**. +- **Pre-vote** (disruption guard): asks "would you vote for me in term+1?" + without bumping any term. A voter refuses if it heard from a live leader + within `election_timeout_min` (lease check) or if the candidate's log + loses the up-to-date comparison. Only a pre-vote majority proceeds — a + partitioned or flapping node cannot inflate terms or depose a healthy + leader from the outside. +- **Vote**: candidate persists `term+1, voted_for=self`, then requests + votes. A voter grants iff `req.term ≥ current_term`, it hasn't voted for + another candidate in that term (durable before reply), and the candidate + is **up-to-date**: `(wal_tail_term, durable_frontier) ≥` the voter's, + compared lexicographically — Raft's §5.4.1 restriction with both values + derived from the voter's own WAL (§1b). Majority → leader. +- **Becoming leader**: persist election state, then journal the + `TermMarker(T)` (§1b), then the promote mechanics verbatim: baseline = + own `flushed_seq()` (now ≥ the marker's seqno), `persist_stream_baseline`, + `ship_queue.activate_from(baseline, term)` (which activates the commit + index *scoped to T* and bumps its epoch), leader view updated, heartbeats + start. +- **Step-down ordering** (design-review C3): on any step-down — higher-term + contact, check-quorum loss, fenced transfer — the **leadership view flips + first** (`leader = None` or the new leader), THEN the ship queue + deactivates. The write path's `is_leader()` reads the view, so once it + flips no new write is accepted; the queue deactivation then fails + in-flight `ack=quorum` waits with the existing `Demoted` path. The + residual window (a request that read the old view and stages after the + flip) produces a leader-local-durable, never-shipped write — exactly the + documented `ack=leader` crash contract, surfaced at rejoin as a divergent + suffix (§5). +- **Check-quorum / lease step-down**: a leader that cannot reach a majority + of peers (heartbeat acks) within `leader_lease_ms` **steps down** (stops + accepting writes; quorum waits fail). The lease bound (design-review C2): + the leader's countdown starts at its last successful quorum contact while + a follower's election timer starts at the last heartbeat it *received* — + the two anchors differ by up to one heartbeat interval plus RTT, NOT by + wall-clock offset (all timers are monotonic `Instant` intervals; clock + offsets cancel). The enforced constraint is therefore + `leader_lease_ms + heartbeat_interval_ms < election_timeout_min_ms` + (validated at topology load), guaranteeing the deposed leader stops + before any successor can be elected. This bounds the + minority-partitioned-leader window for `ack=leader` writes; `ack=quorum` + writes were never at risk (the commit index cannot reach majority from a + minority partition). +- **No pre-vote deadlock** (design-review C9): the lease refusal can only + outlive a real leader if something keeps *sending* heartbeats. A + stepped-down leader stops heartbeating by construction (only the Leader + role sends), so after a leader dies or steps down every follower's lease + window drains within one `election_timeout` and pre-votes start granting. + Asymmetric partitions degrade exactly one node (it pre-votes, is refused + by peers that still hear the leader, and never inflates a term — the + pre-vote's whole point); the majority side keeps serving throughout. +- **Timers** use `std::time::Instant` exclusively — monotonic, immune to + the chaos suites' `TIDAL_HLC_SKEW_MS` wall-clock skew injection (HLC + remains data-plane-only for hard-negative LWW). + +### 4. Fencing everywhere + +Every replication RPC carries the sender's term; every receiver gates: + +| RPC | Stamp | Stale-term reaction | Higher-term reaction | +|---|---|---|---| +| `ShipSegment` | `term, leader_region` | reject `FAILED_PRECONDITION` before enqueue | persist & adopt term, treat as leader contact | +| `StreamSegments` request | puller's `term` | source at a higher term refuses the pull | source at a lower term steps down (it is deposed and must not serve its possibly-divergent tail — design-review C14) | +| `StreamSegments` chunks | same as ship (chunks are `ShipSegmentRequest`s) | puller drops the chunk + aborts the pull (a stale source can never inject history) | adopt | +| `Heartbeat` | `term, leader_region, stream_baseline` | respond `accepted=false, term` (sender steps down) | adopt, reset election timer, run the §5 term-join check | +| `ReportApplied` | reporter's `term` | leader folds **only if reporter term == the commit index's activation term** (design-review C4/C8/C12: the gate reads the term the index was activated with, so a stale, future, or cross-leadership report can never advance it; heal's `resume_from` folds through the same gate — C13) | leader steps down | +| `RequestVote` / `TimeoutNow` | `term` | refuse, return own term | adopt | +| HTTP forward (internal-marked writes) | already rejected on a non-leader (`NotLeader` 503); the error body now names `(term, leader-or-none, leader_http_addr)` so the forwarder follows one redirect to the named leader or returns a retryable 503 "election in progress" (design-review C11) | — | — | + +A ship/report/vote response carrying a higher term is a **step-down +signal** to the sender — the deposed leader learns its term is stale from +its own outbound traffic even if it never hears a heartbeat. The +`CommitIndex` itself is term-scoped: `activate(baseline, term)` records the +term, `update_peer` folds only same-term reports, so every quorum input — +ship-ack hints, `ReportApplied`, heal resumes — passes one gate. + +### 5. Rejoin, and the divergent-suffix boundary + +The check runs exactly once per term, at **term-join** (the first valid +leader contact carrying term T — heartbeat or ship — when adopting T), and +compares the node's durable WAL frontier against the term's activation +baseline B (persisted by the leader at activation and announced in every +heartbeat — design-review C10: B is immutable for the term, so the check +never races a moving target; a frontier that grows past B *within* term T +is just normal replication, never re-checked): + +- **frontier ≤ B**: clean rejoin. Catch-up via the existing `StreamSegments` + pull (gap-triggered, timer-retried). The node's WAL-tail term advances to + T when the term's marker record applies (§1b). +- **frontier > B with WAL-tail term < T**: the node holds a **divergent + suffix** — writes that were leader-durable on a dead leader but never + quorum-committed, while the cluster elected past them. The node + **quarantines**: it refuses to apply or report (receiver halt latch — the + existing `died()` health surface), logs ERROR naming the suffix range and + the reseed runbook, sets `tidaldb_cluster_divergence` and stays fenced + from the data plane. It still votes (its stale WAL-tail term makes it + lose every up-to-date comparison against healthy peers, so it can never + win). +- **frontier > B with WAL-tail term == T**: impossible at term-join (T-term + entries only exist after the leader's activation, and joining is what + admits them) — and after joining, it is the normal shape of a follower + tracking the leader. Restarts *within* a term re-join idempotently: the + tail term already equals T, so no quarantine. + +Why quarantine instead of Raft-style suffix truncation: tidalDB followers +apply segments **on receipt** (before quorum commit) — by the time +divergence is discovered, the suffix's effects are already folded into +storage (decayed counters), and unapplying signal arithmetic is not a thing. +Truncating the WAL alone would lie about state. The honest recovery is a +reseed — which is m11p5's snapshot+stream work, and exactly what the +already-shipped p4 increment's `FAILED_PRECONDITION ("snapshot required")` +path points at. Probability note: with RF=3 a quorum-committed suffix holder +is a member of *every* election majority (the vote restriction protects it), +so wrongful quarantine is impossible at n=3; the quarantined case is +strictly leader-ack-only data, which is the documented leader-ack contract. + +### 6. Promote becomes a fenced transfer + +`POST /cluster/promote {region}`: + +- **Leader alive** (the maintenance drain): forwarded to the current + leader, which waits until the target's durable mark reaches the leader's + flushed frontier (bounded; 503 naming the lag on timeout), then sends + `TimeoutNow` — the target starts an immediate election (skipping pre-vote + and timeout; the leader sanctioned it), wins on term+1, and the old leader + steps down on the first higher-term contact. No drain-the-world: the + catch-up wait IS the drain. +- **Leader dead/unreachable** (the legacy failover drill): the target runs a + normal pre-vote + election among survivors. With auto-election on, the + operator usually never gets here — the verb remains as a manual override + and a way to *choose* the successor. +- **Mixed-version fallback**: if a majority of peers answer `Unimplemented` + to vote RPCs (pre-m11p4 binaries), promote falls back to the legacy + fan-out promote at term 0 with a loud WARN. Auto-election quietly waits + until the fleet is upgraded (vote quorum impossible until then). + +### 7. What goes where + +| Crate | Work | +|---|---| +| `tidal` | `replication/election.rs`: pure `ElectionState` step machine (inputs → actions) + config; `replication/election_durable.rs`: the fsync'd `(current_term, voted_for)` file with magic/version/checksum + the four-case boot classification; the `TermMarker` WAL record kind + recovery/apply tracking of the WAL-tail term (§1b); `CommitIndex.activate(baseline, term)` + same-term fold gate; `WalSegmentPayload.term/leader_region`; ship queue stamps terms (shared `Arc`); receiver-side divergence latch. | +| `tidal-net` | Proto: term fields on `ShipSegmentRequest/Response`, `StreamRequest`, `HeartbeatRequest/Response`, `AppliedReport`; new `RequestVote` (with `prevote`/`transfer` flags) + `TimeoutNow` RPCs. Server: late-bound `ElectionHooks` (vote/heartbeat/observed-term) mirroring the `AppliedSink` pattern; term gates on ship/stream/report handlers. Client: `PeerPool::{request_vote, timeout_now, heartbeat}` — vote RPCs bypass the circuit breaker (an election must be able to probe a peer the breaker quarantined; vote traffic is rare and self-limiting). Config: `election` knob block. | +| `tidal-server` | The **election driver** (one tokio task per region state): ticks the state machine, executes actions (persist → send → transition), owns the heartbeat fan-out and failure detection. Boot path: durable-state detection → rejoin-as-follower. Leadership transitions route through the existing `promote_local` mechanics. `leader` becomes `RwLock>` (leaderless windows are real and reported honestly — forwards during an election return retryable 503 "election in progress"). `/cluster/status` grows `term`, `role`; election metrics (`tidaldb_cluster_term`, `_role`, `_elections_started_total`, `_leader_changes_total`, `_stepdowns_total`, `_votes_{granted,denied}_total`, `_divergence`). Promote rework per §6. | +| tests | Pure state-machine property/unit tests (no clocks); `tidal-net` vote-RPC tests over real sockets; tier-3 `cluster_election.rs`: the three exit gates below. | + +### 8. Configuration (topology `election` block, all serde-defaulted) + +| Knob | Default | Constraint | +|---|---|---| +| `heartbeat_interval_ms` | 300 | | +| `election_timeout_min_ms` | 1500 | ≥ 3× heartbeat interval | +| `election_timeout_max_ms` | 3000 | > min | +| `leader_lease_ms` | 900 | `leader_lease_ms + heartbeat_interval_ms < election_timeout_min_ms` (the C2 bound — validated at topology load) | +| `auto_election` | true | false = detector + fencing only; elections fire only through `/cluster/promote`. After a full-cluster restart in this mode the cluster is deliberately leaderless until an operator promotes (design-review C16 — that is the posture the knob selects). | + +Failover arithmetic: detect (≤3 s worst-case timeout) + pre-vote+vote (2 +RTTs + 2 fsyncs) + activation (µs) ≈ **3–4 s typical, comfortably inside +the 10 s p99 gate**, without making the detector so twitchy that a fsync +stall triggers an election (the 10–50 ms F_FULLFSYNC tail from p1 informs +the 1500 ms floor). Heartbeats are async fan-out tasks on the transport's +runtime with a per-call timeout well under the interval — N peers at 300 ms +is negligible load and can never stall the data plane (design-review C15). + +## Exit gate (from the roadmap, restated as tests) + +1. **Auto-failover, zero acked loss**: SIGKILL the leader under `ack=quorum` + load with auto-election on → writes resume on the new leader <10 s p99, + zero operator verbs, and the m11p3 ledger checker (frontier + content + proofs) passes across repeated random kill-points. +2. **Fencing under partition + restart**: partition the old leader away + mid-election, let it restart → it boots a follower (durable state), its + ships/votes carry a stale term and are rejected, and it **cannot accept + a single write** (every write attempt 503s) until it heals and rejoins. +3. **Bounded churn**: flapping links (proxy sever/heal cycles) produce + bounded elections (pre-vote absorbs the flaps; no term explosion, no + livelock), and the cluster converges to exactly one leader per term. + +## Status + +- [x] Design adopted (this document; 16 confirmed adversarial-review holes folded in pre-implementation) +- [x] Election core (state machine + durable state) — `tidal` +- [x] Transport (proto, hooks, gates, client verbs) — `tidal-net` +- [x] Driver + boot + fencing + promote rework — `tidal-server` +- [x] Exit-gate suites green +- [x] Docs (runbook §9, CHANGELOG, roadmap, status fields) + +## As built (deltas vs the adopted design) + +1. **The vote-restriction frontier is STREAM-relative, not own-WAL-relative.** + The gate run exposed it: a reseeded node's local WAL numbering diverges + from stream numbering after a baseline jump (observed live: stream-applied + 38 vs own-WAL 22), so raw local frontiers are not comparable across nodes. + `LogPosition.frontier` is the node's durable position **in the last joined + term's stream numbering**: own `flushed_seq()` when it led that term, + `applied_seqno(shard of that term's leader)` otherwise. The term marker + therefore carries `leader_region`, recovered with the term. +2. **Divergence is judged against the leader's ELECTION-TIME position, only + from heartbeats.** The adopted baseline-comparison was cross-numbering for + the same reason. The heartbeat carries `(prev_log_term, prev_log_seq)` — + the same pair the vote restriction compared — and a joining node is + divergent iff its own position exceeds it lexicographically. A NEW-term + ship arriving before the first heartbeat defers with a retryable + `JoinPending` (UNAVAILABLE) instead of guessing; `stream_baseline` on the + heartbeat is used only for the frontier jump. +3. **"Topology era" ends at the first JOINED election, not at one's own + durable term.** A failed campaign (an isolated operator override, the + chaos drills' deliberate split-brain) inflates a node's durable term with + no elected leadership existing; gating term-0 traffic, the legacy promote + and the check-quorum fallback on `joined_term` instead of `current_term` + keeps that world functional while elected clusters stay absolutely fenced. +4. **Check-quorum step-down is gated on `auto_election`** — with elections + off no successor can exist, so stepping down would only convert a follower + outage into total write unavailability; the knob deliberately preserves + the m11p3 availability posture. +5. The transfer's catch-up wait reads the **commit index's peer mark** (fed + by `ReportApplied`), not the ship queue's acked frontier — the latter + stalls ~30s behind an open circuit breaker after the target restarts. +6. The frontier-report gate at the handler references the **commit index's + activation term** (the same value the sink's race-free in-lock gate uses), + not the machine's possibly-candidacy-inflated term. +7. A rejoining term-0 ex-leader's boot view is `None`, never + `Some(self-from-topology)` — caught by the gate before it shipped: the + adopted text only forbade *assuming the role*, but a self-pointing VIEW + still opened the §1.4-1 write-acceptance hole with the queue parked. + +## Exit-gate evidence (local 3-process clusters, tier-3, 2026-06-12) + +| Gate | Result | +|---|---| +| Auto-failover, zero acked loss | 5 rounds × random kill points under `ack=quorum` load, ZERO operator verbs: elections in **0.59–0.98 s** (budget 10 s); frontier invariant (elected leader's election-time position ≥ every acked seq) + content invariant (every acked item searchable on the new leader) held every round, including rounds where the killed node rejoined via quarantine + reseed | +| Fencing under partition + restart | leader isolated (bidirectional TCP severs) → survivors elected in ~0.7 s; the ex-leader restarted while partitioned and was **0-for-~30** on write attempts across the fence window, then healed and rejoined the new term as a converged follower | +| Bounded churn | 4 sub-timeout flaps absorbed with no elections + 3 super-timeout flaps → converged to ONE leader on ONE term ≤ 15, serving quorum writes | + +Full verification: 2,312 default-feature tests + all six tier-3 suites green; +workspace clippy clean (`--all-targets --all-features`, 0 warnings). +`cluster_quorum`/`cluster_lifecycle` pin `auto_election: false` (they validate +the manual drill, which the knob preserves); the quorum ledger drill gained +the promote-retry step (the election can refuse a stale max-applied sample — +the protocol now enforces the rule the runbook used to assign the operator). + +## Post-implementation review (seven-dimension, 2026-06-12) + +Twenty-one findings, adversarially verified: **10 confirmed → all fixed**, +11 refuted. The fixes that mattered: + +- **`joined_term` boots at ZERO** (three reviewers converged on it): it + tracks terms whose JOIN CHECK passed, and the durable term is not evidence + of that — initializing from it both fenced legitimate topology-era traffic + after a failed candidacy and skipped the divergence check on restart. The + fix surfaced a latent hole in the join check itself: the design's + `tail_term == joining term ⇒ clean` clause (§5, third bullet) was never + implemented, so the corrected init would have false-quarantined every + mid-term rejoiner. Both landed together; the failover gates re-proved them. +- **Torn-pair races**: `WalTermMark`'s `(term, seq, leader_region)` and the + node's activation `(prev_log_term, prev_log_seq)` were split atomics — a + reader between stores could pick the new term with the previous marker's + REGION and read a frontier in the wrong stream's numbering. Both are now + single-lock values (written once per term, read a few times a second), + and the activation pair publishes only AFTER the term marker is durable, + so an aborted activation never exposes it. +- Unit coverage added for `on_activation_failed` and `transfer_to` (the two + uncovered machine transitions). + +Notable refutations (recorded for the next reviewer): the "stale +`wal_term_mark` during join" claim — the mark and the frontier advance +through the same apply path, so a node's position is always internally +consistent; a behind position simply joins clean and catches up. And the +quarantine-metric-never-cleared claim — clearing rides m11p5's reseed +machinery by design. + +## Carried hazards (tracked, not regressions) + +- A reseeded node converges on the live stream but does not hold pre-baseline + history; quorum marks it contributes cover ranges it genuinely holds, but + full-history re-replication is **m11p5's snapshot transfer** (same gap the + d0a52e4 increment's "snapshot required" refusal points at). +- `mp_rolling_upgrade_no_loss_no_stall` is load-sensitive on saturated hosts + (the post-restart heal races the 30 s circuit breaker inside its 60 s + budget; widen via the harness env on slow runners). m11p8's self-driving + heal removes the pattern. +- Data below a term's activation baseline is unreachable through that term's + stream for followers that were behind at the transfer — pre-existing m11p2 + semantics, unchanged by this phase, resolved by p5 snapshots. diff --git a/docs/roadmap-to-cluster.md b/docs/roadmap-to-cluster.md index 2dae772..e967f12 100644 --- a/docs/roadmap-to-cluster.md +++ b/docs/roadmap-to-cluster.md @@ -1,6 +1,6 @@ # Roadmap to an Enterprise-Grade Cluster -**Status:** ADOPTED as M11 — m11p1 ✅, m11p2 ✅, m11p3 ✅ complete (2026-06-11), p2–p9 planned · **Date:** 2026-06-10 · **Baseline evidence:** +**Status:** ADOPTED as M11 — m11p1 ✅, m11p2 ✅, m11p3 ✅, m11p4 ✅ complete (2026-06-12 — closes **G5**; v0.9 "Credible HA" wave done) · p5–p9 planned · **Date:** 2026-06-10 · **Baseline evidence:** [stress-test-thepeach.md](ops/stress-test-thepeach.md), [cluster runbook](runbooks/cluster.md), [ROADMAP M8 Known Gaps](planning/ROADMAP.md) (G4/G5/G6), live k3s deployment (3 regions × 2-vCPU pods). @@ -267,8 +267,42 @@ channel (items/embeddings) is deleted. > still blocked on k3s access). Details: > [planning/milestone-11/phase-3.md](planning/milestone-11/phase-3.md). -### m11p4 — Failure detection, election, fencing (size: XL) — closes ROADMAP gap **G5** -**Goal:** "a machine died" is a non-event, not a runbook page. +### m11p4 — Failure detection, election, fencing (size: XL) — ✅ COMPLETE (2026-06-12) — closes ROADMAP gap **G5** + +> **✅ COMPLETE, as built:** purpose-built election-only Raft (pre-vote + +> check-quorum + fenced transfer) over the existing transport — no raft +> crate; the WAL is the log, and an elected leader's FIRST entry is a +> replicated kind-3 **term marker**, so Raft's `(lastLogTerm, lastLogIndex)` +> derives from one fsync stream (frontiers compared in the last joined +> term's STREAM numbering — a reseeded node's local numbering diverges). +> Hard state `(term, voted_for)` in `data_dir/election_state` (corrupt = +> refuse boot; lost-with-WAL = forced follower), fsynced before any vote or +> claim leaves. Term fencing on every RPC; the commit index folds only +> activation-term reports; a restarted ex-leader boots follower from durable +> state (§1.4-1 closed by construction). Divergent suffixes QUARANTINE +> (reseed is recovery; p5 snapshots automate). Promote = fenced transfer +> that refuses lagging targets — the runbook's "max-applied survivor" rule +> is now protocol. `auto_election: false` preserves the operator posture. +> **Exit gates (local tier-3, 3 processes): elections in 0.59–0.98 s under +> ack=quorum kill loads with zero acked loss across random kill points; a +> restarted partitioned ex-leader accepted 0 writes; 7 link flaps converged +> to one leader at term ≤ 15.** Details: +> [planning/milestone-11/phase-4.md](planning/milestone-11/phase-4.md). + +> **✅ Shipped increment (2026-06-11, d0a52e4) — catch-up self-healing + WAL segment versioning.** +> Failed `StreamSegments` pulls now arm a 30 s timer retry; re-arm-on-skip is the load-bearing +> liveness fix — without it a pull that fails in an idle cluster never re-fires and the follower stays +> permanently lagged (root cause: 2026-06-11 p3 rollout left both followers stuck at lag=136,507 with +> zero writes to trigger a retry). WAL segments carry a `TSEG` 8-byte header (magic + version byte + +> reserved); pre-p4 headerless segments are implicit v0, no migration; unknown format → +> `WalError::SegmentFormatUnknown` with no repair or truncation of foreign files. +> `FAILED_PRECONDITION` is the typed refusal for unservable catch-up. Regression gate +> (ops/stress-test-p4-regression.md): timer-only catch-up confirmed (lag=0 in 10 s with zero writes); +> p3 gate (≥1,000 quorum/s, p99 ≤50 ms, <1% error) holds through p4. The `Heartbeat` RPC and +> `ControlPlane::record_shard_heartbeat` already exist and health-track node liveness — they become +> the failure-detector input for the election work below. + +**Goal (main p4 work, pending):** "a machine died" is a non-event, not a runbook page. - **Replicated metadata state** (term/epoch, leader id, membership): Raft-style election over the existing transport — the quorum machinery from p3 is the diff --git a/docs/runbooks/cluster.md b/docs/runbooks/cluster.md index 44aca29..873e2fb 100644 --- a/docs/runbooks/cluster.md +++ b/docs/runbooks/cluster.md @@ -743,40 +743,90 @@ stay readable — no migration). Three behaviors follow: treats the header as a torn tail and may truncate the final segment. Downgrading across the m11p4 boundary requires reseeding the node's WAL. -## 9. Failover drill (multi-process) +## 9. Failover (multi-process) -Move the write leader to another region. Scripted exactly as the runbook-verification -suite (`cluster_runbook.rs::runbook_s9_failover_drill`) executes it: +### 9.1 Automatic failover (m11p4 — the default) -1. **Baseline.** `GET /cluster/status`; confirm the expected leader and - `lag_events: 0` on every region. -2. **Pre-seed reads.** Issue a region-pinned read against the target region - (`?region=eu-west`) to confirm it is serving and roughly caught up. -3. **Promote.** `POST /cluster/promote { "region": "eu-west" }`. Confirm the - `{ ok, leader, acked, failed }` response. If the OLD leader is dead, expect it - in `failed` — that is fine. **Under `ack=quorum`, promote the survivor with - the highest `applied_events`** (compare `/cluster/status/local` across - survivors): a quorum ack guarantees the write is on at least one follower's - contiguous frontier, so the max-applied survivor holds every acked write — - promoting any other node may discard acked data (m11p4's elections encode - this rule; until then it is the operator's). -4. **Verify.** `GET /cluster/status` now reports `eu-west` as leader. Send a write - (`POST /signals`) to the new leader and confirm `relay_log_len` advances and the - other regions' `applied_events` follow within a heartbeat. -5. **Cut over traffic.** Point your client's writes at **any** region gateway — a - write to a non-leader forwards to the new leader transparently (204), no client - change needed. +**"A machine died" is a non-event.** Every node runs a failure detector +(leader heartbeats every `election.heartbeat_interval_ms`, default 300) and a +Raft-style election (pre-vote + vote, randomized +`election.election_timeout_{min,max}_ms`, default 1500–3000). Kill the +leader and the survivors elect a successor — typically in **under one +second** with the defaults, bounded well inside 10s — with **zero operator +verbs** and zero acknowledged-write loss (the vote restriction only elects a +node whose log covers every quorum-acked write; the tier-3 +`cluster_election.rs::mp_auto_failover_writes_resume_zero_acked_loss` gate +proves it across repeated random kill points under `ack=quorum` load). -**Crash failover** is the same drill triggered by a real outage: a region's process -dies (its gateway stops answering), you detect it (monitoring on `/health` / -`/cluster/status` `reachable`), and you `POST /cluster/promote` a survivor via -**another** survivor's gateway. The tier-3 `mp_uat_step2_leader_crash_failover_under_10s` -test SIGKILLs the leader and proves promote→first-successful-write < 10s with zero -data loss. There is **no automatic detector/election** — promotion is the operator -step. +What the operator sees: -> This is a *leadership move*, not a quorum hand-off. Use it for "move the write -> region during maintenance" and for "a region died — promote a survivor." +- `/cluster/status/local` carries `term` (the election term, 0 = the + pre-election "topology era"), `role` (`leader`/`follower`/`pre-candidate`/ + `candidate`) and `quarantined`. +- During the brief leaderless window, writes return a retryable 503 naming + the election (`leader: "none (election in progress)"` plus the responding + node's `term`); clients retry and land on the new leader. +- A **restarted ex-leader can never re-claim leadership from its topology + file**: its durable election state (`data_dir/election_state`) boots it as + a follower, and every replication RPC is term-fenced — a deposed leader's + ships, heartbeats and frontier reports are rejected until it rejoins the + current term (the §1.4-1 split-brain incident is closed by construction; + proven by `mp_fenced_ex_leader_restart_cannot_write`). +- A leader that loses contact with a majority **steps down within + `election.leader_lease_ms`** (default 900) and stops accepting writes: + `ack=leader` writes during a minority partition are bounded by the lease, + and `ack=quorum` writes were never at risk. + +**Divergent suffix / quarantine.** A node that held leader-acked (never +quorum-acked) writes when it died can rejoin into a cluster that elected +past them. It detects this at term-join and **quarantines**: it serves +status (`quarantined: true`, metric +`tidaldb_cluster_divergence_quarantined`) and keeps voting, but refuses the +data plane. Recovery is a **reseed**: stop it, wipe its data dir, restart — +it recovers the current log via the catch-up stream (m11p5's snapshot +transfer automates full-history reseeds). This is strictly leader-ack-only +data, within the documented `ack=leader` crash contract (§8). + +To run the pre-m11p4 posture (operator-driven failover, no automatic +elections, no check-quorum step-down), set in the topology: + +```yaml +election: + auto_election: false +``` + +### 9.2 Manual promote: a FENCED TRANSFER (maintenance / override) + +`POST /cluster/promote { "region": "eu-west" }` remains the maintenance verb +— but it is now a **fenced leadership transfer**, not a view flip: + +1. With a live leader: the leader waits for the target to hold the full + flushed prefix (the catch-up wait IS the drain), then sanctions an + immediate election (`TimeoutNow`); the target wins term+1 and the old + leader steps down on first higher-term contact. Response: + `{ ok, leader, term, transfer: "elected" }`. +2. With a dead leader: the target campaigns among the survivors directly — + the manual override of the automatic path (and the way to *choose* the + successor). +3. The election can **refuse a target that lags** (its log loses the + up-to-date comparison — e.g. the survivor you sampled fell behind between + your status read and the vote). The verb 503s naming the cause; promote + the other survivor. **You can no longer accidentally promote a node that + would discard acked data** — the m11p3 "max-applied survivor" operator + rule is now enforced by the protocol. +4. Promoting the node that already leads is a no-op 200. + +The legacy term-0 fan-out promote survives only for clusters that have never +elected (mixed-version rollouts mid-upgrade, and the deliberate +isolated-node override used by the chaos drills); the first joined election +permanently retires it on each node. + +> Rolling upgrade m11p3 → m11p4: upgrade ALL binaries before relying on +> auto-failover (pre-m11p4 peers answer vote RPCs with `Unimplemented`, so +> no election can reach quorum until a majority is upgraded — the cluster +> simply keeps its m11p3 behavior until then). The first ELECTED leader +> journals a kind-3 term-marker WAL record; pre-m11p4 binaries cannot decode +> it, so do not downgrade a node after the first election without reseeding. ## 10. Partition drill (multi-process) diff --git a/tidal-net/benches/transport_throughput.rs b/tidal-net/benches/transport_throughput.rs index 5aa748c..d948a93 100644 --- a/tidal-net/benches/transport_throughput.rs +++ b/tidal-net/benches/transport_throughput.rs @@ -24,6 +24,8 @@ fn make_payload(seqno: u64) -> WalSegmentPayload { event_count: 10, leader_last_seq: seqno, stream_baseline: 0, + term: 0, + leader_region: 0, } } diff --git a/tidal-net/proto/wal_shipping.proto b/tidal-net/proto/wal_shipping.proto index 5e0b37b..09bfa68 100644 --- a/tidal-net/proto/wal_shipping.proto +++ b/tidal-net/proto/wal_shipping.proto @@ -27,6 +27,13 @@ message ShipSegmentRequest { // default, and what every live unary ship carries) means "stream from the // beginning". uint64 stream_baseline = 5; + // The sender's leadership term (m11p4 fencing). 0 = the topology era or a + // pre-m11p4 sender; a receiver at term >= 1 rejects term-0 traffic, and any + // receiver rejects a term below its own (FAILED_PRECONDITION) — a deposed + // leader's ships can never apply. + uint64 term = 6; + // The region claiming leadership of `term` (m11p4). + uint32 leader_region = 7; } // Response to a segment shipment. @@ -37,12 +44,19 @@ message ShipSegmentResponse { // frontier so retries of already-applied data prune and heal needs no // separate status fetch. 0 = unknown (older peer / no applied source wired). uint64 applied_seqno = 2; + // The receiver's current term (m11p4): a value above the sender's term is + // the sender's step-down signal. + uint64 term = 3; } // Request to stream segments from a given sequence number. message StreamRequest { uint32 shard_id = 1; uint64 from_seqno = 2; + // The puller's current term (m11p4). The source serves only when the terms + // match: a stale puller must rejoin first, and a stale SOURCE must step + // down rather than serve its possibly-divergent tail. + uint64 term = 3; } // Heartbeat request matching ControlPlane's ShardStats. @@ -55,11 +69,34 @@ message HeartbeatRequest { // Replication lag per peer region (region_id -> lag in events). map replication_lag = 6; uint64 last_heartbeat_ns = 7; + // The sender's leadership term (m11p4): a leader heartbeat is the lease + // assertion + failure-detector input. 0 = the topology era / a non-leader + // health probe. + uint64 term = 8; + // The region asserting leadership of `term` (m11p4). + uint32 leader_region = 9; + // The asserted term's activation stream baseline (m11p4): immutable for + // the term; a joining follower jumps its applied frontier for the leader's + // stream to it (seqnos at or below are pre-stream history). + uint64 stream_baseline = 10; + // The leader's ELECTION-TIME log position (m11p4): the term and frontier + // of its log in the PREVIOUS stream's numbering — the same pair the vote + // restriction compares. A joining node is DIVERGENT iff its own position + // exceeds this lexicographically (it holds entries the new leadership's + // history does not subsume). + uint64 prev_log_term = 11; + uint64 prev_log_seq = 12; } // Heartbeat acknowledgement. message HeartbeatResponse { bool acknowledged = 1; + // The responder's current term (m11p4): above the sender's term = the + // sender's step-down signal. + uint64 term = 2; + // Whether the responder accepted the sender's leadership assertion + // (false = the sender's term is stale). + bool accepted = 3; } // A follower's self-report of its durable frontier (m11p3). @@ -74,6 +111,10 @@ message AppliedReport { uint32 source_shard = 2; // The reporter's contiguous durably-applied seqno for that stream. uint64 applied_seqno = 3; + // The reporter's current term (m11p4): the leader folds a report into its + // quorum commit index ONLY when this matches the index's activation term — + // a stale or cross-leadership report can never advance commitment. + uint64 reporter_term = 4; } // Applied-report acknowledgement. @@ -81,6 +122,43 @@ message AppliedReportAck { bool acknowledged = 1; } +// A pre-vote or vote request (m11p4 leader election). +message VoteRequest { + // The term votes are requested for. For a pre-vote this is the PROPOSED + // term (candidate's current + 1) — nothing has been bumped. + uint64 term = 1; + uint32 candidate_region = 2; + // The candidate's log position for the up-to-date restriction, + // compared lexicographically: (last_log_term, last_log_seq). + uint64 last_log_term = 3; + uint64 last_log_seq = 4; + // Pre-vote probe: changes no voter state, never inflates terms. + bool prevote = 5; + // Leadership-transfer election (`TimeoutNow`): voters skip the + // leader-freshness refusal — the current leader sanctioned this. + bool transfer = 6; +} + +// A vote (or pre-vote) reply. +message VoteResponse { + // The voter's current term (above the candidate's = step-down signal). + uint64 term = 1; + bool granted = 2; +} + +// The current leader tells `target` to start an immediate transfer election +// (m11p4 fenced promote). +message TimeoutNowRequest { + // The sanctioning leader's current term. + uint64 term = 1; + uint32 leader_region = 2; +} + +message TimeoutNowResponse { + // Whether the target started an election. + bool accepted = 1; +} + // WAL segment shipping service between tidalDB shards. service WalShipping { // Ship a single WAL segment to a peer shard (unary). @@ -89,9 +167,16 @@ service WalShipping { // Stream WAL segments from a given sequence number (server-streaming). rpc StreamSegments(StreamRequest) returns (stream ShipSegmentRequest); - // Periodic health check for the ControlPlane. + // Periodic health check for the ControlPlane; with m11p4, the leader's + // lease assertion and the failure detector's input. rpc Heartbeat(HeartbeatRequest) returns (HeartbeatResponse); // Follower -> leader durable-frontier report (m11p3 quorum acks). rpc ReportApplied(AppliedReport) returns (AppliedReportAck); + + // Pre-vote / vote (m11p4 leader election). + rpc RequestVote(VoteRequest) returns (VoteResponse); + + // Fenced leadership transfer: start an immediate election (m11p4). + rpc TimeoutNow(TimeoutNowRequest) returns (TimeoutNowResponse); } diff --git a/tidal-net/src/client.rs b/tidal-net/src/client.rs index 1712d36..312d215 100644 --- a/tidal-net/src/client.rs +++ b/tidal-net/src/client.rs @@ -115,9 +115,11 @@ impl PeerPool { /// Send a WAL segment to a peer shard. /// - /// On success returns the peer's self-reported applied seqno from the ack - /// (`0` = unknown / older peer) — the m11p2 piggyback the ship queue folds - /// into its acked frontier. + /// On success returns `(applied_seqno, responder_term)`: the peer's + /// self-reported applied seqno from the ack (`0` = unknown / older peer) + /// — the m11p2 piggyback the ship queue folds into its acked frontier — + /// and the peer's current term (m11p4; a value above the sender's is the + /// sender's step-down signal). /// /// # Errors /// @@ -132,7 +134,7 @@ impl PeerPool { &self, shard: ShardId, payload: WalSegmentPayload, - ) -> Result { + ) -> Result<(u64, u64), GrpcTransportError> { let peer = self .peers .get(&shard) @@ -166,7 +168,7 @@ impl PeerPool { let inner = response.into_inner(); if inner.accepted { peer.circuit_breaker.record_success(); - Ok(inner.applied_seqno) + Ok((inner.applied_seqno, inner.term)) } else { peer.circuit_breaker.record_backpressure(); Err(GrpcTransportError::SegmentRejected(shard)) @@ -192,6 +194,7 @@ impl PeerPool { to: ShardId, reporter: ShardId, applied: u64, + reporter_term: u64, ) -> Result<(), GrpcTransportError> { let Some(peer) = self.peers.get(&to) else { return Err(GrpcTransportError::PeerUnreachable(to)); @@ -204,6 +207,7 @@ impl PeerPool { reporter_shard: u32::from(reporter.0), source_shard: u32::from(to.0), applied_seqno: applied, + reporter_term, }); client .report_applied(request) @@ -214,7 +218,8 @@ impl PeerPool { /// Open a `StreamSegments` catch-up stream from `shard` starting at /// `from_seqno` (m11p2: follower-pulled catch-up over the leader's - /// durable WAL). + /// durable WAL). `term` is the puller's current term (m11p4): the source + /// serves only a same-term puller. /// /// Deliberately bypasses the circuit breaker: pulls are already /// rate-limited and single-flight per shard at the transport layer, and a @@ -229,6 +234,7 @@ impl PeerPool { &self, shard: ShardId, from_seqno: u64, + term: u64, ) -> Result, GrpcTransportError> { let peer = self .peers @@ -238,12 +244,85 @@ impl PeerPool { let request = crate::proto::StreamRequest { shard_id: u32::from(shard.0), from_seqno, + term, }; match client.stream_segments(request).await { Ok(response) => Ok(response.into_inner()), Err(status) => Err(GrpcTransportError::Grpc(Box::new(status))), } } + + // ── Election RPCs (m11p4) ─────────────────────────────────────────────── + // + // None of these touch the circuit breaker: an election MUST be able to + // probe a peer the breaker quarantined (the breaker reflects the DATA + // plane), the traffic is rare and self-limiting, and classifying a vote + // refusal as a transport failure would starve the very mechanism that + // replaces a dead leader. + + /// Send a pre-vote or vote request. + /// + /// # Errors + /// + /// Unknown peer or transport/RPC failure (including `Unimplemented` from + /// a pre-m11p4 peer — the caller counts it as not-granted). + pub async fn request_vote( + &self, + to: ShardId, + request: crate::proto::VoteRequest, + ) -> Result { + let Some(peer) = self.peers.get(&to) else { + return Err(GrpcTransportError::PeerUnreachable(to)); + }; + let mut client = peer.client.clone(); + client + .request_vote(request) + .await + .map(tonic::Response::into_inner) + .map_err(|status| GrpcTransportError::Grpc(Box::new(status))) + } + + /// Send a leader heartbeat / lease assertion. + /// + /// # Errors + /// + /// Unknown peer or transport/RPC failure. + pub async fn heartbeat( + &self, + to: ShardId, + request: crate::proto::HeartbeatRequest, + ) -> Result { + let Some(peer) = self.peers.get(&to) else { + return Err(GrpcTransportError::PeerUnreachable(to)); + }; + let mut client = peer.client.clone(); + client + .heartbeat(request) + .await + .map(tonic::Response::into_inner) + .map_err(|status| GrpcTransportError::Grpc(Box::new(status))) + } + + /// Tell `to` to start an immediate transfer election. + /// + /// # Errors + /// + /// Unknown peer or transport/RPC failure. + pub async fn timeout_now( + &self, + to: ShardId, + request: crate::proto::TimeoutNowRequest, + ) -> Result { + let Some(peer) = self.peers.get(&to) else { + return Err(GrpcTransportError::PeerUnreachable(to)); + }; + let mut client = peer.client.clone(); + client + .timeout_now(request) + .await + .map(tonic::Response::into_inner) + .map_err(|status| GrpcTransportError::Grpc(Box::new(status))) + } } #[cfg(test)] diff --git a/tidal-net/src/convert.rs b/tidal-net/src/convert.rs index bc69cb3..3eb1128 100644 --- a/tidal-net/src/convert.rs +++ b/tidal-net/src/convert.rs @@ -22,6 +22,8 @@ impl From for proto::ShipSegmentRequest { event_count: p.event_count, leader_last_seq: p.leader_last_seq, stream_baseline: p.stream_baseline, + term: p.term, + leader_region: p.leader_region.into(), } } } @@ -39,12 +41,18 @@ impl TryFrom for WalSegmentPayload { .shard_id .try_into() .map_err(|_| "shard_id exceeds u16 range")?; + let leader_region: u16 = req + .leader_region + .try_into() + .map_err(|_| "leader_region exceeds u16 range")?; Ok(Self { id: WalSegmentId::new(RegionId(region_id), ShardId(shard_id), id.seqno), bytes: req.payload, event_count: req.event_count, leader_last_seq: req.leader_last_seq, stream_baseline: req.stream_baseline, + term: req.term, + leader_region, }) } } @@ -62,6 +70,8 @@ mod tests { event_count: 7, leader_last_seq: 48, stream_baseline: 9, + term: 0, + leader_region: 0, }; let proto_req: proto::ShipSegmentRequest = original.into(); @@ -98,6 +108,8 @@ mod tests { event_count: 1, leader_last_seq: 0, stream_baseline: 0, + term: 0, + leader_region: 0, }; let restored = WalSegmentPayload::try_from(req).unwrap(); assert_eq!(restored.leader_last_seq, 0); @@ -111,6 +123,8 @@ mod tests { event_count: 0, leader_last_seq: 0, stream_baseline: 0, + term: 0, + leader_region: 0, }; assert!(WalSegmentPayload::try_from(req).is_err()); } @@ -127,6 +141,8 @@ mod tests { event_count: 0, leader_last_seq: 0, stream_baseline: 0, + term: 0, + leader_region: 0, }; assert!(WalSegmentPayload::try_from(req).is_err()); } @@ -143,6 +159,8 @@ mod tests { event_count: 0, leader_last_seq: 0, stream_baseline: 0, + term: 0, + leader_region: 0, }; assert!(WalSegmentPayload::try_from(req).is_err()); } diff --git a/tidal-net/src/error.rs b/tidal-net/src/error.rs index 7872297..0bd6f2d 100644 --- a/tidal-net/src/error.rs +++ b/tidal-net/src/error.rs @@ -60,9 +60,13 @@ impl GrpcTransportError { /// server will return identically on retry — `Unauthenticated` / /// `PermissionDenied` (mTLS / authz rejection), `InvalidArgument` / /// `OutOfRange` (the payload itself is malformed), `Unimplemented` (the - /// RPC does not exist on the peer). Everything else (`Unavailable`, - /// `ResourceExhausted`, `DeadlineExceeded`, codec-size on a transient - /// over-large segment, …) is transient. + /// RPC does not exist on the peer), `FailedPrecondition` (the m11p4 + /// term fence: a stale-term ship fails identically until leadership + /// changes — quarantining the peer is right, and the deposed sender's + /// step-down deactivates the whole queue moments later anyway). + /// Everything else (`Unavailable`, `ResourceExhausted`, + /// `DeadlineExceeded`, codec-size on a transient over-large segment, …) + /// is transient. /// - [`TonicTransport`](Self::TonicTransport): a connect/handshake error. /// These are predominantly transient (peer not up yet) and are treated as /// such; a genuinely permanent handshake fault surfaces as a `Grpc` TLS @@ -81,6 +85,7 @@ impl GrpcTransportError { | tonic::Code::InvalidArgument | tonic::Code::OutOfRange | tonic::Code::Unimplemented + | tonic::Code::FailedPrecondition ), Self::CircuitOpen(_) | Self::PeerUnreachable(_) @@ -148,6 +153,7 @@ mod tests { tonic::Code::InvalidArgument, tonic::Code::OutOfRange, tonic::Code::Unimplemented, + tonic::Code::FailedPrecondition, ] { let e = GrpcTransportError::Grpc(Box::new(tonic::Status::new(code, "x"))); assert!(e.is_permanent(), "{code:?} must be permanent"); diff --git a/tidal-net/src/lib.rs b/tidal-net/src/lib.rs index 12e4aa4..8a3ca0c 100644 --- a/tidal-net/src/lib.rs +++ b/tidal-net/src/lib.rs @@ -39,4 +39,5 @@ pub mod proto { pub use config::{GrpcTransportConfig, TlsConfig}; pub use error::GrpcTransportError; -pub use transport::{GrpcTransport, GrpcTransportFactory}; +pub use sources::{ClaimRejection, ElectionHooks, HeartbeatExchange}; +pub use transport::{ElectionNet, ElectionNetEvent, GrpcTransport, GrpcTransportFactory}; diff --git a/tidal-net/src/server.rs b/tidal-net/src/server.rs index e88d387..39638bd 100644 --- a/tidal-net/src/server.rs +++ b/tidal-net/src/server.rs @@ -11,7 +11,8 @@ use crate::{ config::GrpcTransportConfig, proto::{ AppliedReport, AppliedReportAck, HeartbeatRequest, HeartbeatResponse, ShipSegmentRequest, - ShipSegmentResponse, StreamRequest, WalSegmentId, + ShipSegmentResponse, StreamRequest, TimeoutNowRequest, TimeoutNowResponse, VoteRequest, + VoteResponse, WalSegmentId, wal_shipping_server::{WalShipping, WalShippingServer}, }, sources::ServingSources, @@ -88,6 +89,10 @@ impl WalShippingService { } } +// The async_trait expansion folds every handler into one item, so the lint +// must sit on the impl: ship_segment is one linear pass (size guard -> term +// fence -> convert -> enqueue) whose split would scatter the ack contract. +#[allow(clippy::too_many_lines)] #[tonic::async_trait] impl WalShipping for WalShippingService { async fn ship_segment( @@ -105,6 +110,40 @@ impl WalShipping for WalShippingService { ))); } + // Term fence (m11p4): a stale-term ship is rejected BEFORE it can + // enter the inbound queue — a deposed leader's traffic never reaches + // the apply path. A current/higher claim records leader contact (and + // any adoption persisted before this returns). + let response_term = if let Some(hooks) = self.sources.election.get() { + let leader_region = + u16::try_from(req.leader_region).map_err(|_| { + Status::invalid_argument("leader_region exceeds u16 range") + })?; + let first_seq = req.id.as_ref().map_or(0, |id| id.seqno); + match hooks.observe_leader_claim(req.term, leader_region, first_seq) { + Ok(()) => {} + Err(crate::sources::ClaimRejection::Stale { current_term }) => { + return Err(Status::failed_precondition(format!( + "stale leadership term {} (current term {current_term}); \ + this ship is fenced", + req.term + ))); + } + Err(crate::sources::ClaimRejection::JoinPending) => { + // Transient: the next heartbeat round runs the term-join + // check; UNAVAILABLE keeps the sender retrying instead of + // quarantining the peer. + return Err(Status::unavailable(format!( + "term {} not joined yet (awaiting the leader's heartbeat); retry", + req.term + ))); + } + } + hooks.self_claim().0 + } else { + 0 + }; + let source_shard = req.id.as_ref().map_or(0, |id| id.shard_id); // Convert proto to domain type. @@ -129,6 +168,7 @@ impl WalShipping for WalShippingService { Ok(()) => Ok(Response::new(ShipSegmentResponse { accepted: true, applied_seqno, + term: response_term, })), Err(mpsc::error::TrySendError::Full(payload)) => { tracing::warn!("inbound channel full; yielding and retrying"); @@ -137,10 +177,12 @@ impl WalShipping for WalShippingService { Ok(()) => Ok(Response::new(ShipSegmentResponse { accepted: true, applied_seqno, + term: response_term, })), Err(_) => Ok(Response::new(ShipSegmentResponse { accepted: false, applied_seqno, + term: response_term, })), } } @@ -166,6 +208,32 @@ impl WalShipping for WalShippingService { )); }; + // Term fence (m11p4, design-review C14): the source serves only a + // SAME-TERM puller. A newer-term puller means this source is deposed + // — it must step down, not serve its possibly-divergent tail; a + // stale puller must rejoin the current term before pulling. + let (chunk_term, chunk_leader_region) = if let Some(hooks) = self.sources.election.get() { + let (current_term, self_region) = hooks.self_claim(); + if req.term > current_term { + hooks.on_observed_term(req.term); + return Err(Status::failed_precondition(format!( + "this source's term {current_term} is behind the puller's {}; \ + stepping down — pull from the current leader", + req.term + ))); + } + if req.term < current_term { + return Err(Status::failed_precondition(format!( + "puller term {} is behind the source's {current_term}; \ + rejoin the current term before pulling", + req.term + ))); + } + (current_term, u32::from(self_region)) + } else { + (0, 0) + }; + // The stream serves THIS node's own log; a request for any other // shard's stream is a routing error. let serving = source.source_shard(); @@ -266,6 +334,8 @@ impl WalShipping for WalShippingService { event_count: chunk.event_count, leader_last_seq: chunk.last_seq, stream_baseline: baseline, + term: chunk_term, + leader_region: chunk_leader_region, }; if tx.send(Ok(msg)).await.is_err() { return; // client went away; stop reading @@ -283,34 +353,46 @@ impl WalShipping for WalShippingService { &self, request: Request, ) -> Result, Status> { - // MINIMAL REAL BEHAVIOR, not a silent stub: a heartbeat that reaches - // this handler proves the gRPC server is up, the listener is accepting, - // and (under mTLS) the peer's certificate was accepted — i.e. a genuine - // network-liveness probe. Returning `acknowledged: true` is therefore a - // truthful answer to "are you reachable", which is exactly what the - // failure detector needs from the transport layer. + // A heartbeat that reaches this handler proves the gRPC server is up, + // the listener is accepting, and (under mTLS) the peer's certificate + // was accepted — a genuine network-liveness probe. With m11p4 it is + // also the leader's LEASE ASSERTION: the election hooks fold the + // term/leadership claim into the failure detector and answer with + // this node's term (a higher one is the sender's step-down signal). let req = request.into_inner(); - // Record the peer's claimed identity at debug! so a probe is observable - // (which shard/region reached us, when) without being silently dropped on - // the floor. Until the identity is wired into the engine's health table - // (below) this is the only place the claim is captured, so logging it is - // load-bearing for "who is heartbeating me" during a cluster incident. tracing::debug!( shard_id = req.shard_id, region_id = req.region_id, + term = req.term, "received heartbeat", ); - // DEFERRED, tracked: forwarding `req.shard_id` / `req.region_id` liveness - // into the engine's `ControlPlane` health table (so a missed heartbeat - // demotes the claiming peer, and a spoofed/mismatched identity is flagged) - // is an additive enrichment, not a contract change — it only widens who - // observes the claim, captured here. Known gap in the M8 cluster scope; - // see docs/runbooks/cluster.md (health checks) and CHANGELOG.md "Known - // gaps". The transport carries no `ControlPlane` handle today; wiring one - // is the remaining work. - Ok(Response::new(HeartbeatResponse { acknowledged: true })) + if let Some(hooks) = self.sources.election.get() { + let leader_region = u16::try_from(req.leader_region) + .map_err(|_| Status::invalid_argument("leader_region exceeds u16 range"))?; + let verdict = hooks.on_heartbeat( + req.term, + leader_region, + req.stream_baseline, + tidaldb::replication::LogPosition { + tail_term: req.prev_log_term, + frontier: req.prev_log_seq, + }, + ); + return Ok(Response::new(HeartbeatResponse { + acknowledged: true, + term: verdict.term, + accepted: verdict.accepted, + })); + } + // Pre-m11p4 behavior (no election driver wired): a truthful + // reachability ack. + Ok(Response::new(HeartbeatResponse { + acknowledged: true, + term: 0, + accepted: true, + })) } async fn report_applied( @@ -337,12 +419,73 @@ impl WalShipping for WalShippingService { segments.source_shard().0 ))); } + // Term fence (m11p4, design-review C4/C8/C12): a report from any + // term but the current leadership's never reaches the hint map or + // the commit index — a deposed leader's followers (or a delayed + // report from a previous leadership) cannot advance commitment. The + // sink re-checks under the commit index's own lock (the race-free + // gate); this handler-level check keeps the HINT MAP equally clean. + if let Some(hooks) = self.sources.election.get() + && !hooks.report_term_acceptable(report.reporter_term) + { + return Err(Status::failed_precondition(format!( + "frontier report term {} does not match the current leadership term {}; \ + report dropped", + report.reporter_term, + hooks.self_claim().0 + ))); + } fold_peer_applied(&self.peer_applied, reporter, report.applied_seqno); if let Some(sink) = self.sources.applied_sink.get() { - sink.peer_applied(reporter, report.applied_seqno); + sink.peer_applied(reporter, report.applied_seqno, report.reporter_term); } Ok(Response::new(AppliedReportAck { acknowledged: true })) } + + async fn request_vote( + &self, + request: Request, + ) -> Result, Status> { + let Some(hooks) = self.sources.election.get() else { + return Err(Status::unimplemented( + "RequestVote: no election driver on this node (pre-m11p4 binary or bare \ + transport)", + )); + }; + let req = request.into_inner(); + let candidate = u16::try_from(req.candidate_region) + .map_err(|_| Status::invalid_argument("candidate_region exceeds u16 range"))?; + let reply = hooks.on_vote(tidaldb::replication::VoteRpc { + term: req.term, + candidate: tidaldb::replication::shard::RegionId(candidate), + log: tidaldb::replication::LogPosition { + tail_term: req.last_log_term, + frontier: req.last_log_seq, + }, + prevote: req.prevote, + transfer: req.transfer, + }); + Ok(Response::new(VoteResponse { + term: reply.term, + granted: reply.granted, + })) + } + + async fn timeout_now( + &self, + request: Request, + ) -> Result, Status> { + let Some(hooks) = self.sources.election.get() else { + return Err(Status::unimplemented( + "TimeoutNow: no election driver on this node", + )); + }; + let req = request.into_inner(); + let leader_region = u16::try_from(req.leader_region) + .map_err(|_| Status::invalid_argument("leader_region exceeds u16 range"))?; + let accepted = hooks.on_timeout_now(req.term, leader_region); + Ok(Response::new(TimeoutNowResponse { accepted })) + } } /// Start the gRPC server on the given address. @@ -435,6 +578,8 @@ mod tests { event_count: 1, leader_last_seq: 1, stream_baseline: 0, + term: 0, + leader_region: 0, } } @@ -512,6 +657,7 @@ mod tests { applied: Some(Arc::new(FixedApplied)), segments: None, applied_sink: Arc::default(), + election: Arc::default(), }; let service = WalShippingService::new(tx, 1024, sources, Arc::new(Mutex::new(HashMap::new()))); @@ -535,7 +681,7 @@ mod tests { fn report_applied_folds_marks_and_feeds_the_sink() { struct RecordingSink(Mutex>); impl crate::sources::AppliedSink for RecordingSink { - fn peer_applied(&self, peer: ShardId, applied: u64) { + fn peer_applied(&self, peer: ShardId, applied: u64, _reporter_term: u64) { self.0.lock().unwrap().push((peer, applied)); } } @@ -569,6 +715,7 @@ mod tests { applied: None, segments: Some(Arc::new(FixedSegments)), applied_sink: Arc::default(), + election: Arc::default(), }; sources.set_applied_sink(Arc::clone(&sink) as Arc); let map: PeerAppliedMap = Arc::new(Mutex::new(HashMap::new())); @@ -579,6 +726,7 @@ mod tests { reporter_shard: reporter, source_shard: source, applied_seqno: applied, + reporter_term: 0, }; service .report_applied(Request::new(report(2, 0, 9))) @@ -661,6 +809,7 @@ mod tests { applied: None, segments: Some(Arc::new(FakeSegments)), applied_sink: Arc::default(), + election: Arc::default(), }; let service = WalShippingService::new(tx, 1024, sources, Arc::new(Mutex::new(HashMap::new()))); @@ -670,7 +819,8 @@ mod tests { .stream_segments(Request::new(StreamRequest { shard_id: 3, from_seqno: 1, - })) + term: 0, + })) .await .expect("stream must open"); let mut stream = resp.into_inner(); @@ -690,7 +840,8 @@ mod tests { .stream_segments(Request::new(StreamRequest { shard_id: 9, from_seqno: 1, - })) + term: 0, + })) .await .expect_err("wrong shard must be refused"); assert_eq!(err.code(), tonic::Code::NotFound); @@ -738,6 +889,7 @@ mod tests { applied: None, segments: Some(Arc::new(UnservableSegments)), applied_sink: Arc::default(), + election: Arc::default(), }; let service = WalShippingService::new(tx, 1024, sources, Arc::new(Mutex::new(HashMap::new()))); @@ -746,7 +898,8 @@ mod tests { .stream_segments(Request::new(StreamRequest { shard_id: 0, from_seqno: 5, - })) + term: 0, + })) .await .expect("the stream opens; the failure arrives as the first message"); let mut stream = resp.into_inner(); diff --git a/tidal-net/src/sources.rs b/tidal-net/src/sources.rs index 605fdd1..2dd7532 100644 --- a/tidal-net/src/sources.rs +++ b/tidal-net/src/sources.rs @@ -46,8 +46,12 @@ pub trait AppliedSource: Send + Sync + 'static { /// [`ServingSources`]. pub trait AppliedSink: Send + Sync + 'static { /// Fold a follower's durable mark (monotonic; stale or unknown-peer - /// reports must be ignored by the implementation). - fn peer_applied(&self, peer: ShardId, applied: u64); + /// reports must be ignored by the implementation). `reporter_term` is + /// the reporter's current term (m11p4): the implementation must fold + /// ONLY when it matches the commit index's activation term — the gate + /// runs under the index's own lock so a leadership change can never + /// race a fold (design-review C4/C8/C12). + fn peer_applied(&self, peer: ShardId, applied: u64, reporter_term: u64); } /// Why a segment read-back could not serve a catch-up request. @@ -118,6 +122,102 @@ pub trait SegmentSource: Send + Sync + 'static { ) -> Result, SegmentReadError>; } +/// The heartbeat exchange verdict an [`ElectionHooks`] implementation +/// returns: the responder's (possibly just-raised) term and whether the +/// sender's leadership assertion was accepted. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HeartbeatExchange { + /// The responder's current term. + pub term: u64, + /// Whether the sender's term was accepted as current leadership. + pub accepted: bool, +} + +/// Why an inbound leadership-stamped payload was refused (m11p4). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ClaimRejection { + /// The claim's term is below this node's (or this node is quarantined): + /// permanent for that leadership — the sender steps down or stays fenced. + Stale { + /// This node's current term. + current_term: u64, + }, + /// The claim's term is NEW to this node and the term-join check (which + /// only a heartbeat, carrying the leader's election-time position, can + /// run) has not happened yet. TRANSIENT: the sender retries after the + /// next heartbeat round joins the term. + JoinPending, +} + +/// Node-side election integration (m11p4), late-bound like [`AppliedSink`]. +/// +/// The gRPC service consults it to FENCE inbound traffic (stale-term ships, +/// stream chunks and frontier reports never reach the apply path) and to +/// answer the election RPCs. Absent hooks = the pre-m11p4 behavior: every +/// inbound is accepted, vote RPCs answer `Unimplemented` — exactly what a +/// bare transport (tests, in-process harnesses) wants. +/// +/// Implementations own the election state machine and its durable hard +/// state; every method that can adopt a term MUST persist before returning +/// (the reply is the externally visible action). +pub trait ElectionHooks: Send + Sync + 'static { + /// This node's `(current_term, region)` — stamped on outbound responses + /// and served stream chunks. + fn self_claim(&self) -> (u64, u16); + + /// An inbound leadership-stamped data payload (live ship or catch-up + /// chunk) covering seqnos from `first_seq`. `Ok(())` = accept (the claim + /// was current; leader contact was recorded; any higher term was + /// adopted-and-persisted before return). `Err(current_term)` = the + /// payload is fenced; the caller rejects it with `FAILED_PRECONDITION` + /// carrying the current term. + /// + /// # Errors + /// + /// [`ClaimRejection::Stale`] when `term` is below this node's current + /// term or this node is quarantined (divergent suffix — no data-plane + /// participation until reseeded); [`ClaimRejection::JoinPending`] when + /// the term is new and only the next heartbeat (which carries the + /// leader's election-time position) can run the divergence check — + /// transient, the sender retries. + fn observe_leader_claim( + &self, + term: u64, + leader_region: u16, + first_seq: u64, + ) -> Result<(), ClaimRejection>; + + /// A leader heartbeat: term + leadership + the term's activation + /// baseline + the leader's election-time log position + /// `(prev_log_term, prev_log_seq)`. Drives the failure detector and the + /// term-join divergence check. + fn on_heartbeat( + &self, + term: u64, + leader_region: u16, + stream_baseline: u64, + prev_log: tidaldb::replication::LogPosition, + ) -> HeartbeatExchange; + + /// A pre-vote or vote request. The grant (and any term adoption) is + /// durable before this returns. + fn on_vote(&self, rpc: tidaldb::replication::VoteRpc) -> tidaldb::replication::VoteReply; + + /// The current leader sanctioned an immediate transfer election. + /// Returns whether a candidacy started. + fn on_timeout_now(&self, term: u64, leader_region: u16) -> bool; + + /// A response/request carried a term above ours outside the paths above + /// (e.g. a stale-source `StreamSegments` request from a newer-term + /// puller): adopt + persist + step down. + fn on_observed_term(&self, term: u64); + + /// Whether a frontier report stamped `reporter_term` may fold into the + /// quorum commit index / hint map (true iff it matches the current + /// leadership term). + fn report_term_acceptable(&self, reporter_term: u64) -> bool; +} + /// The optional node-side sources handed to [`crate::GrpcTransport`]. #[derive(Clone, Default)] pub struct ServingSources { @@ -131,6 +231,10 @@ pub struct ServingSources { /// [`ServingSources::set_applied_sink`] once available. Reports arriving /// before it is set fold into the transport's hint map only. pub applied_sink: Arc>>, + /// Election integration (m11p4), late-bound for the same reason as + /// `applied_sink` (the election driver is built after the transport). + /// Absent = pre-m11p4 behavior (no fencing, vote RPCs unimplemented). + pub election: Arc>>, } impl ServingSources { @@ -138,6 +242,11 @@ impl ServingSources { pub fn set_applied_sink(&self, sink: Arc) { let _ = self.applied_sink.set(sink); } + + /// Late-bind the election hooks (idempotent; first set wins). + pub fn set_election_hooks(&self, hooks: Arc) { + let _ = self.election.set(hooks); + } } impl std::fmt::Debug for ServingSources { @@ -146,6 +255,7 @@ impl std::fmt::Debug for ServingSources { .field("applied", &self.applied.is_some()) .field("segments", &self.segments.is_some()) .field("applied_sink", &self.applied_sink.get().is_some()) + .field("election", &self.election.get().is_some()) .finish() } } diff --git a/tidal-net/src/transport.rs b/tidal-net/src/transport.rs index 7209c74..953cfcc 100644 --- a/tidal-net/src/transport.rs +++ b/tidal-net/src/transport.rs @@ -23,7 +23,7 @@ use tokio::sync::{Notify, mpsc}; use crate::{ client::PeerPool, config::GrpcTransportConfig, error::GrpcTransportError, server, - sources::ServingSources, + sources::{ElectionHooks, ServingSources}, }; /// Minimum spacing between catch-up pull attempts per source shard. The gap @@ -80,6 +80,10 @@ pub struct GrpcTransport { /// The catch-up pull machinery (single-flight + rate limit + the m11p4 /// timer retry). `Arc` because the retry tasks outlive `&self` borrows. catchup: Arc, + /// Late-bound election hooks (m11p4), shared with the gRPC service: the + /// transport consults them to stamp catch-up pulls with the current term + /// and to surface higher terms observed on ship acks. + election: Arc>>, server_handle: tokio::task::JoinHandle>, /// Shutdown signal for the receiver. The [`AtomicBool`] **latches** the request /// so a `recv_segment` that has not yet parked still observes it on entry (no @@ -149,6 +153,10 @@ struct CatchupRunner { /// transports, tests) falls back to the original seqno — correct either /// way, since the receiver gates idempotently; fresh is just cheaper. applied: Option>, + /// Election hooks (m11p4): pulls are stamped with the puller's current + /// term, and every received chunk's leadership claim is gated BEFORE the + /// inbound channel — a stale source can never inject history. + election: Arc>>, } impl CatchupRunner { @@ -213,7 +221,13 @@ impl CatchupRunner { /// Open the stream and drain it into the inbound channel. async fn run_pull(&self, from_shard: ShardId, from_seqno: u64) -> PullOutcome { - let mut stream = match self.pool.stream_from(from_shard, from_seqno).await { + // The puller's current term (m11p4): the source serves only a + // same-term pull, and every received chunk re-proves its claim. + let term = self + .election + .get() + .map_or(0, |hooks| hooks.self_claim().0); + let mut stream = match self.pool.stream_from(from_shard, from_seqno, term).await { Ok(stream) => stream, Err(e) => { tracing::warn!( @@ -232,6 +246,26 @@ impl CatchupRunner { match stream.message().await { Ok(Some(msg)) => match WalSegmentPayload::try_from(msg) { Ok(payload) => { + // Term fence on the PULL path (m11p4, design-review + // C14): a chunk whose leadership claim is stale — + // e.g. a deposed source racing its own step-down — + // aborts the pull before the inbound channel. + if let Some(hooks) = self.election.get() + && let Err(rejection) = hooks.observe_leader_claim( + payload.term, + payload.leader_region, + payload.id.seqno, + ) + { + tracing::warn!( + shard = from_shard.0, + chunk_term = payload.term, + ?rejection, + "catch-up chunk refused by the term gate; aborting pull \ + (the retry timer re-pulls once the term is joined)" + ); + return PullOutcome::Failed; + } chunks += 1; // Bounded send = natural backpressure: the // puller pauses while the receiver drains. @@ -458,6 +492,9 @@ impl GrpcTransport { // The catch-up retry timer reads this node's applied frontier through // the same source the server piggybacks on acks (m11p4). let applied_for_catchup = sources.applied.clone(); + // The election hooks cell is shared with the gRPC service (late-bound + // by the embedding application alongside the applied sink). + let election = Arc::clone(&sources.election); let (server_handle, pool) = runtime.block_on(async { let handle = server::start_server(&config, server_tx, sources, server_map)?; let pool = PeerPool::new(&config)?; @@ -473,6 +510,7 @@ impl GrpcTransport { states: Mutex::new(HashMap::new()), retry_interval: config.catchup_retry_interval, applied: applied_for_catchup, + election: Arc::clone(&election), }); Ok(Self { @@ -484,11 +522,26 @@ impl GrpcTransport { last_reported: Mutex::new(HashMap::new()), report_failing: Arc::new(Mutex::new(HashSet::new())), catchup, + election, server_handle, shutdown, }) } + /// A handle for the election driver's outbound RPCs (m11p4): vote and + /// heartbeat fan-outs plus the transfer `TimeoutNow`, all fired as async + /// tasks on this transport's runtime with per-call timeouts — callable + /// from any thread, never blocking the caller. Results arrive on the + /// caller's channel as [`ElectionNetEvent`]s. + #[must_use] + pub fn election_net(&self) -> ElectionNet { + ElectionNet { + pool: Arc::clone(&self.pool), + handle: self.runtime().handle().clone(), + shutdown: Arc::clone(&self.shutdown), + } + } + /// The embedded tokio runtime. /// /// # Infallible by construction @@ -594,11 +647,22 @@ impl Transport for GrpcTransport { }); } - let applied = self + let (applied, responder_term) = self .runtime() .block_on(self.pool.send_to(to, payload)) .map_err(TransportError::from)?; crate::server::fold_peer_applied(&self.peer_applied, to, applied); + // A follower answering from a higher term is this sender's step-down + // signal (m11p4) — a deposed leader learns its term is stale from its + // own outbound traffic even if it never hears a heartbeat. + if responder_term > 0 + && let Some(hooks) = self.election.get() + { + let (current, _) = hooks.self_claim(); + if responder_term > current { + hooks.on_observed_term(responder_term); + } + } Ok(()) } @@ -635,8 +699,15 @@ impl Transport for GrpcTransport { let pool = Arc::clone(&self.pool); let reporter = self.config.local_shard; let failing = Arc::clone(&self.report_failing); + // The reporter's current term (m11p4): the source folds this report + // into its quorum commit index ONLY when it matches the leadership + // the index was activated with. + let reporter_term = self.election.get().map_or(0, |hooks| hooks.self_claim().0); self.runtime().spawn(async move { - match pool.report_applied(source_shard, reporter, applied).await { + match pool + .report_applied(source_shard, reporter, applied, reporter_term) + .await + { Ok(()) => { let recovered = failing .lock() @@ -794,6 +865,147 @@ impl Drop for GrpcTransport { } } +/// One completed (or failed) election RPC, delivered to the driver's inbox. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ElectionNetEvent { + /// A peer answered our pre-vote/vote. + VoteReply { + from: ShardId, + prevote: bool, + term: u64, + granted: bool, + }, + /// The vote RPC failed (unreachable, timeout, or `Unimplemented` from a + /// pre-m11p4 peer) — counts as not-granted. + VoteUnreachable { from: ShardId, prevote: bool }, + /// A peer answered our heartbeat. + HeartbeatReply { + from: ShardId, + term: u64, + accepted: bool, + }, + /// The heartbeat RPC failed — no ack for the check-quorum lease. + HeartbeatUnreachable { from: ShardId }, + /// The transfer `TimeoutNow` was delivered (`accepted` = the target + /// started an election). + TimeoutNowDelivered { target: ShardId, accepted: bool }, + /// The transfer `TimeoutNow` could not be delivered. + TimeoutNowUnreachable { target: ShardId }, +} + +/// Per-RPC deadline for election traffic: well under the heartbeat interval +/// and the election timeout, so one stalled peer can never serialize a +/// fan-out round. +const ELECTION_RPC_TIMEOUT: Duration = Duration::from_secs(1); + +/// The election driver's outbound surface (m11p4). +/// +/// Fire-and-forget fan-outs on the transport's runtime, results delivered as +/// [`ElectionNetEvent`]s on the driver's channel. Callable from any thread; +/// never blocks. +/// +/// Deliberately bypasses the circuit breaker: an election must be able to +/// probe a peer the data-plane breaker quarantined, and the traffic is rare +/// and self-limiting. +pub struct ElectionNet { + pool: Arc, + handle: tokio::runtime::Handle, + shutdown: Arc, +} + +impl ElectionNet { + /// Fan `request` to every peer in `peers` as a pre-vote/vote round. + pub fn fan_votes( + &self, + peers: &[ShardId], + request: crate::proto::VoteRequest, + tx: &std::sync::mpsc::Sender, + ) { + for &peer in peers { + if self.shutdown.is_requested() { + return; + } + let pool = Arc::clone(&self.pool); + let tx = tx.clone(); + let prevote = request.prevote; + self.handle.spawn(async move { + let outcome = + tokio::time::timeout(ELECTION_RPC_TIMEOUT, pool.request_vote(peer, request)) + .await; + let event = match outcome { + Ok(Ok(reply)) => ElectionNetEvent::VoteReply { + from: peer, + prevote, + term: reply.term, + granted: reply.granted, + }, + Ok(Err(_)) | Err(_) => ElectionNetEvent::VoteUnreachable { + from: peer, + prevote, + }, + }; + let _ = tx.send(event); + }); + } + } + + /// Fan one heartbeat round to every peer. + pub fn fan_heartbeats( + &self, + peers: &[ShardId], + request: &crate::proto::HeartbeatRequest, + tx: &std::sync::mpsc::Sender, + ) { + for &peer in peers { + if self.shutdown.is_requested() { + return; + } + let pool = Arc::clone(&self.pool); + let req = request.clone(); + let tx = tx.clone(); + self.handle.spawn(async move { + let outcome = + tokio::time::timeout(ELECTION_RPC_TIMEOUT, pool.heartbeat(peer, req)).await; + let event = match outcome { + Ok(Ok(reply)) => ElectionNetEvent::HeartbeatReply { + from: peer, + term: reply.term, + accepted: reply.accepted, + }, + Ok(Err(_)) | Err(_) => ElectionNetEvent::HeartbeatUnreachable { from: peer }, + }; + let _ = tx.send(event); + }); + } + } + + /// Deliver a transfer `TimeoutNow` to `target`. + pub fn send_timeout_now( + &self, + target: ShardId, + request: crate::proto::TimeoutNowRequest, + tx: &std::sync::mpsc::Sender, + ) { + if self.shutdown.is_requested() { + return; + } + let pool = Arc::clone(&self.pool); + let tx = tx.clone(); + self.handle.spawn(async move { + let outcome = + tokio::time::timeout(ELECTION_RPC_TIMEOUT, pool.timeout_now(target, request)).await; + let event = match outcome { + Ok(Ok(reply)) => ElectionNetEvent::TimeoutNowDelivered { + target, + accepted: reply.accepted, + }, + Ok(Err(_)) | Err(_) => ElectionNetEvent::TimeoutNowUnreachable { target }, + }; + let _ = tx.send(event); + }); + } +} + /// Factory for building a set of [`GrpcTransport`] instances, one per shard. /// /// Analogous to `InProcessTransportFactory` but for gRPC connections. diff --git a/tidal-net/tests/catchup_retry.rs b/tidal-net/tests/catchup_retry.rs index d5dee04..6c4ef00 100644 --- a/tidal-net/tests/catchup_retry.rs +++ b/tidal-net/tests/catchup_retry.rs @@ -105,6 +105,7 @@ fn failed_pull_retries_on_timer_with_no_push() { opens: Arc::clone(&opens), })), applied_sink: Arc::default(), + election: Arc::default(), }; let _leader = GrpcTransport::new_with_sources( GrpcTransportConfig { @@ -157,6 +158,7 @@ fn clean_completion_does_not_keep_retrying() { opens: Arc::clone(&opens), })), applied_sink: Arc::default(), + election: Arc::default(), }; let _leader = GrpcTransport::new_with_sources( GrpcTransportConfig { diff --git a/tidal-net/tests/election_rpc.rs b/tidal-net/tests/election_rpc.rs new file mode 100644 index 0000000..a76eec2 --- /dev/null +++ b/tidal-net/tests/election_rpc.rs @@ -0,0 +1,419 @@ +// Integration-test exemptions (same posture as the tidaldb integration tests): +// unwrap/unwrap_err on known-good fixtures and short-lived read guards are +// idiomatic here. +#![allow(clippy::unwrap_used, clippy::significant_drop_tightening)] +//! m11p4 election RPC + term-fencing contract tests over real sockets. +//! +//! Proves the wire half of the election design: +//! 1. `RequestVote` round-trips through the late-bound [`ElectionHooks`] +//! (grant and refusal, with the voter's term in the reply). +//! 2. A node WITHOUT hooks answers `Unimplemented` — what a pre-m11p4 binary +//! looks like to a candidate (counts as not-granted, never an error). +//! 3. A stale-term ship is fenced with `FAILED_PRECONDITION` BEFORE the +//! inbound queue; a current-term ship passes and acks the responder term. +//! 4. The heartbeat exchange carries the responder's term + acceptance. +//! 5. `ElectionNet` fan-outs deliver events (reply or unreachable) for every +//! peer — the driver's inbox never starves on a dead peer. + +use std::{ + collections::HashMap, + net::SocketAddr, + sync::{Arc, Mutex, mpsc}, + thread, + time::Duration, +}; + +use tidal_net::{ + ClaimRejection, ElectionHooks, ElectionNetEvent, GrpcTransport, HeartbeatExchange, + config::GrpcTransportConfig, proto, sources::ServingSources, +}; +use tidaldb::replication::{ + VoteReply, VoteRpc, WalSegmentId, + shard::{RegionId, ShardId}, + transport::{Transport, TransportError, WalSegmentPayload}, +}; + +fn free_addr() -> SocketAddr { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.local_addr().unwrap() +} + +fn make_config( + shard: ShardId, + listen: SocketAddr, + peers: HashMap, +) -> GrpcTransportConfig { + GrpcTransportConfig { + local_shard: shard, + listen_addr: listen, + peers, + insecure: true, + ..Default::default() + } +} + +/// A scriptable hooks impl: a fixed current term + a recorded log of inbound +/// election traffic. +struct ScriptedHooks { + term: u64, + region: u16, + grant_votes: bool, + seen: Mutex>, +} + +impl ScriptedHooks { + const fn new(term: u64, region: u16, grant_votes: bool) -> Self { + Self { + term, + region, + grant_votes, + seen: Mutex::new(Vec::new()), + } + } +} + +impl ElectionHooks for ScriptedHooks { + fn self_claim(&self) -> (u64, u16) { + (self.term, self.region) + } + + fn observe_leader_claim( + &self, + term: u64, + leader_region: u16, + _first_seq: u64, + ) -> Result<(), ClaimRejection> { + self.seen + .lock() + .unwrap() + .push(format!("claim:{term}:{leader_region}")); + if term < self.term { + return Err(ClaimRejection::Stale { + current_term: self.term, + }); + } + Ok(()) + } + + fn on_heartbeat( + &self, + term: u64, + leader_region: u16, + baseline: u64, + _prev_log: tidaldb::replication::LogPosition, + ) -> HeartbeatExchange { + self.seen + .lock() + .unwrap() + .push(format!("hb:{term}:{leader_region}:{baseline}")); + HeartbeatExchange { + term: self.term, + accepted: term >= self.term, + } + } + + fn on_vote(&self, rpc: VoteRpc) -> VoteReply { + self.seen.lock().unwrap().push(format!( + "vote:{}:{}:{}:{}", + rpc.term, rpc.candidate.0, rpc.prevote, rpc.transfer + )); + VoteReply { + term: self.term, + granted: self.grant_votes && rpc.term > self.term, + } + } + + fn on_timeout_now(&self, term: u64, leader_region: u16) -> bool { + self.seen + .lock() + .unwrap() + .push(format!("timeoutnow:{term}:{leader_region}")); + true + } + + fn on_observed_term(&self, term: u64) { + self.seen.lock().unwrap().push(format!("observed:{term}")); + } + + fn report_term_acceptable(&self, reporter_term: u64) -> bool { + reporter_term == self.term + } +} + +/// Two transports; node 1 carries scripted hooks, node 0 carries none. +fn build_pair_with_hooks(hooks: Arc) -> (GrpcTransport, GrpcTransport) { + let addr0 = free_addr(); + let addr1 = free_addr(); + + let t0 = GrpcTransport::new(make_config( + ShardId(0), + addr0, + HashMap::from([(ShardId(1), addr1)]), + )) + .expect("transport 0"); + + let sources = ServingSources::default(); + sources.set_election_hooks(hooks as Arc); + let t1 = GrpcTransport::new_with_sources( + make_config(ShardId(1), addr1, HashMap::from([(ShardId(0), addr0)])), + sources, + ) + .expect("transport 1"); + + thread::sleep(Duration::from_millis(100)); + (t0, t1) +} + +const fn vote_request(term: u64, prevote: bool) -> proto::VoteRequest { + proto::VoteRequest { + term, + candidate_region: 0, + last_log_term: 0, + last_log_seq: 10, + prevote, + transfer: false, + } +} + +fn payload_with_term(term: u64, leader_region: u16, seqno: u64) -> WalSegmentPayload { + WalSegmentPayload { + id: WalSegmentId::new(RegionId::SINGLE, ShardId(0), seqno), + bytes: vec![0xAB; 64], + event_count: 1, + leader_last_seq: seqno, + stream_baseline: 0, + term, + leader_region, + } +} + +fn drain_events(rx: &mpsc::Receiver, expect: usize) -> Vec { + let mut events = Vec::new(); + while events.len() < expect { + match rx.recv_timeout(Duration::from_secs(5)) { + Ok(e) => events.push(e), + Err(e) => panic!("expected {expect} election events, got {events:?} ({e})"), + } + } + events +} + +#[test] +fn vote_roundtrip_grant_and_refusal_through_hooks() { + let hooks = Arc::new(ScriptedHooks::new(3, 1, true)); + let (t0, _t1) = build_pair_with_hooks(Arc::clone(&hooks)); + + let net = t0.election_net(); + let (tx, rx) = mpsc::channel(); + + // Term 5 > voter's 3 with grant_votes: granted, reply carries voter term. + net.fan_votes(&[ShardId(1)], vote_request(5, true), &tx); + let events = drain_events(&rx, 1); + assert_eq!( + events[0], + ElectionNetEvent::VoteReply { + from: ShardId(1), + prevote: true, + term: 3, + granted: true, + }, + ); + + // Term 2 < voter's 3: refused, the higher reply term is the step-down cue. + net.fan_votes(&[ShardId(1)], vote_request(2, false), &tx); + let events = drain_events(&rx, 1); + assert_eq!( + events[0], + ElectionNetEvent::VoteReply { + from: ShardId(1), + prevote: false, + term: 3, + granted: false, + }, + ); + + let seen = hooks.seen.lock().unwrap(); + assert!( + seen.contains(&"vote:5:0:true:false".to_string()), + "the pre-vote reached the hooks verbatim: {seen:?}" + ); +} + +#[test] +fn pre_m11p4_peer_without_hooks_counts_as_not_granted() { + // Node 0 (no hooks) is the "old binary"; node 1 campaigns against it. + let hooks = Arc::new(ScriptedHooks::new(0, 1, true)); + let (_t0, t1) = build_pair_with_hooks(Arc::clone(&hooks)); + + let net = t1.election_net(); + let (tx, rx) = mpsc::channel(); + net.fan_votes(&[ShardId(0)], vote_request(1, false), &tx); + let events = drain_events(&rx, 1); + assert_eq!( + events[0], + ElectionNetEvent::VoteUnreachable { + from: ShardId(0), + prevote: false, + }, + "Unimplemented from a pre-m11p4 peer is not-granted, never a crash" + ); +} + +#[test] +fn stale_term_ship_is_fenced_current_term_passes() { + let hooks = Arc::new(ScriptedHooks::new(4, 1, false)); + let (t0, t1) = build_pair_with_hooks(Arc::clone(&hooks)); + + // Stale term 2 < 4: fenced as a permanent failure (FAILED_PRECONDITION), + // never enqueued. + let err = t0 + .send_segment(ShardId(1), payload_with_term(2, 0, 1)) + .expect_err("a stale-term ship must be fenced"); + assert!( + matches!(err, TransportError::Permanent { ref reason } if reason.contains("stale")), + "got: {err:?}" + ); + + // Current term passes and the receiver sees it. + t0.send_segment(ShardId(1), payload_with_term(4, 0, 1)) + .expect("a current-term ship must pass the fence"); + let received = t1.recv_segment().expect("the fenced node received it"); + assert_eq!(received.term, 4); + assert_eq!(received.id.seqno, 1); + + let seen = hooks.seen.lock().unwrap(); + assert!( + seen.contains(&"claim:2:0".to_string()) && seen.contains(&"claim:4:0".to_string()), + "both claims reached the hooks: {seen:?}" + ); +} + +#[test] +fn heartbeat_exchange_carries_term_and_acceptance() { + let hooks = Arc::new(ScriptedHooks::new(7, 1, false)); + let (t0, _t1) = build_pair_with_hooks(Arc::clone(&hooks)); + + let net = t0.election_net(); + let (tx, rx) = mpsc::channel(); + + // A heartbeat from term 7 (current): accepted. + net.fan_heartbeats( + &[ShardId(1)], + &proto::HeartbeatRequest { + shard_id: 0, + region_id: 0, + term: 7, + leader_region: 0, + stream_baseline: 12, + ..Default::default() + }, + &tx, + ); + // A heartbeat from a deposed term 6: refused, reply carries 7. + net.fan_heartbeats( + &[ShardId(1)], + &proto::HeartbeatRequest { + shard_id: 0, + region_id: 0, + term: 6, + leader_region: 0, + stream_baseline: 0, + ..Default::default() + }, + &tx, + ); + let events = drain_events(&rx, 2); + assert!( + events.contains(&ElectionNetEvent::HeartbeatReply { + from: ShardId(1), + term: 7, + accepted: true, + }), + "current-term heartbeat accepted: {events:?}" + ); + assert!( + events.contains(&ElectionNetEvent::HeartbeatReply { + from: ShardId(1), + term: 7, + accepted: false, + }), + "stale-term heartbeat refused with the responder term: {events:?}" + ); + + let seen = hooks.seen.lock().unwrap(); + assert!( + seen.contains(&"hb:7:0:12".to_string()), + "the term's activation baseline rode the heartbeat: {seen:?}" + ); +} + +#[test] +fn timeout_now_reaches_hooks_and_unreachable_peer_reports() { + let hooks = Arc::new(ScriptedHooks::new(5, 1, false)); + let (t0, _t1) = build_pair_with_hooks(Arc::clone(&hooks)); + + let net = t0.election_net(); + let (tx, rx) = mpsc::channel(); + net.send_timeout_now( + ShardId(1), + proto::TimeoutNowRequest { + term: 5, + leader_region: 0, + }, + &tx, + ); + let events = drain_events(&rx, 1); + assert_eq!( + events[0], + ElectionNetEvent::TimeoutNowDelivered { + target: ShardId(1), + accepted: true, + }, + ); + + // An unknown peer reports unreachable instead of hanging the driver. + net.send_timeout_now( + ShardId(9), + proto::TimeoutNowRequest { + term: 5, + leader_region: 0, + }, + &tx, + ); + let events = drain_events(&rx, 1); + assert_eq!( + events[0], + ElectionNetEvent::TimeoutNowUnreachable { target: ShardId(9) }, + ); + + assert!( + hooks + .seen + .lock() + .unwrap() + .contains(&"timeoutnow:5:0".to_string()), + "the transfer request reached the target's hooks" + ); +} + +#[test] +fn stale_term_frontier_report_is_refused() { + let hooks = Arc::new(ScriptedHooks::new(3, 1, false)); + let (t0, t1) = build_pair_with_hooks(Arc::clone(&hooks)); + let _ = &t1; + + // notify_applied on the hook-less node 0 reports term 0 ≠ 3: the report + // must be REFUSED by node 1's handler (failed_precondition) — which on + // the reporting side is just a logged streak, never a crash. We can only + // observe the refusal indirectly: the hint fold never happens, so node + // 1's transport hint map for shard 0 stays empty. The structural assert + // is that nothing panicked and the claim never reached the sink-less + // hooks as accepted traffic. + t0.notify_applied(ShardId(1), 42); + thread::sleep(Duration::from_millis(300)); + assert_eq!( + t1.peer_applied_hint(ShardId(0)), + 0, + "a stale-term report must not fold into the responder's hint map" + ); +} diff --git a/tidal-net/tests/large_payload.rs b/tidal-net/tests/large_payload.rs index 0b91b4b..e84dfb3 100644 --- a/tidal-net/tests/large_payload.rs +++ b/tidal-net/tests/large_payload.rs @@ -77,6 +77,8 @@ fn ships_payload_larger_than_tonic_default_codec_limit() { event_count: 3, leader_last_seq: 7, stream_baseline: 0, + term: 0, + leader_region: 0, }; t0.send_segment(ShardId(1), payload) diff --git a/tidal-net/tests/mtls.rs b/tidal-net/tests/mtls.rs index de151bb..b347646 100644 --- a/tidal-net/tests/mtls.rs +++ b/tidal-net/tests/mtls.rs @@ -157,6 +157,8 @@ fn untrusted_client_cert_is_rejected() { event_count: 1, leader_last_seq: seq, stream_baseline: 0, + term: 0, + leader_region: 0, }; if client.send_segment(ShardId(1), payload).is_ok() { last_ok = true; @@ -212,6 +214,8 @@ fn absent_client_cert_is_rejected() { event_count: 1, leader_last_seq: seq, stream_baseline: 0, + term: 0, + leader_region: 0, }; if client.send_segment(ShardId(1), payload).is_ok() { last_ok = true; @@ -264,6 +268,8 @@ fn mtls_send_and_receive() { event_count: 2, leader_last_seq: 99, stream_baseline: 0, + term: 0, + leader_region: 0, }; t0.send_segment(ShardId(1), payload).unwrap(); diff --git a/tidal-net/tests/multi_node_uat.rs b/tidal-net/tests/multi_node_uat.rs index 8d6ecc1..d88390a 100644 --- a/tidal-net/tests/multi_node_uat.rs +++ b/tidal-net/tests/multi_node_uat.rs @@ -191,6 +191,8 @@ fn write_and_ship( event_count: 1, leader_last_seq: seqno, stream_baseline: 0, + term: 0, + leader_region: 0, }; node.transport .send_segment(target_shard, payload) @@ -269,6 +271,8 @@ fn uat_step2_idempotent_replay() { event_count: 1, leader_last_seq: 1, stream_baseline: 0, + term: 0, + leader_region: 0, }; leader.transport.send_segment(ShardId(1), payload).unwrap(); } @@ -433,6 +437,8 @@ fn uat_step5_three_node_replication() { event_count: 1, leader_last_seq: seq, stream_baseline: 0, + term: 0, + leader_region: 0, }; transports[0].send_segment(target, payload).unwrap(); } @@ -546,6 +552,8 @@ fn partition_heal_convergence() { event_count: 1, leader_last_seq: i, stream_baseline: 0, + term: 0, + leader_region: 0, }); } diff --git a/tidal-net/tests/reconnection.rs b/tidal-net/tests/reconnection.rs index a02010a..a3c6daf 100644 --- a/tidal-net/tests/reconnection.rs +++ b/tidal-net/tests/reconnection.rs @@ -46,6 +46,8 @@ fn make_payload(seqno: u64) -> WalSegmentPayload { event_count: 1, leader_last_seq: seqno, stream_baseline: 0, + term: 0, + leader_region: 0, } } diff --git a/tidal-net/tests/transport_contract.rs b/tidal-net/tests/transport_contract.rs index 454b9ab..d6e19f9 100644 --- a/tidal-net/tests/transport_contract.rs +++ b/tidal-net/tests/transport_contract.rs @@ -52,6 +52,8 @@ fn make_payload(shard: ShardId, seqno: u64) -> WalSegmentPayload { event_count: 5, leader_last_seq: seqno, stream_baseline: 0, + term: 0, + leader_region: 0, } } @@ -116,6 +118,8 @@ fn payload_too_large_rejected() { event_count: 0, leader_last_seq: 1, stream_baseline: 0, + term: 0, + leader_region: 0, }; let result = t0.send_segment(ShardId(1), payload); assert!(result.is_err()); diff --git a/tidal-server/src/cluster/election_driver.rs b/tidal-server/src/cluster/election_driver.rs new file mode 100644 index 0000000..e1609b8 --- /dev/null +++ b/tidal-server/src/cluster/election_driver.rs @@ -0,0 +1,613 @@ +//! The m11p4 election driver: the impure half of the leader-election design. +//! +//! The pure state machine lives in the engine +//! ([`tidaldb::replication::ElectionState`]); this module owns everything the +//! machine deliberately does not — timers, RPC fan-outs, durable persistence, +//! and the leadership transitions on [`RegionClusterState`]: +//! +//! * **One driver thread per node** ticks the machine every +//! [`TICK_INTERVAL`] and drains the [`ElectionNetEvent`] inbox (vote +//! replies, heartbeat acks) between ticks. +//! * **[`NodeElectionHooks`]** is the inbound surface, late-bound into the +//! gRPC transport like the applied sink: vote requests, heartbeats and +//! term-stamped data traffic step the machine synchronously, with hard +//! state persisted BEFORE any reply leaves. +//! * **Action execution order is the safety contract**: a +//! `PersistHardState` always completes (fsync'd) before any send the same +//! step produced; a persist failure drops the step's remaining actions — +//! no vote or claim escapes that durable state does not back +//! (phase-4.md §1). +//! * **Term-join + divergence** (phase-4.md §5): joining a term runs the +//! divergence check once — against the leader's announced activation +//! baseline (heartbeats) or the segment-shaped equivalent (a new-term +//! payload starting at or below this node's frontier). A diverged node +//! QUARANTINES: it refuses the data plane (ships, chunks, reports) but +//! keeps voting; its stale WAL-tail term makes it lose every up-to-date +//! comparison, so it can never win. + +use std::sync::{ + Arc, Mutex, OnceLock, Weak, + atomic::{AtomicBool, AtomicU64, Ordering}, + mpsc, +}; +use std::time::{Duration, Instant}; + +use tidal_net::{ClaimRejection, ElectionNet, ElectionNetEvent, HeartbeatExchange, proto}; +use tidaldb::replication::{ + ElectionAction, ElectionConfig, ElectionState, HardState, LogPosition, Role, VoteReply, + VoteRpc, + shard::{RegionId, ShardId}, +}; + +use super::{node::RegionClusterState, topology::shard_of_region}; + +/// Driver tick cadence: the machine's own deadlines (election timeout, +/// heartbeat due, lease) are all ≥ 100ms-scale, so 50ms keeps every deadline +/// honest to within a heartbeat-interval fraction at negligible cost. +const TICK_INTERVAL: Duration = Duration::from_millis(50); + +/// The shared election runtime: the machine, its durable store, the +/// quarantine latch, and the lock-free mirrors hot paths read. +pub struct ElectionRuntime { + machine: Mutex, + store: tidaldb::replication::ElectionStore, + /// Lock-free mirror of the machine's current term (hot-path reads: + /// payload gates, frontier reports, status). + term_cell: AtomicU64, + /// The highest term whose join check PASSED on this node (phase-4.md §5: + /// the check runs once per term). + joined_term: AtomicU64, + /// Divergent-suffix quarantine (design-review C7/C10 lineage): when set, + /// this node refuses all data-plane participation until reseeded. + quarantined: AtomicBool, + /// The node, for transitions and log-position reads. Weak: the runtime + /// is owned BY the node; a strong ref would leak both. + node: Weak, + net: ElectionNet, + /// The driver's inbox for async RPC outcomes. + inbox_tx: mpsc::Sender, + /// Peer shards for fan-outs (1:1 with the machine's peer regions). + peer_shards: Vec, + stop: AtomicBool, +} + +impl ElectionRuntime { + /// This node's current term (lock-free mirror). + pub fn current_term(&self) -> u64 { + self.term_cell.load(Ordering::Acquire) + } + + /// Whether this node is quarantined (divergent suffix). + pub fn is_quarantined(&self) -> bool { + self.quarantined.load(Ordering::Acquire) + } + + /// The highest ELECTED term this node has joined (0 = it has only ever + /// seen the topology era; a node's own failed candidacies do not count). + pub fn joined_term(&self) -> u64 { + self.joined_term.load(Ordering::Acquire) + } + + /// The machine's current role (for status). + pub fn role(&self) -> Role { + self.lock_machine().role() + } + + /// Signal the driver thread to exit on its next tick. + pub fn stop(&self) { + self.stop.store(true, Ordering::Release); + } + + /// Bridge for the legacy term-0 fan-out promote: keep the machine's view + /// in step with the externally applied verb (no-op at term ≥ 1). + pub fn force_term0_view(&self, leader: RegionId) { + self.lock_machine().force_term0_view(leader, Instant::now()); + } + + fn lock_machine(&self) -> std::sync::MutexGuard<'_, ElectionState> { + self.machine + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + /// This node's log position: `(wal_tail_term, durable_frontier)`, both + /// read from the node's WAL state (§1b — the same fsync stream). + fn my_log(&self) -> LogPosition { + self.node.upgrade().map_or( + LogPosition { + tail_term: 0, + frontier: 0, + }, + |node| node.election_log_position(), + ) + } + + /// Trigger an immediate transfer election on THIS node (the operator + /// promote path when no live leader can sanction the transfer, and the + /// target leg of a leader-sanctioned one). + pub fn campaign_now(&self) -> bool { + let now = Instant::now(); + let my_log = self.my_log(); + let actions = { + let mut machine = self.lock_machine(); + let term = machine.term(); + machine.on_timeout_now(term, now, my_log) + }; + let started = !actions.is_empty(); + self.execute(actions, now); + started + } + + /// Leader-side fenced transfer: emit the `TimeoutNow` at `target`. + pub fn transfer_to(&self, target: RegionId) -> bool { + let now = Instant::now(); + let actions = self.lock_machine().transfer_to(target); + let any = !actions.is_empty(); + self.execute(actions, now); + any + } + + /// The term-join divergence check (phase-4.md §5), run once per term — + /// and ONLY from a heartbeat, which carries the leader's ELECTION-TIME + /// log position (`prev_log`): this node is divergent iff its own + /// position exceeds the leader's lexicographically — it holds entries + /// the new leadership's history does not subsume. Both positions are in + /// the PREVIOUS stream's numbering (the vote restriction's comparison), + /// the only cross-node-comparable space; the new term's baseline is in + /// the NEW leader's numbering and is used solely for the frontier jump. + /// Returns `false` when the node is (or just became) quarantined. + fn join_term_check(&self, term: u64, prev_log: LogPosition) -> bool { + if term == 0 || term <= self.joined_term.load(Ordering::Acquire) { + return !self.is_quarantined(); + } + if self.is_quarantined() { + return false; + } + let Some(node) = self.node.upgrade() else { + return false; + }; + let position = node.election_log_position(); + // A tail already carrying THIS term's marker is manifestly part of + // this term's history — the restart-within-term rejoin (phase-4.md + // §5, third clause). Without this, a mid-term rejoiner whose tail + // term (T) exceeds the leader's election-time tail (T-1) would + // false-quarantine on every restart. + let rejoining_own_term = position.tail_term == term; + if !rejoining_own_term && position > prev_log { + self.quarantined.store(true, Ordering::Release); + node.note_quarantined( + term, + position.tail_term, + position.frontier, + Some(prev_log.frontier), + ); + return false; + } + self.joined_term.store(term, Ordering::Release); + true + } + + /// Execute a step's actions IN ORDER. A failed hard-state persist drops + /// every remaining action: no send may escape that durable state does + /// not back. + fn execute(&self, actions: Vec, now: Instant) { + for action in actions { + match action { + ElectionAction::PersistHardState { term, voted_for } => { + if let Err(e) = self.store.persist(HardState { + current_term: term, + voted_for, + }) { + tracing::error!( + term, + error = %e, + "election hard-state persist FAILED; dropping this step's \ + remaining actions (no vote/claim leaves without durable \ + backing) — fix the data dir" + ); + return; + } + self.term_cell.store(term, Ordering::Release); + } + ElectionAction::SendVotes(rpc) => { + tracing::info!( + term = rpc.term, + prevote = rpc.prevote, + transfer = rpc.transfer, + log_term = rpc.log.tail_term, + log_seq = rpc.log.frontier, + "election: vote round starting" + ); + if rpc.prevote + && let Some(node) = self.node.upgrade() + { + node.cluster_metrics().incr_elections_started(); + } + self.net.fan_votes( + &self.peer_shards, + proto::VoteRequest { + term: rpc.term, + candidate_region: u32::from(rpc.candidate.0), + last_log_term: rpc.log.tail_term, + last_log_seq: rpc.log.frontier, + prevote: rpc.prevote, + transfer: rpc.transfer, + }, + &self.inbox_tx, + ); + } + ElectionAction::BecomeLeader { term } => { + let Some(node) = self.node.upgrade() else { + return; + }; + if let Err(e) = node.become_leader_for_term(term) { + tracing::error!( + term, + error = %e, + "leadership activation failed (term marker / queue); \ + abandoning the won election" + ); + let follow_up = self.lock_machine().on_activation_failed(now); + // Re-entrancy is bounded: on_activation_failed only + // yields a BecomeFollower view change. + self.execute(follow_up, now); + } + } + ElectionAction::BecomeFollower { term, leader } => { + if let Some(node) = self.node.upgrade() { + node.step_down_view(term, leader); + node.cluster_metrics().incr_leader_changes(); + } + } + ElectionAction::SendHeartbeats { term } => { + tracing::trace!(term, "sending heartbeat round"); + let Some(node) = self.node.upgrade() else { + return; + }; + self.net.fan_heartbeats( + &self.peer_shards, + &node.election_heartbeat(term), + &self.inbox_tx, + ); + } + ElectionAction::SendTimeoutNow { target, term } => { + self.net.send_timeout_now( + shard_of_region(target), + proto::TimeoutNowRequest { + term, + leader_region: self + .node + .upgrade() + .map_or(0, |n| u32::from(n.self_region().0)), + }, + &self.inbox_tx, + ); + } + ElectionAction::ConflictingTermZeroLeader { other } => { + tracing::error!( + other = other.0, + "TWO term-0 leaders detected: the topology files disagree on the \ + leader (deployment error). Arbitrating by election — fix the \ + topology to stop this recurring" + ); + } + } + } + } + + /// Feed one async RPC outcome into the machine. + fn on_net_event(&self, event: ElectionNetEvent, now: Instant) { + let my_log = self.my_log(); + let actions = { + let mut machine = self.lock_machine(); + match event { + ElectionNetEvent::VoteReply { + from, + prevote, + term, + granted, + } => { + tracing::info!(from = from.0, prevote, term, granted, "election: vote reply"); + machine.on_vote_response( + RegionId(from.0), + VoteReply { term, granted }, + prevote, + now, + my_log, + ) + } + ElectionNetEvent::VoteUnreachable { from, prevote } => { + tracing::info!(from = from.0, prevote, "election: vote rpc unreachable"); + Vec::new() + } + ElectionNetEvent::HeartbeatUnreachable { .. } => Vec::new(), + ElectionNetEvent::HeartbeatReply { + from, + term, + accepted, + } => { + tracing::trace!(from = from.0, term, accepted, "heartbeat reply"); + machine.on_heartbeat_ack(RegionId(from.0), term, accepted, now) + } + ElectionNetEvent::TimeoutNowDelivered { target, accepted } => { + tracing::info!( + target = target.0, + accepted, + "leadership transfer TimeoutNow delivered" + ); + Vec::new() + } + ElectionNetEvent::TimeoutNowUnreachable { target } => { + tracing::warn!( + target = target.0, + "leadership transfer TimeoutNow undeliverable; the lease will \ + re-elect normally if this leader is actually gone" + ); + Vec::new() + } + } + }; + self.execute(actions, now); + } +} + +/// The inbound gRPC surface, late-bound into the transport's serving sources. +pub struct NodeElectionHooks { + runtime: Arc, +} + +impl tidal_net::ElectionHooks for NodeElectionHooks { + fn self_claim(&self) -> (u64, u16) { + let region = self + .runtime + .node + .upgrade() + .map_or(0, |n| n.self_region().0); + (self.runtime.current_term(), region) + } + + fn observe_leader_claim( + &self, + term: u64, + leader_region: u16, + first_seq: u64, + ) -> Result<(), ClaimRejection> { + let now = Instant::now(); + // Quarantine + stale-term fences first (no state change). + if self.runtime.is_quarantined() { + tracing::warn!(term, first_seq, "payload fenced: node is quarantined"); + return Err(ClaimRejection::Stale { + current_term: self.runtime.current_term(), + }); + } + { + let machine = self.runtime.lock_machine(); + // The topology era ends at the first JOINED election, not at this + // node's own failed candidacy: a term-0 payload from the topology + // leader stays acceptable while joined_term == 0 even if a + // can't-reach-anyone campaign inflated our durable term (the + // isolated-operator-override scenario). Elected leaderships + // (joined >= 1) fence absolutely. + let topology_era_claim = + term == 0 && self.runtime.joined_term.load(Ordering::Acquire) == 0; + if term < machine.term() && !topology_era_claim { + tracing::warn!( + term, + current = machine.term(), + first_seq, + "payload fenced: stale leadership term" + ); + return Err(ClaimRejection::Stale { + current_term: machine.term(), + }); + } + } + // A payload from a term this node has not JOINED yet defers (the + // divergence check needs the leader's election-time position, which + // only heartbeats carry — phase-4.md §5). Transient by design. + if term > self.runtime.joined_term.load(Ordering::Acquire) { + tracing::debug!(term, first_seq, "payload deferred: term not joined yet"); + return Err(ClaimRejection::JoinPending); + } + let actions = self + .runtime + .lock_machine() + .on_leader_contact(term, RegionId(leader_region), now); + self.runtime.execute(actions, now); + Ok(()) + } + + fn on_heartbeat( + &self, + term: u64, + leader_region: u16, + stream_baseline: u64, + prev_log: LogPosition, + ) -> HeartbeatExchange { + let now = Instant::now(); + let current = self.runtime.current_term(); + // Same topology-era allowance as the payload gate: a term-0 leader's + // lease must not be broken by a peer whose own failed candidacy + // inflated its term without any elected leadership existing. + let topology_era_claim = + term == 0 && self.runtime.joined_term.load(Ordering::Acquire) == 0; + if term < current && !topology_era_claim { + return HeartbeatExchange { + term: current, + accepted: false, + }; + } + // The join check runs even while quarantined: leadership KNOWLEDGE + // is control-plane (the node keeps voting and reporting status); + // only the data plane is fenced. + let before = self.runtime.joined_term.load(Ordering::Acquire); + let joined = self.runtime.join_term_check(term, prev_log); + if joined + && term > before + && let Some(node) = self.runtime.node.upgrade() + { + node.note_term_joined(term, RegionId(leader_region), stream_baseline); + } + let actions = self + .runtime + .lock_machine() + .on_leader_contact(term, RegionId(leader_region), now); + self.runtime.execute(actions, now); + HeartbeatExchange { + term: self.runtime.current_term(), + accepted: true, + } + } + + fn on_vote(&self, rpc: VoteRpc) -> VoteReply { + let now = Instant::now(); + let my_log = self.runtime.my_log(); + let (reply, actions) = self + .runtime + .lock_machine() + .on_vote_request(rpc, my_log, now); + // The persist (and any step-down) executes BEFORE the reply leaves — + // the hard-state fence. A failed persist drops the remaining actions; + // refuse the vote in that case rather than reply with an unbacked + // grant. + self.runtime.execute(actions, now); + if reply.granted && self.runtime.current_term() != reply.term { + // The persist that was supposed to back this grant did not land. + return VoteReply { + term: self.runtime.current_term(), + granted: false, + }; + } + reply + } + + fn on_timeout_now(&self, term: u64, leader_region: u16) -> bool { + let now = Instant::now(); + let my_log = self.runtime.my_log(); + tracing::info!( + term, + from = leader_region, + "TimeoutNow received: starting an immediate transfer election" + ); + let actions = { + let mut machine = self.runtime.lock_machine(); + machine.on_timeout_now(term, now, my_log) + }; + let started = !actions.is_empty(); + self.runtime.execute(actions, now); + started + } + + fn on_observed_term(&self, term: u64) { + let now = Instant::now(); + let actions = self.runtime.lock_machine().on_observed_term(term); + self.runtime.execute(actions, now); + } + + fn report_term_acceptable(&self, reporter_term: u64) -> bool { + // The reference is the COMMIT INDEX's activation term (the sink's + // own race-free gate uses the same value): the machine's term can + // run ahead of it through failed candidacies without any leadership + // change, and fencing on it would starve a legitimate quorum. + self.runtime + .node + .upgrade() + .is_some_and(|node| node.commit_active_term() == reporter_term) + } +} + +/// Build the runtime + hooks, late-bind the hooks into the transport, and +/// spawn the driver thread. Returns the runtime handle the node stores (for +/// status reads and shutdown). +pub fn start( + node: &Arc, + machine: ElectionState, + store: tidaldb::replication::ElectionStore, + hooks_cell: &Arc>>, +) -> Arc { + let (inbox_tx, inbox_rx) = mpsc::channel::(); + let peer_shards: Vec = machine + .config() + .peers + .iter() + .map(|&r| shard_of_region(r)) + .collect(); + let term = machine.term(); + // joined_term starts at ZERO on every boot: it tracks terms whose JOIN + // CHECK passed, and the durable term is not evidence of that — a failed + // candidacy inflates it with no elected leadership existing (initializing + // from it would both fence legitimate term-0 topology traffic forever + // and skip the divergence check after a restart). The first heartbeat + // re-joins: a restart-within-term rejoiner passes via the + // tail-term==term clause; ships before that heartbeat defer with the + // retryable JoinPending. + let joined = 0; + let runtime = Arc::new(ElectionRuntime { + machine: Mutex::new(machine), + store, + term_cell: AtomicU64::new(term), + joined_term: AtomicU64::new(joined), + quarantined: AtomicBool::new(false), + node: Arc::downgrade(node), + net: node.election_net(), + inbox_tx, + peer_shards, + stop: AtomicBool::new(false), + }); + + let _ = hooks_cell.set(Arc::new(NodeElectionHooks { + runtime: Arc::clone(&runtime), + }) as Arc); + + let driver = Arc::clone(&runtime); + std::thread::Builder::new() + .name("tidal-election".into()) + .spawn(move || { + tracing::info!("election driver started"); + while !driver.stop.load(Ordering::Acquire) { + // Drain async RPC outcomes between ticks; the timeout IS the + // tick cadence. + match inbox_rx.recv_timeout(TICK_INTERVAL) { + Ok(event) => driver.on_net_event(event, Instant::now()), + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => break, + } + let now = Instant::now(); + let my_log = driver.my_log(); + let actions = driver.lock_machine().tick(now, my_log); + driver.execute(actions, now); + if let Some(node) = driver.node.upgrade() { + let role = match driver.role() { + Role::Follower => 0, + Role::PreCandidate => 1, + Role::Candidate => 2, + Role::Leader => 3, + }; + node.cluster_metrics() + .set_election_view(driver.current_term(), role); + } + } + tracing::info!("election driver stopped"); + }) + .expect("spawn election driver thread"); + + runtime +} + +/// Build the engine's [`ElectionConfig`] from the topology's election block. +pub fn election_config( + spec: &super::topology::ElectionSpec, + self_region: RegionId, + peers: Vec, + auto_override: bool, +) -> ElectionConfig { + ElectionConfig { + self_region, + peers, + heartbeat_interval: spec.heartbeat_interval(), + election_timeout_min: spec.election_timeout_min(), + election_timeout_max: spec.election_timeout_max(), + leader_lease: spec.leader_lease(), + auto_election: spec.auto() && auto_override, + } +} diff --git a/tidal-server/src/cluster/mod.rs b/tidal-server/src/cluster/mod.rs index c4a5ae0..56b3386 100644 --- a/tidal-server/src/cluster/mod.rs +++ b/tidal-server/src/cluster/mod.rs @@ -33,6 +33,7 @@ //! [`SimulatedCluster`]: tidaldb::testing::SimulatedCluster //! [`TidalDb`]: tidaldb::TidalDb +pub(crate) mod election_driver; pub(crate) mod forward; pub(crate) mod node; pub(crate) mod routes; @@ -46,8 +47,8 @@ pub use node::{RegionClusterState, build_region_router}; pub use routes::build_cluster_router; pub use state::{ClusterMode, ClusterState, EXPERIMENTAL_CLUSTER_ENV, ensure_experimental_enabled}; pub use topology::{ - GrpcTlsSpec, RegionSpec, ReplicationSpec, TimeoutsSpec, TopologySpec, WalSpec, load_topology, - validate_multiproc, + ElectionSpec, GrpcTlsSpec, RegionSpec, ReplicationSpec, TimeoutsSpec, TopologySpec, WalSpec, + load_topology, validate_multiproc, }; // The OpenAPI documents (`crate::openapi`) reach the handlers/DTOs through diff --git a/tidal-server/src/cluster/node.rs b/tidal-server/src/cluster/node.rs index 50a91b6..9e48d8d 100644 --- a/tidal-server/src/cluster/node.rs +++ b/tidal-server/src/cluster/node.rs @@ -141,6 +141,14 @@ impl AckMode { /// double-apply it on followers. const STREAM_BASELINE_FILE: &str = "stream_baseline"; +/// How long a leader-sanctioned transfer waits for the target to hold the +/// full flushed prefix before `TimeoutNow` (the drain, m11p4). +const TRANSFER_CATCHUP_WAIT: Duration = Duration::from_secs(5); + +/// How long `/cluster/promote` waits for the transfer election to take +/// (target leads at a higher term) before reporting failure. +const TRANSFER_TAKEOVER_WAIT: Duration = Duration::from_secs(10); + /// Cap on one commit-watch bridge condvar wait: the longest the bridge /// thread can go without re-checking its stop flag, i.e. the worst-case /// shutdown latency the bridge adds. Deliberately a constant, not a @@ -148,6 +156,18 @@ const STREAM_BASELINE_FILE: &str = "stream_baseline"; /// (commit-index CHANGES wake the bridge immediately regardless). const COMMIT_BRIDGE_WAKE_INTERVAL: Duration = Duration::from_secs(1); +/// The election driver's boot bundle: prepared in [`RegionClusterState::new`] +/// (where the durable classification runs), consumed by +/// [`RegionClusterState::start_election_driver`] once the node is in its +/// final `Arc`. +struct ElectionBoot { + config: tidaldb::replication::ElectionConfig, + hard: tidaldb::replication::HardState, + boots_as_leader: bool, + store: tidaldb::replication::ElectionStore, + topology_leader: RegionId, +} + /// A multi-process cluster node owning exactly one region. pub struct RegionClusterState { /// This process's region id (index into the topology declaration order). @@ -175,11 +195,34 @@ pub struct RegionClusterState { /// The node's data dir (baseline persistence). Multi-process cluster /// mode requires one — validated in [`Self::new`]. data_dir: std::path::PathBuf, + /// The topology's term-0 leader (the stream owner for pre-election logs + /// in the vote restriction's frontier comparison). + boot_topology_leader: RegionId, + /// This node's ELECTION-TIME log position, captured at leadership + /// activation BEFORE the term marker bumps the tail term (m11p4): + /// announced in every heartbeat so a joining follower can run the + /// divergence check in a comparable numbering. ONE lock: the pair is a + /// single logical value (a torn term/seq read would mis-judge + /// divergence), it is written once per won election and read a few + /// times per second. Stored only AFTER the activation's term marker is + /// durable, so readers never see values for an aborted activation. + activation_prev: std::sync::Mutex, /// Shared `tidaldb_cluster_*` metrics cell (ship path, write pool, relay /// frontiers), rendered by this node's `/metrics` listener. cluster_metrics: Arc, - /// Current leadership view (which region this node believes is the leader). - leader: RwLock, + /// Current leadership view: which region this node believes leads, or + /// `None` during an election (leaderless windows are real and reported + /// honestly — forwards then return a retryable 503, m11p4). + leader: RwLock>, + /// The election hooks cell shared with the gRPC transport (late-bound by + /// [`Self::start_election_driver`], like the applied sink). + election_hooks_cell: + Arc>>, + /// The election runtime (m11p4), set by [`Self::start_election_driver`]. + election_runtime: std::sync::OnceLock>, + /// Everything the driver build needs, prepared at construction and taken + /// once by [`Self::start_election_driver`]. + election_boot: std::sync::Mutex>, /// Leader-side ship-skip set: peers we are partitioned from do not receive /// eager ships until healed. partitioned: RwLock>, @@ -310,6 +353,66 @@ impl RegionClusterState { .expect("validate_multiproc proved the leader is declared"); let my_shard = shard_of_region(region); + // m11p4 boot classification (phase-4.md §1): durable election state + // decides the boot role — NEVER the topology file alone. A clean + // rejoin always boots a follower (the §1.4-1 restart-amnesia fix); + // only a genuinely fresh node takes the topology's term-0 roles. + let election_store = tidaldb::replication::ElectionStore::new(&data_dir); + let wal_exists = data_dir + .join("wal") + .read_dir() + .map(|mut it| it.next().is_some()) + .unwrap_or(false); + let boot_state = election_store.load(wal_exists).map_err(|e| { + ServerError::Cluster(format!( + "refusing to boot: {e} (a node that cannot prove its term must not guess term 0)" + )) + })?; + let (hard_state, is_leader_at_boot) = match boot_state { + tidaldb::replication::BootState::Rejoin(h) => { + tracing::info!( + term = h.current_term, + "election state recovered; booting as a FOLLOWER (leadership is learned, never assumed from a restart)" + ); + (h, false) + } + tidaldb::replication::BootState::Fresh => { + let h = tidaldb::replication::HardState { + current_term: 0, + voted_for: None, + }; + election_store.persist(h).map_err(|e| { + ServerError::Cluster(format!("persist boot election state: {e}")) + })?; + (h, leader == region) + } + tidaldb::replication::BootState::StateFileLost => { + tracing::error!( + "election_state is MISSING but a WAL exists: the file was deleted out from under a node that has run before. Booting as a follower at term 0 (never re-assuming topology leadership on a guess); terms recover from the first leader contact" + ); + let h = tidaldb::replication::HardState { + current_term: 0, + voted_for: None, + }; + election_store.persist(h).map_err(|e| { + ServerError::Cluster(format!("persist recovered election state: {e}")) + })?; + (h, false) + } + }; + // The boot leadership view: a fresh term-0 node trusts the topology; + // a rejoining node trusts only contact. Critically, a rejoining + // TERM-0 EX-LEADER gets `None`, never `Some(self)` — a self-leading + // view would re-open the §1.4-1 write-acceptance hole with the ship + // queue parked (locally-durable, never-shipped writes). + let initial_view: Option = if is_leader_at_boot { + Some(region) + } else if hard_state.current_term == 0 && leader != region { + Some(leader) + } else { + None + }; + // Sibling region shards + their gRPC/HTTP addresses. let PeerTables { peer_shards, @@ -356,6 +459,7 @@ impl RegionClusterState { // sources (m11p2) hold WEAK db handles so the gRPC layer can never keep // the database alive past this node's shutdown. let sources = ServingSources { + election: Arc::default(), applied: Some(Arc::new(NodeAppliedSource { db: Arc::downgrade(&db), })), @@ -370,6 +474,7 @@ impl RegionClusterState { applied_sink: Arc::default(), }; let applied_sink_cell = Arc::clone(&sources.applied_sink); + let election_hooks_cell = Arc::clone(&sources.election); let listen_addr = resolve_grpc_addr(my_grpc_spec.as_deref(), region_name)?; let transport_defaults = GrpcTransportConfig::default(); let transport = GrpcTransport::new_with_sources( @@ -411,7 +516,6 @@ impl RegionClusterState { .map_err(ServerError::Tidal)?; let pool_config = topology.write_pool_config(); - let is_leader_at_boot = leader == region; tracing::info!( region = region_name, %listen_addr, @@ -431,7 +535,9 @@ impl RegionClusterState { Arc::new(WalFeedSource::new(my_shard, Arc::clone(&ship_feed))), Arc::clone(&transport) as Arc, &peer_shards, - topology.ship_queue_config(), + topology + .ship_queue_config() + .with_leader_region(region.0), is_leader_at_boot, Some(Arc::clone(&cluster_metrics)), ); @@ -497,6 +603,25 @@ impl RegionClusterState { .expect("spawn commit-watch bridge thread"); } + // Prepared for the election driver (started once the node is in its + // final Arc — see `start_election_driver`). + let election_boot = ElectionBoot { + config: super::election_driver::election_config( + &topology.election, + region, + name_to_id + .values() + .copied() + .filter(|&r| r != region) + .collect(), + true, + ), + hard: hard_state, + boots_as_leader: is_leader_at_boot, + store: election_store, + topology_leader: leader, + }; + Ok(Self { region, region_name: region_name.to_string(), @@ -506,8 +631,16 @@ impl RegionClusterState { ship_queue, stream_baseline, data_dir, + boot_topology_leader: leader, + activation_prev: std::sync::Mutex::new(tidaldb::replication::LogPosition { + tail_term: 0, + frontier: 0, + }), cluster_metrics, - leader: RwLock::new(leader), + leader: RwLock::new(initial_view), + election_hooks_cell, + election_runtime: std::sync::OnceLock::new(), + election_boot: std::sync::Mutex::new(Some(election_boot)), partitioned: RwLock::new(HashSet::new()), admin_op: std::sync::Mutex::new(()), name_to_id, @@ -543,6 +676,9 @@ impl RegionClusterState { /// signal the segment receiver to exit. Idempotent. pub fn shutdown(&mut self) { self.set_shutting_down(); + if let Some(rt) = self.election_runtime.get() { + rt.stop(); + } self.commit_bridge_stop.store(true, Ordering::Release); // Join the ship-queue senders FIRST so no batch ship races the // transport/db teardown below (their threads hold their own Arcs, but @@ -593,34 +729,46 @@ impl RegionClusterState { self.id_to_name.get(&id).map_or("unknown", String::as_str) } - /// This node's current leadership view. + /// This node's current leadership view (`None` during an election). #[must_use] - fn current_leader(&self) -> RegionId { + fn current_leader(&self) -> Option { *read_recovered(&self.leader, "leader") } /// True iff this node believes it leads. #[must_use] fn is_leader(&self) -> bool { - self.current_leader() == self.region + self.current_leader() == Some(self.region) } /// The HTTP address of the current leader, for a `NotLeader` body. fn leader_http(&self) -> Option { - let leader = self.current_leader(); - if leader == self.region { - None - } else { - self.peer_http.get(&leader).cloned() + match self.current_leader() { + Some(leader) if leader != self.region => self.peer_http.get(&leader).cloned(), + _ => None, } } - /// A typed `NotLeader` error naming the leader and its HTTP address. + /// This node's current election term (0 = the topology era / driver not + /// started). + #[must_use] + pub(crate) fn election_term(&self) -> u64 { + self.election_runtime + .get() + .map_or(0, |rt| rt.current_term()) + } + + /// A typed `NotLeader` error naming the leader (or the in-progress + /// election), its HTTP address, and this node's term so the forwarder + /// can tell a stale answer from a fresh one (m11p4). fn not_leader(&self) -> ServerError { - let leader = self.current_leader(); ServerError::NotLeader { - leader: self.region_name_of(leader).to_string(), + leader: self.current_leader().map_or_else( + || "none (election in progress)".to_string(), + |l| self.region_name_of(l).to_string(), + ), http_addr: self.leader_http(), + term: self.election_term(), } } @@ -656,20 +804,22 @@ impl RegionClusterState { out } - /// The current leader's HTTP base address, or `None` when THIS node leads or - /// the leader has no declared `http_addr`. + /// The current leader's HTTP base address, or `None` when THIS node leads, + /// no leader is known (election in progress), or the leader has no + /// declared `http_addr`. fn leader_http_addr(&self) -> Option { - let leader = self.current_leader(); - if leader == self.region { - None - } else { - self.peer_http.get(&leader).cloned() + match self.current_leader() { + Some(leader) if leader != self.region => self.peer_http.get(&leader).cloned(), + _ => None, } } /// The leader's region name (for forwarding/error bodies). fn leader_name(&self) -> String { - self.region_name_of(self.current_leader()).to_string() + self.current_leader().map_or_else( + || "none (election in progress)".to_string(), + |l| self.region_name_of(l).to_string(), + ) } /// Heal a peer (m11p2): clear the partition, resume the ship queue past @@ -710,11 +860,32 @@ impl RegionClusterState { // tail floor, leaving a gap the follower's pull closes). A failed // fetch degrades to a plain resume — the ack piggyback re-learns the // follower's applied on the first successful ship. - let reported_applied = self.fetch_remote_applied(id); - match reported_applied { - Some(applied) => self.ship_queue.resume_from(shard_of_region(id), applied), - None => self.ship_queue.resume(shard_of_region(id)), - } + let remote = self.fetch_remote_applied(id); + let my_term = self.election_term(); + let reported_applied = match remote { + // The fold into the acked frontier / commit index is term-gated + // (m11p4, design-review C13): a follower still on another term + // resumes dispatch but contributes no quorum mark until it joins + // this leadership (its next ReportApplied after joining does). + Some((applied, remote_term)) if remote_term == my_term => { + self.ship_queue.resume_from(shard_of_region(id), applied); + Some(applied) + } + Some((applied, remote_term)) => { + tracing::warn!( + healed = region_name, + remote_term, + my_term, + "heal: follower is on another term; resuming dispatch without folding its mark (it re-reports after joining this term)" + ); + self.ship_queue.resume(shard_of_region(id)); + Some(applied) + } + None => { + self.ship_queue.resume(shard_of_region(id)); + None + } + }; // Nudge the follower to pull catch-up NOW rather than waiting for the // next live ship to expose its gap (a quiet cluster would otherwise @@ -763,9 +934,10 @@ impl RegionClusterState { Ok(()) } - /// Blocking fetch of a peer's `/cluster/status/local` `applied_events`. + /// Blocking fetch of a peer's `/cluster/status/local`: its + /// `applied_events` and election `term` (0 for a pre-m11p4 peer). /// `None` on any transport/parse failure (caller falls back to full redeliver). - fn fetch_remote_applied(&self, peer: RegionId) -> Option { + fn fetch_remote_applied(&self, peer: RegionId) -> Option<(u64, u64)> { let http_addr = self.peer_http.get(&peer)?; let url = super::forward::peer_url(http_addr, "/cluster/status/local"); let resp = self @@ -778,8 +950,14 @@ impl RegionClusterState { return None; } let body: serde_json::Value = resp.json().ok()?; - body.get("applied_events") + let applied = body + .get("applied_events") + .and_then(serde_json::Value::as_u64)?; + let term = body + .get("term") .and_then(serde_json::Value::as_u64) + .unwrap_or(0); + Some((applied, term)) } // ── Write path ────────────────────────────────────────────────────────── @@ -966,14 +1144,39 @@ impl RegionClusterState { /// /// 400 on an unknown region. fn promote_local(&self, region_name: &str, baseline: Option) -> Result> { + // The legacy fan-out promote is a TOPOLOGY-ERA verb (m11p4): once + // this node has JOINED an elected leadership, leadership moves only + // through elections / fenced transfers — a topology-era fan-out must + // not depose an elected leader. The gate is the JOINED term, not the + // node's own durable term: a failed candidacy inflates the latter + // without any elected leadership existing (the isolated + // operator-override case). + let joined = self + .election_runtime + .get() + .map_or(0, |rt| rt.joined_term()); + if joined >= 1 { + return Err(ServerError::Cluster(format!( + "legacy promote is fenced: this node has joined an elected \ + leadership (term {joined}); use /cluster/promote without the \ + internal marker (the fenced transfer path)" + ))); + } let id = self.resolve_region(region_name)?; - *write_recovered(&self.leader, "leader") = id; + *write_recovered(&self.leader, "leader") = Some(id); + // Keep the election machine's view in step so its term-0 heartbeats + // do not fight the verb. + if let Some(rt) = self.election_runtime.get() { + rt.force_term0_view(id); + } if id == self.region { let baseline = self.ship_feed.flushed_seq(); persist_stream_baseline(&self.data_dir, baseline); self.stream_baseline.store(baseline, Ordering::Release); - self.ship_queue.activate_from(baseline); + // Legacy fan-out promote stays in the topology era (term 0); the + // election driver's transitions stamp real terms (m11p4). + self.ship_queue.activate_from(baseline, 0); tracing::info!( region = %self.region_name, baseline, @@ -1038,12 +1241,227 @@ impl RegionClusterState { .map_err(ServerError::Tidal) } + // ── Election integration (m11p4) ──────────────────────────────────────── + + /// This node's region id (for the election driver). + pub(crate) const fn self_region(&self) -> RegionId { + self.region + } + + /// The term the quorum commit index is currently activated with (m11p4): + /// the reference for the frontier-report gate. Equals the machine's term + /// whenever an ELECTED leadership activated the queue, and 0 for the + /// topology era / legacy activations — using the MACHINE term here + /// instead would drop legitimate reports whenever a node's own failed + /// candidacy inflated its durable term past its activation. + pub(crate) fn commit_active_term(&self) -> u64 { + self.commit.active_term() + } + + /// The shared cluster metrics cell (election gauges live here too). + pub(crate) const fn cluster_metrics( + &self, + ) -> &Arc { + &self.cluster_metrics + } + + /// The election runtime, when the driver has started. + pub(crate) fn election_runtime( + &self, + ) -> Option<&Arc> { + self.election_runtime.get() + } + + /// This node's log position for the vote restriction: + /// `(wal_tail_term, durable_frontier)`. + /// + /// The frontier is read in the LAST JOINED TERM'S STREAM numbering — the + /// only numbering comparable across nodes. A node that LED that term + /// reads its own flushed frontier (its WAL IS the stream); a follower + /// reads its durable applied frontier for that leader's shard. The + /// node's own WAL numbering would lie after a reseed (the baseline jump + /// makes local seqnos diverge from stream seqnos). Term-0 logs compare + /// in the topology leader's stream. + pub(crate) fn election_log_position(&self) -> tidaldb::replication::LogPosition { + let Some(db) = self.db.as_ref() else { + return tidaldb::replication::LogPosition { + tail_term: 0, + frontier: 0, + }; + }; + let (tail_term, _, marker_region) = db.wal_term_mark(); + let stream_region = if tail_term == 0 { + self.boot_topology_leader + } else { + RegionId(marker_region) + }; + let frontier = if stream_region == self.region { + self.ship_feed.flushed_seq() + } else { + db.replication_state() + .applied_seqno(shard_of_region(stream_region)) + .unwrap_or(0) + }; + tidaldb::replication::LogPosition { + tail_term, + frontier, + } + } + + /// The transport's election fan-out handle. + pub(crate) fn election_net(&self) -> tidal_net::ElectionNet { + self.transport.election_net() + } + + /// Build one leader heartbeat: the lease assertion carrying + /// `(term, leadership, the term's activation baseline)`. + pub(crate) fn election_heartbeat(&self, term: u64) -> tidal_net::proto::HeartbeatRequest { + let prev = *self + .activation_prev + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + tidal_net::proto::HeartbeatRequest { + shard_id: u32::from(shard_of_region(self.region).0), + region_id: u32::from(self.region.0), + term, + leader_region: u32::from(self.region.0), + stream_baseline: self.stream_baseline.load(Ordering::Acquire), + prev_log_term: prev.tail_term, + prev_log_seq: prev.frontier, + ..Default::default() + } + } + + /// The leadership activation sequence for an ELECTED term (phase-4.md + /// §3): baseline = own flushed frontier (persisted), queue activated + /// term-scoped, then the term marker journaled as the term's FIRST log + /// entry (it lands at baseline+1 and ships to every follower), and only + /// then the leader view flips so the write path opens. + /// + /// A marker failure aborts the leadership: a leader whose term marker is + /// not durable cannot prove its term. + pub(crate) fn become_leader_for_term(&self, term: u64) -> Result<()> { + // Capture the election-time position BEFORE the marker bumps the + // tail term: this is the heartbeat-announced reference a joining + // follower compares against for divergence (same numbering as the + // vote restriction). It is PUBLISHED only after the marker is + // durable (below), so an aborted activation never exposes it. + let prev = self.election_log_position(); + let baseline = self.ship_feed.flushed_seq(); + persist_stream_baseline(&self.data_dir, baseline); + self.stream_baseline.store(baseline, Ordering::Release); + self.ship_queue.activate_from(baseline, term); + let db = self.db()?; + if let Err(e) = db.append_term_marker(term, self.region.0) { + self.ship_queue.deactivate(); + return Err(ServerError::Tidal(e)); + } + *self + .activation_prev + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = prev; + *write_recovered(&self.leader, "leader") = Some(self.region); + tracing::info!( + term, + baseline, + region = %self.region_name, + "election won: term marker journaled, ship queue active, writes open" + ); + Ok(()) + } + + /// Step down (or learn a new leader): the VIEW flips first so no new + /// write is accepted, THEN the ship queue deactivates (failing in-flight + /// `ack=quorum` waits) — design-review C3's ordering. + pub(crate) fn step_down_view(&self, term: u64, leader: Option) { + *write_recovered(&self.leader, "leader") = leader; + self.ship_queue.deactivate(); + tracing::info!( + term, + leader = leader.map(|l| self.region_name_of(l).to_string()), + region = %self.region_name, + "leadership view updated (follower)" + ); + } + + /// A clean term-join (phase-4.md §5): jump this node's applied frontier + /// for the new leader's stream to the term's activation baseline — + /// seqnos at or below it are pre-stream history this node already holds. + pub(crate) fn note_term_joined(&self, term: u64, leader: RegionId, baseline: u64) { + if baseline > 0 + && let Ok(db) = self.db() + { + db.replication_state() + .advance(shard_of_region(leader), baseline); + } + tracing::info!( + term, + leader = %self.region_name_of(leader), + baseline, + "joined leadership term" + ); + } + + /// The divergence quarantine (phase-4.md §5): loud, actionable, metered. + pub(crate) fn note_quarantined( + &self, + term: u64, + tail_term: u64, + frontier: u64, + baseline: Option, + ) { + self.cluster_metrics.set_divergence_quarantined(true); + tracing::error!( + term, + tail_term, + frontier, + baseline, + region = %self.region_name, + "DIVERGENT SUFFIX: this node's WAL extends past the elected term's baseline with pre-term data (leader-acked writes the cluster elected past). QUARANTINED from the data plane — it still votes, but applies and reports nothing. Recovery: reseed this node from the leader (runbook §8); m11p5's snapshot transfer automates this" + ); + } + + /// Start the election driver (m11p4). Called once the node is in its + /// final `Arc` (the driver holds a `Weak` back-reference). + pub fn start_election_driver(self: &Arc) { + let Some(boot) = self + .election_boot + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + else { + return; + }; + // Seed the randomized election timeouts from wall-clock entropy + + // the region id, so simultaneous boots draw different timeouts. + let seed = u64::from(boot.config.self_region.0) + ^ std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0x5EED, |d| d.as_nanos() as u64); + let machine = tidaldb::replication::ElectionState::new( + boot.config, + boot.hard.current_term, + boot.hard.voted_for, + boot.boots_as_leader, + boot.topology_leader, + seed, + std::time::Instant::now(), + ); + let runtime = + super::election_driver::start(self, machine, boot.store, &self.election_hooks_cell); + let _ = self.election_runtime.set(runtime); + } + /// Build the `GET /cluster/status/local` body. fn local_status(&self) -> Result { let db = self.db()?; let leader = self.current_leader(); - let leader_shard = shard_of_region(leader); - let is_leader = leader == self.region; + let is_leader = leader == Some(self.region); + // Leaderless (mid-election): compute applied/lag against this node's + // own shard — a transient zero-lag self-view that the next heartbeat + // replaces. Honest fields below (`leader: null`, `role`) carry the + // real story. + let leader_shard = shard_of_region(leader.unwrap_or(self.region)); let last_seq = if is_leader { self.ship_feed.flushed_seq() @@ -1086,10 +1504,27 @@ impl RegionClusterState { 0 }; + let status_prev = *self + .activation_prev + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let (term, role, quarantined) = self.election_runtime.get().map_or_else( + || (0, "unknown".to_string(), false), + |rt| { + let role = match rt.role() { + tidaldb::replication::Role::Leader => "leader", + tidaldb::replication::Role::Follower => "follower", + tidaldb::replication::Role::PreCandidate => "pre-candidate", + tidaldb::replication::Role::Candidate => "candidate", + }; + (rt.current_term(), role.to_string(), rt.is_quarantined()) + }, + ); + Ok(LocalStatusResponse { region: self.region_name.clone(), is_leader, - leader: self.region_name_of(leader).to_string(), + leader: leader.map(|l| self.region_name_of(l).to_string()), last_seq, applied_events, lag_events, @@ -1097,6 +1532,11 @@ impl RegionClusterState { commit_index, ack: self.ack_default.as_str().to_string(), reachable: true, + term, + role, + quarantined, + prev_log_term: status_prev.tail_term, + prev_log_seq: status_prev.frontier, }) } } @@ -1570,7 +2010,7 @@ async fn region_health( "mode": "cluster", "process": "single-region", "region": state.region_name, - "leader": state.region_name_of(leader), + "leader": leader.map(|l| state.region_name_of(l).to_string()), })), )) } @@ -1584,8 +2024,8 @@ pub struct LocalStatusResponse { region: String, /// Whether this node currently believes it is the leader. is_leader: bool, - /// The region this node believes leads. - leader: String, + /// The region this node believes leads (`null` during an election). + leader: Option, /// The leader's relay seqno (only meaningful when `is_leader`). last_seq: u64, /// Replication events applied on this node from the current leader. @@ -1602,6 +2042,21 @@ pub struct LocalStatusResponse { ack: String, /// Always true (this node is serving its own status request). reachable: bool, + /// This node's current election term (m11p4; 0 = the topology era). + term: u64, + /// This node's election role: `leader`, `follower`, `pre-candidate`, + /// `candidate` (or `unknown` before the driver starts). + role: String, + /// While leading: this node's ELECTION-TIME log position + /// `(term, frontier)` in the previous stream's numbering — the value the + /// vote restriction compared, and the zero-acked-loss frontier proof's + /// reference (the leader's own `last_seq` is in its NEW stream's + /// numbering and is not comparable to pre-election seqs). + prev_log_term: u64, + prev_log_seq: u64, + /// Whether this node is quarantined with a divergent suffix (m11p4): + /// fenced from the data plane until reseeded. + quarantined: bool, } /// Local replication / leadership status for THIS region. @@ -1858,14 +2313,18 @@ pub struct RegionRequest { ), security(("bearerAuth" = [])), )] +// One linear protocol pass (marked leg -> fenced transfer -> takeover wait -> +// legacy fallback); splitting it would scatter the transfer's ordering rules. +#[allow(clippy::too_many_lines)] pub async fn cluster_promote( State(state): State>, headers: HeaderMap, Json(req): Json, ) -> std::result::Result, ClusterAppError> { if is_internal(&headers) { - // Marked fan-out leg: apply locally and terminate. The target leg - // returns its baseline to the fan-out initiator. + // Marked fan-out leg (the LEGACY term-0 protocol): apply locally and + // terminate. promote_local fences this once the cluster is + // term-governed (m11p4). let baseline = state .promote_local(&req.region, req.baseline) .map_err(ClusterAppError)?; @@ -1876,7 +2335,154 @@ pub async fn cluster_promote( }))); } - // External request: the TARGET's baseline must exist before the fan-out + // m11p4: promote is a FENCED TRANSFER — leadership moves through an + // election (term+1), never an un-fenced view flip. The legacy term-0 + // fan-out remains only as the mixed-version fallback below. + let target = state.resolve_region(&req.region).map_err(ClusterAppError)?; + // Topology-era re-assert: promoting the node that ALREADY leads, before + // any election has ever been joined, is the legacy verb's re-broadcast + // shape — the collapse step after a split-brain heal depends on the + // fan-out reaching every peer. Skip the transfer machinery and fall + // through to the legacy fan-out below. + let legacy_reassert = state.current_leader() == Some(target) + && state + .election_runtime() + .is_none_or(|rt| rt.joined_term() == 0); + if let Some(rt) = state.election_runtime().filter(|_| !legacy_reassert) { + let prior_term = rt.current_term(); + // Promoting the CURRENT leader once an elected leadership exists is + // a no-op success, whether this node IS that leader or merely knows + // it — the operator's "assert who leads" shape. + if state.current_leader() == Some(target) { + return Ok(Json(serde_json::json!({ + "ok": true, + "leader": req.region, + "term": prior_term, + "transfer": "already-leader", + }))); + } + + if state.is_leader() { + // Leader-sanctioned transfer: the catch-up wait IS the drain — + // the target must hold the full flushed prefix before TimeoutNow. + // The signal is the COMMIT INDEX's per-peer durable mark (fed by + // the target's own ReportApplied pushes): unlike the ship + // queue's acked frontier it cannot stall behind an open circuit + // breaker after the target's restart — the target's catch-up + // pull and frontier reports flow regardless of this leader's + // outbound breaker state. + let target_shard = shard_of_region(target); + let target_mark = |state: &RegionClusterState| { + state + .commit + .peer_marks() + .into_iter() + .find(|&(p, _)| p == target_shard) + .map_or(0, |(_, m)| m) + }; + let deadline = std::time::Instant::now() + TRANSFER_CATCHUP_WAIT; + loop { + let flushed = state.ship_feed.flushed_seq(); + if target_mark(&state) >= flushed { + break; + } + if std::time::Instant::now() >= deadline { + return Err(ClusterAppError(ServerError::Cluster(format!( + "transfer target '{}' lags the flushed frontier ({} < {}); \ + heal it first, then retry the promote", + req.region, + target_mark(&state), + state.ship_feed.flushed_seq() + )))); + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + rt.transfer_to(target); + } else if target == state.region { + // Target-side request. Prefer the leader-sanctioned path when a + // live leader is known; otherwise campaign directly (the dead- + // leader failover drill). + let sanctioned = if let Some(addr) = state.leader_http_addr() { + let url = peer_url(&addr, "/cluster/promote"); + let auth = forwarded_auth(&headers); + let body = serde_json::json!({ "region": req.region }); + forward_json(&state.client, &url, &body, auth.as_deref(), false) + .await + .map(|resp| resp.status.is_success()) + .unwrap_or(false) + } else { + false + }; + if !sanctioned { + rt.campaign_now(); + } + } else { + // Neither the leader nor the target: hand the request to the + // target (it sanctions through its leader or campaigns). + let Some(addr) = state.peer_http.get(&target).cloned() else { + return Err(ClusterAppError(ServerError::BadRequest(format!( + "region '{}' has no http_addr to promote", + req.region + )))); + }; + let url = peer_url(&addr, "/cluster/promote"); + let auth = forwarded_auth(&headers); + let body = serde_json::json!({ "region": req.region }); + if let Err(e) = + forward_json(&state.client, &url, &body, auth.as_deref(), false).await + { + return Err(ClusterAppError(ServerError::RegionUnreachable { + region: req.region, + cause: e, + })); + } + } + + // Wait for the transfer to take: the target leads. A genuine + // takeover bumps the term; a forwarded request that discovers the + // target ALREADY led resolves at the same term — both are success + // (hence >=, not >). + let deadline = std::time::Instant::now() + TRANSFER_TAKEOVER_WAIT; + while std::time::Instant::now() < deadline { + if state.current_leader() == Some(target) && state.election_term() >= prior_term { + return Ok(Json(serde_json::json!({ + "ok": true, + "leader": req.region, + "term": state.election_term(), + "transfer": "elected", + }))); + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + + // The election did not take. When the cluster has NEVER had an + // elected leadership (this node never JOINED a term ≥ 1 — its own + // failed candidacy may have inflated its durable term, but no leader + // exists at it), this is the topology era: the mixed-version and + // partitioned-operator-override signature — fall back to the legacy + // fan-out with a loud WARN. Once an elected leadership has been + // joined there is no safe fallback: report honestly. + if prior_term >= 1 || rt.joined_term() >= 1 { + return Err(ClusterAppError(ServerError::Cluster(format!( + "leadership transfer to '{}' did not complete within {:?} \ + (term {} -> {}); the cluster keeps its current leader — retry, \ + or check the target's health", + req.region, + TRANSFER_TAKEOVER_WAIT, + prior_term, + state.election_term() + )))); + } + tracing::warn!( + target = %req.region, + "election-based transfer did not take with no elected leadership \ + in sight (mixed-version cluster, or an isolated operator \ + override); falling back to the LEGACY fan-out promote" + ); + } + + // Legacy term-0 fan-out (pre-m11p4 protocol; also the mixed-version + // fallback): the TARGET's baseline must exist before the fan-out // (peers need it to jump their frontiers). Resolve it locally when this // node is the target; otherwise ask the target first via a marked promote. let target = state.resolve_region(&req.region).map_err(ClusterAppError)?; @@ -2386,8 +2992,15 @@ struct CommitIndexSink { } impl tidal_net::sources::AppliedSink for CommitIndexSink { - fn peer_applied(&self, peer: ShardId, applied: u64) { - self.commit.update_peer(peer, applied); + fn peer_applied(&self, peer: ShardId, applied: u64, reporter_term: u64) { + // m11p4 (design-review C4/C8/C12): fold ONLY a report stamped with + // the term this index was activated with — the check runs under the + // index's own lock, so a leadership change can never race the fold. + // Term-0 reports against a term-0 activation pass (the topology era + // and pre-m11p4 reporters). + let _ = self + .commit + .update_peer_for_term(peer, applied, reporter_term); } } diff --git a/tidal-server/src/cluster/routes.rs b/tidal-server/src/cluster/routes.rs index 21e0b81..01f1b14 100644 --- a/tidal-server/src/cluster/routes.rs +++ b/tidal-server/src/cluster/routes.rs @@ -882,10 +882,15 @@ impl IntoResponse for ClusterAppError { // body so a client can re-target the write/read itself (task 03 turns // these into transparent forwarding). Other errors keep the flat shape. let body = match &self.0 { - ServerError::NotLeader { leader, http_addr } => serde_json::json!({ + ServerError::NotLeader { + leader, + http_addr, + term, + } => serde_json::json!({ "error": self.0.to_string(), "leader": leader, "leader_http_addr": http_addr, + "term": term, }), ServerError::LeaderUnreachable { leader, diff --git a/tidal-server/src/cluster/topology.rs b/tidal-server/src/cluster/topology.rs index f3990e7..4c347c4 100644 --- a/tidal-server/src/cluster/topology.rs +++ b/tidal-server/src/cluster/topology.rs @@ -51,6 +51,73 @@ pub struct TopologySpec { /// `tidaldb_cluster_wal_fsync_us` on the deployment's volume. #[serde(default)] pub wal: WalSpec, + /// Optional election / failure-detector tuning (m11p4). Omitted = + /// defaults (300ms heartbeats, 1500–3000ms election timeout, 900ms + /// leader lease, auto-election on). + #[serde(default)] + pub election: ElectionSpec, +} + +/// Election / failure-detector tuning (the optional `election:` YAML block, +/// m11p4). +#[derive(Debug, Default, Deserialize)] +pub struct ElectionSpec { + /// Leader → follower heartbeat cadence in milliseconds (default 300). + #[serde(default)] + pub heartbeat_interval_ms: Option, + /// Election-timeout range floor in milliseconds (default 1500): a + /// follower that hears no valid leader for a randomized draw from + /// `[min, max)` starts a pre-vote. + #[serde(default)] + pub election_timeout_min_ms: Option, + /// Election-timeout range ceiling in milliseconds (default 3000). + #[serde(default)] + pub election_timeout_max_ms: Option, + /// Check-quorum window in milliseconds (default 900): a leader that + /// cannot reach a majority of peers within it steps down. Constrained: + /// `leader_lease_ms + heartbeat_interval_ms < election_timeout_min_ms` + /// — the deposed leader must stop before any successor can be elected. + #[serde(default)] + pub leader_lease_ms: Option, + /// Whether timeouts start elections automatically (default true). When + /// false the failure detector and term fencing still run; elections fire + /// only through `/cluster/promote` (the fenced transfer). After a + /// full-cluster restart in this mode the cluster is deliberately + /// leaderless until an operator promotes. + #[serde(default)] + pub auto_election: Option, +} + +impl ElectionSpec { + /// Resolved heartbeat interval. + #[must_use] + pub fn heartbeat_interval(&self) -> Duration { + Duration::from_millis(self.heartbeat_interval_ms.unwrap_or(300)) + } + + /// Resolved election-timeout floor. + #[must_use] + pub fn election_timeout_min(&self) -> Duration { + Duration::from_millis(self.election_timeout_min_ms.unwrap_or(1_500)) + } + + /// Resolved election-timeout ceiling. + #[must_use] + pub fn election_timeout_max(&self) -> Duration { + Duration::from_millis(self.election_timeout_max_ms.unwrap_or(3_000)) + } + + /// Resolved check-quorum lease. + #[must_use] + pub fn leader_lease(&self) -> Duration { + Duration::from_millis(self.leader_lease_ms.unwrap_or(900)) + } + + /// Resolved auto-election flag. + #[must_use] + pub fn auto(&self) -> bool { + self.auto_election.unwrap_or(true) + } } /// Replication ship-path tuning (the optional `replication:` YAML block). @@ -269,6 +336,42 @@ fn validate_spec_values(spec: &TopologySpec) -> Result<()> { "wal.batch_timeout_ms must be >= 1 (omit it for the 10ms default)".into(), )); } + // Election timing invariants (m11p4). The C2 lease bound is load-bearing + // for safety: the leader's check-quorum countdown starts at its last + // successful quorum contact while a follower's election timer starts at + // the last heartbeat it RECEIVED — the anchors differ by up to one + // heartbeat interval, so the deposed leader stops accepting writes + // before any successor can be elected only when + // `lease + heartbeat_interval < election_timeout_min`. + { + let hb = spec.election.heartbeat_interval(); + let min = spec.election.election_timeout_min(); + let max = spec.election.election_timeout_max(); + let lease = spec.election.leader_lease(); + if hb.is_zero() || min.is_zero() || lease.is_zero() { + return Err(ServerError::SchemaConfig( + "election timings must all be >= 1ms (omit them for the defaults)".into(), + )); + } + if max <= min { + return Err(ServerError::SchemaConfig(format!( + "election.election_timeout_max_ms ({}) must exceed \ + election_timeout_min_ms ({})", + max.as_millis(), + min.as_millis() + ))); + } + if lease + hb >= min { + return Err(ServerError::SchemaConfig(format!( + "election.leader_lease_ms ({}) + heartbeat_interval_ms ({}) must be \ + strictly below election_timeout_min_ms ({}): a deposed leader must stop \ + accepting writes before any successor can be elected", + lease.as_millis(), + hb.as_millis(), + min.as_millis() + ))); + } + } Ok(()) } @@ -310,6 +413,7 @@ impl TopologySpec { .replication .retry_ms .map_or(defaults.retry_backoff, Duration::from_millis), + leader_region: defaults.leader_region, } } @@ -440,6 +544,7 @@ mod tests { timeouts: TimeoutsSpec::default(), replication: ReplicationSpec::default(), wal: WalSpec::default(), + election: ElectionSpec::default(), } } diff --git a/tidal-server/src/error.rs b/tidal-server/src/error.rs index b09ae73..d692cd8 100644 --- a/tidal-server/src/error.rs +++ b/tidal-server/src/error.rs @@ -42,10 +42,13 @@ pub enum ServerError { /// Maps to 503 with a JSON body naming the leader (and its HTTP address when /// known) so the node is honest standalone; task 03 upgrades this to /// transparent leader forwarding. - #[error("not the leader; current leader is '{leader}'")] + #[error("not the leader; current leader is '{leader}' (term {term})")] NotLeader { leader: String, http_addr: Option, + /// The responder's election term (m11p4): lets a forwarder tell a + /// stale answer from a fresh one during election churn. + term: u64, }, /// A region-pinned read named a region this process does not own (multi- /// process cluster mode). Maps to 400 naming the region; task 03 upgrades diff --git a/tidal-server/src/main.rs b/tidal-server/src/main.rs index 2acfcd6..f26effb 100644 --- a/tidal-server/src/main.rs +++ b/tidal-server/src/main.rs @@ -270,6 +270,10 @@ async fn run_region_cluster(args: ClusterArgs, region: String) -> Result<()> { trait ServeState: Send + Sync + 'static { /// Human label for the post-serve log lines. const WHAT: &'static str; + /// Called once the state is in its final `Arc`, before serving begins + /// (the m11p4 election driver holds a `Weak` back-reference, so it can + /// only start here). Default: nothing. + fn started(self: &Arc) {} /// Flip `/health` to not-ready BEFORE axum starts draining. fn set_shutting_down(&self); /// Final deterministic shutdown (checkpoint + WAL fsync + thread join), @@ -306,6 +310,9 @@ impl ServeState for ClusterState { impl ServeState for RegionClusterState { const WHAT: &'static str = "region"; + fn started(self: &Arc) { + self.start_election_driver(); + } fn set_shutting_down(&self) { Self::set_shutting_down(self); // the inherent method, as above } @@ -335,6 +342,7 @@ async fn serve_state( tracing::info!("listening on http://{actual}"); let state = Arc::new(state); + state.started(); let shutdown_state = state.clone(); axum::serve(listener, build_router(state, api_key)) diff --git a/tidal-server/tests/cluster_election.rs b/tidal-server/tests/cluster_election.rs new file mode 100644 index 0000000..14861f8 --- /dev/null +++ b/tidal-server/tests/cluster_election.rs @@ -0,0 +1,530 @@ +//! m11p4 exit gates — automatic failover, fencing, bounded churn — over a +//! REAL 3-process cluster (tier 3, `cluster-e2e` feature). +//! +//! The three gates from docs/roadmap-to-cluster.md §4/m11p4: +//! +//! 1. **Auto-failover, zero acked loss** (`mp_auto_failover_*`): SIGKILL the +//! leader under `ack=quorum` load with ZERO operator verbs → a survivor is +//! elected and writes resume in <10s, and the m11p3 ledger invariants +//! (frontier + content) hold on the new leader, across repeated rounds +//! with pseudo-random kill points. +//! 2. **Fencing under partition + restart** (`mp_fenced_ex_leader_*`): the +//! leader is partitioned away (real TCP severs), the survivors elect, the +//! old leader RESTARTS while still partitioned — and cannot accept a +//! single write (its durable boot state forbids self-leadership, the +//! §1.4-1 fix), then rejoins as a follower on heal. +//! 3. **Bounded churn** (`mp_flapping_links_bounded_churn`): repeated +//! sever/heal cycles on the leader's links produce bounded elections (the +//! pre-vote absorbs flaps; terms never explode) and the cluster converges +//! to exactly one leader that serves quorum writes. +//! +//! Election timings are tuned fast (500–1000ms timeouts) so the suite stays +//! within the tier-3 budget; the production defaults scale the same +//! machinery up, not a different protocol. +#![cfg(feature = "cluster-e2e")] +#![allow(clippy::unwrap_used, clippy::significant_drop_tightening)] + +mod support; + +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; +use std::time::{Duration, Instant}; + +use support::{ + multiproc::{ClusterOptions, MultiProcCluster, convergence_budget}, + partition::{ProxyController, proxied_rewrite}, +}; + +/// All three roster regions, so every directed edge gets its own relay and a +/// node can be isolated in BOTH directions (inbound edges via +/// `region(r).sever_all()` cut what reaches it; outbound edges via +/// `edge(r, peer)` cut its own heartbeats/ships — without the outbound cut a +/// "partitioned" LEADER keeps resetting every follower's election timer). +const ALL_REGIONS: [&str; 3] = ["us-east", "eu-west", "ap-south"]; + +/// Fully isolate `region` from `peers` (both directions, gRPC + HTTP). +fn isolate(proxies: &ProxyController, region: &str, peers: &[&str]) { + proxies.region(region).sever_all(); + for peer in peers { + proxies.edge(region, peer).sever_all(); + } +} + +/// Undo [`isolate`]. +fn rejoin(proxies: &ProxyController, region: &str, peers: &[&str]) { + proxies.region(region).heal_all(); + for peer in peers { + proxies.edge(region, peer).heal_all(); + } +} + +/// The fast election block every test in this suite runs with. Constraint: +/// `lease (350) + heartbeat (100) < timeout_min (500)` — the C2 bound. +const FAST_ELECTION_YAML: &str = "election:\n heartbeat_interval_ms: 100\n election_timeout_min_ms: 500\n election_timeout_max_ms: 1000\n leader_lease_ms: 350"; + +/// The exit gate's failover budget: detect → elect → writes resume. +const FAILOVER_BUDGET: Duration = Duration::from_secs(10); + +/// A unique all-alpha search token for an entity id (mirrors the m11p3 +/// ledger checker's probe). +fn item_token(entity_id: u64) -> String { + let mut token = String::from("elx"); + for d in entity_id.to_string().bytes() { + token.push(char::from(b'a' + (d - b'0'))); + } + token +} + +/// POST with `x-tidal-ack` through a dedicated client. `Some(seq)` only for +/// a 2xx carrying `x-tidal-seq` — the ledger's definition of "acknowledged". +fn post_acked( + client: &reqwest::blocking::Client, + base: &str, + path: &str, + ack: &str, + body: &serde_json::Value, +) -> Option { + let resp = client + .post(format!("{base}{path}")) + .header("x-tidal-ack", ack) + .json(body) + .send() + .ok()?; + if !resp.status().is_success() { + return None; + } + resp.headers() + .get("x-tidal-seq")? + .to_str() + .ok()? + .parse() + .ok() +} + +/// Poll the LIVE nodes for an elected leader: a node whose local status +/// reports `role == "leader"` at a term above `after_term`. Returns +/// `(node_idx, term, elapsed)`. +fn await_elected_leader( + cluster: &MultiProcCluster, + candidates: &[usize], + after_term: u64, + budget: Duration, +) -> (usize, u64, Duration) { + let started = Instant::now(); + let deadline = started + budget; + loop { + for &idx in candidates { + if let Some(status) = cluster.local_status(idx) { + let term = status["term"].as_u64().unwrap_or(0); + if status["role"].as_str() == Some("leader") && term > after_term { + return (idx, term, started.elapsed()); + } + } + } + assert!( + Instant::now() < deadline, + "no leader elected among {candidates:?} within {budget:?} \ + (terms must move past {after_term})" + ); + std::thread::sleep(Duration::from_millis(50)); + } +} + +/// Write through `base` until one `ack=quorum` write succeeds (or the +/// deadline passes). Returns the first acked seq. +fn await_write_resumes( + client: &reqwest::blocking::Client, + base: &str, + entity_id: u64, + deadline: Instant, +) -> u64 { + loop { + if let Some(seq) = post_acked( + client, + base, + "/signals", + "quorum", + &serde_json::json!({ "entity_id": entity_id, "signal": "view", "weight": 1.0 }), + ) { + return seq; + } + assert!( + Instant::now() < deadline, + "quorum writes did not resume before the failover budget expired" + ); + std::thread::sleep(Duration::from_millis(50)); + } +} + +/// Gate 1: kill the leader under `ack=quorum` load — zero operator verbs — +/// across repeated rounds. Every round asserts: a survivor is elected and +/// writes resume within the 10s budget; the new leader's durable frontier +/// covers every acknowledged seq (INVARIANT A); every acknowledged item is +/// searchable on the new leader (INVARIANT B). The killed node restarts and +/// rejoins (reseeded if it quarantined with a divergent suffix — writes that +/// were leader-staged but never quorum-acked). +// One linear multi-round drill (load -> kill -> elect -> invariants -> +// rejoin); splitting it would scatter the round's ordering rules. +#[allow(clippy::too_many_lines)] +#[test] +fn mp_auto_failover_writes_resume_zero_acked_loss() { + let rounds: usize = std::env::var("TIDAL_ELECTION_KILLPOINTS") + .ok() + .and_then(|v| v.parse().ok()) + .filter(|&n| n > 0) + .unwrap_or(5); + let mut cluster = MultiProcCluster::start_with( + ClusterOptions::new(3).with_topology_extra(FAST_ELECTION_YAML), + ); + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(3)) + .build() + .unwrap(); + + let mut entity_cursor: u64 = 1; + let mut current_leader = 0usize; + let mut last_term = 0u64; + + for round in 0..rounds { + // ── Load: two writer threads against the current leader. ────────── + let stop = Arc::new(AtomicBool::new(false)); + let leader_base = cluster.node(current_leader); + let mut writers = Vec::new(); + for w in 0..2u64 { + let stop = Arc::clone(&stop); + let base = leader_base.clone(); + let first_entity = entity_cursor + w * 10_000; + writers.push(std::thread::spawn(move || { + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(3)) + .build() + .unwrap(); + let mut acked: Vec<(u64, u64)> = Vec::new(); + let mut entity = first_entity; + while !stop.load(Ordering::Acquire) { + let item_seq = post_acked( + &client, + &base, + "/items", + "quorum", + &serde_json::json!({ + "entity_id": entity, + "metadata": { "title": item_token(entity) }, + }), + ); + let view_seq = post_acked( + &client, + &base, + "/signals", + "quorum", + &serde_json::json!({ + "entity_id": entity, "signal": "view", "weight": 1.0 + }), + ); + if let Some(seq) = item_seq { + acked.push((entity, seq.max(view_seq.unwrap_or(0)))); + } + entity += 1; + } + acked + })); + } + + // Pseudo-random kill point per round (reproducible). + std::thread::sleep(Duration::from_millis(150 + (round as u64 * 97) % 400)); + cluster.kill_hard(current_leader); + stop.store(true, Ordering::Release); + let mut ledger: Vec<(u64, u64)> = Vec::new(); + for w in writers { + ledger.extend(w.join().expect("writer thread")); + } + let max_acked_seq = ledger.iter().map(|&(_, s)| s).max().unwrap_or(0); + + // ── ZERO operator verbs: the survivors elect on their own. ───────── + let survivors: Vec = (0..3).filter(|&i| i != current_leader).collect(); + let (new_leader, new_term, elapsed) = + await_elected_leader(&cluster, &survivors, last_term, FAILOVER_BUDGET); + // Writes must RESUME (not just leadership exist) inside the budget. + let resume_deadline = Instant::now() + FAILOVER_BUDGET; + let _ = await_write_resumes( + &client, + &cluster.node(new_leader), + 900_000 + round as u64, + resume_deadline, + ); + println!( + "round {round}: leader {current_leader} killed -> {new_leader} elected at \ + term {new_term} in {elapsed:?}; {} acked writes (max seq {max_acked_seq})", + ledger.len() + ); + + // ── INVARIANT A (frontier): the elected leader's ELECTION-TIME + // position — in the killed leader's stream numbering, the only + // numbering the acked seqs live in — covers every acknowledged + // write (the vote restriction guarantees it). The leader's own + // `last_seq` is its NEW stream's numbering and is NOT comparable. + let status = cluster.local_status(new_leader).expect("leader status"); + let prev_term = status["prev_log_term"].as_u64().unwrap(); + let prev_seq = status["prev_log_seq"].as_u64().unwrap(); + assert_eq!( + prev_term, last_term, + "round {round}: the elected leader's election-time tail term must be \ + the killed leader's term (same stream numbering as the acked seqs)" + ); + assert!( + prev_seq >= max_acked_seq, + "round {round}: elected leader's election-time frontier {prev_seq} is \ + below an acknowledged seq {max_acked_seq} — acked-write loss" + ); + + // ── INVARIANT B (content): every acked item is searchable on the + // new leader (the text index auto-commits within ~2s; allow 10). + let search_deadline = Instant::now() + Duration::from_secs(10); + for &(entity, _) in &ledger { + let token = item_token(entity); + loop { + let found: serde_json::Value = client + .get(format!( + "{}/search?query={token}&limit=5", + cluster.node(new_leader) + )) + .send() + .unwrap() + .json() + .unwrap(); + let hit = found["items"] + .as_array() + .is_some_and(|r| r.iter().any(|x| x["entity_id"].as_u64() == Some(entity))); + if hit { + break; + } + assert!( + Instant::now() < search_deadline, + "round {round}: acked item {entity} (token {token}) not found on \ + the elected leader — acked-write loss" + ); + std::thread::sleep(Duration::from_millis(200)); + } + } + + // ── Bring the killed node back for the next round. A divergent + // suffix (leader-staged, never quorum-acked writes) legitimately + // quarantines — the documented recovery is a reseed. + cluster.restart(current_leader, &[]); + let rejoin_deadline = Instant::now() + convergence_budget() + Duration::from_secs(10); + loop { + let status = cluster.local_status(current_leader); + let quarantined = status + .as_ref() + .and_then(|s| s["quarantined"].as_bool()) + .unwrap_or(false); + if quarantined { + println!( + "round {round}: restarted node {current_leader} quarantined \ + (divergent suffix) — reseeding, the documented recovery" + ); + cluster.kill_hard(current_leader); + cluster.wipe_data_dir(current_leader); + cluster.restart(current_leader, &[]); + } + let caught_up = cluster.local_status(current_leader).is_some_and(|s| { + s["term"].as_u64().unwrap_or(0) >= new_term + && s["role"].as_str() == Some("follower") + && s["lag_events"].as_u64() == Some(0) + }); + if caught_up { + break; + } + assert!( + Instant::now() < rejoin_deadline, + "round {round}: killed node {current_leader} did not rejoin/converge" + ); + std::thread::sleep(Duration::from_millis(200)); + } + + current_leader = new_leader; + last_term = new_term; + entity_cursor += 100_000; + } +} + +/// Gate 2: partition the leader away with real TCP severs, let the survivors +/// elect, RESTART the old leader while still partitioned — it must boot as a +/// follower (durable election state, never the topology file) and cannot +/// accept a single write; on heal it rejoins the new term as a follower. +#[test] +fn mp_fenced_ex_leader_restart_cannot_write() { + let (rewrite, proxies) = proxied_rewrite(&ALL_REGIONS); + let mut opts = ClusterOptions::new(3) + .with_topology_extra(FAST_ELECTION_YAML) + .with_rewrite(rewrite); + opts.log = "info".into(); + let mut cluster = MultiProcCluster::start_with(opts); + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .unwrap(); + + // Baseline data, fully converged BEFORE the partition so the old leader + // carries no divergent suffix (this gate is about fencing, not reseed). + for entity in 1..=5u64 { + post_acked( + &client, + &cluster.node(0), + "/signals", + "quorum", + &serde_json::json!({ "entity_id": entity, "signal": "view", "weight": 1.0 }), + ) + .expect("baseline quorum write"); + } + cluster.wait_converged_all(convergence_budget()); + + // ── Partition the leader away (BOTH directions); the survivors elect. ── + isolate(&proxies, "us-east", &["eu-west", "ap-south"]); + let (new_leader, new_term, elapsed) = + await_elected_leader(&cluster, &[1, 2], 0, FAILOVER_BUDGET); + println!("survivors elected node {new_leader} at term {new_term} in {elapsed:?}"); + // The new leadership serves quorum writes (2 of 3 replicas). + let _ = await_write_resumes( + &client, + &cluster.node(new_leader), + 700_001, + Instant::now() + FAILOVER_BUDGET, + ); + + // ── Restart the old leader while STILL partitioned. ──────────────────── + cluster.kill_hard(0); + cluster.restart(0, &[]); + + // The §1.4-1 assertion: across a multi-second window, the restarted + // ex-leader REFUSES every write — its durable state boots it as a + // follower and the topology file's `leader: us-east` is dead weight. + let fence_window = Instant::now() + Duration::from_secs(3); + let mut attempts = 0u32; + while Instant::now() < fence_window { + let accepted = post_acked( + &client, + &cluster.node(0), + "/signals", + "leader", + &serde_json::json!({ "entity_id": 700_100, "signal": "view", "weight": 1.0 }), + ); + assert!( + accepted.is_none(), + "the restarted, partitioned ex-leader ACCEPTED a write (seq {accepted:?}) — \ + the §1.4-1 split-brain hole is open" + ); + attempts += 1; + std::thread::sleep(Duration::from_millis(100)); + } + let status = cluster.local_status(0).expect("ex-leader serves status"); + assert_eq!( + status["is_leader"].as_bool(), + Some(false), + "restarted ex-leader must not claim leadership: {status}" + ); + println!("fencing held across {attempts} write attempts: {status}"); + + // ── Heal: the ex-leader joins the new term as a follower and converges. + rejoin(&proxies, "us-east", &["eu-west", "ap-south"]); + let rejoin_deadline = Instant::now() + convergence_budget() + Duration::from_secs(30); + loop { + let status = cluster.local_status(0).expect("status"); + assert_ne!( + status["quarantined"].as_bool(), + Some(true), + "a fully-converged-then-partitioned ex-leader must rejoin CLEAN \ + (no divergent suffix existed): {status}" + ); + if status["term"].as_u64().unwrap_or(0) >= new_term + && status["role"].as_str() == Some("follower") + && status["lag_events"].as_u64() == Some(0) + { + break; + } + assert!( + Instant::now() < rejoin_deadline, + "healed ex-leader did not rejoin term {new_term} and converge: {status}" + ); + std::thread::sleep(Duration::from_millis(200)); + } +} + +/// Gate 3: flapping links produce BOUNDED churn. Short flaps (below the +/// election timeout) are absorbed by the pre-vote/lease machinery; long +/// flaps elect; terms never explode; the cluster converges to exactly one +/// leader that serves quorum writes. +#[test] +fn mp_flapping_links_bounded_churn() { + let (rewrite, proxies) = proxied_rewrite(&ALL_REGIONS); + let mut cluster = MultiProcCluster::start_with( + ClusterOptions::new(3) + .with_topology_extra(FAST_ELECTION_YAML) + .with_rewrite(rewrite), + ); + let _ = &mut cluster; + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .unwrap(); + + // Short flaps: sever 250ms (below the 500ms timeout floor), heal 400ms. + // The lease refusals + timer resets must absorb these without elections. + for _ in 0..4 { + isolate(&proxies, "us-east", &["eu-west", "ap-south"]); + std::thread::sleep(Duration::from_millis(250)); + rejoin(&proxies, "us-east", &["eu-west", "ap-south"]); + std::thread::sleep(Duration::from_millis(400)); + } + // Long flaps: sever past the timeout so real elections happen, then heal. + for _ in 0..3 { + isolate(&proxies, "us-east", &["eu-west", "ap-south"]); + std::thread::sleep(Duration::from_millis(1_500)); + rejoin(&proxies, "us-east", &["eu-west", "ap-south"]); + std::thread::sleep(Duration::from_millis(800)); + } + + // Convergence: exactly one leader, agreed term, bounded churn. + let deadline = Instant::now() + Duration::from_secs(15); + let (leaders, term) = loop { + let statuses: Vec = + (0..3).filter_map(|i| cluster.local_status(i)).collect(); + let leaders: Vec = statuses + .iter() + .enumerate() + .filter(|(_, s)| s["role"].as_str() == Some("leader")) + .map(|(i, _)| i) + .collect(); + let terms: Vec = statuses + .iter() + .map(|s| s["term"].as_u64().unwrap_or(0)) + .collect(); + let agreed = terms.iter().max() == terms.iter().min(); + if statuses.len() == 3 && leaders.len() == 1 && agreed { + break (leaders, terms[0]); + } + assert!( + Instant::now() < deadline, + "cluster did not converge to one leader on one term: \ + leaders={leaders:?} terms={terms:?}" + ); + std::thread::sleep(Duration::from_millis(100)); + }; + // Bounded churn: 3 long flaps + slop can justify a handful of terms, + // never dozens (a term explosion = pre-vote regression / livelock). + assert!( + term <= 15, + "term {term} after 7 flaps — election churn is unbounded" + ); + println!("converged: leader node {} at term {term}", leaders[0]); + + // The survivor of all that chaos still serves quorum writes. + let _ = await_write_resumes( + &client, + &cluster.node(leaders[0]), + 800_001, + Instant::now() + FAILOVER_BUDGET, + ); +} diff --git a/tidal-server/tests/cluster_grpc.rs b/tidal-server/tests/cluster_grpc.rs index 6c55159..4b32b7e 100644 --- a/tidal-server/tests/cluster_grpc.rs +++ b/tidal-server/tests/cluster_grpc.rs @@ -13,7 +13,7 @@ use std::{ }; use tidal_server::cluster::{ - ClusterState, RegionSpec, ReplicationSpec, TimeoutsSpec, TopologySpec, WalSpec, + ClusterState, ElectionSpec, RegionSpec, ReplicationSpec, TimeoutsSpec, TopologySpec, WalSpec, build_cluster_router, }; use tidaldb::schema::{DecaySpec, EntityId, EntityKind, Schema, SchemaBuilder, Window}; @@ -37,6 +37,7 @@ fn three_region_topology() -> TopologySpec { timeouts: TimeoutsSpec::default(), replication: ReplicationSpec::default(), wal: WalSpec::default(), + election: ElectionSpec::default(), } } diff --git a/tidal-server/tests/cluster_lifecycle.rs b/tidal-server/tests/cluster_lifecycle.rs index 34f57d2..3b1bf10 100644 --- a/tidal-server/tests/cluster_lifecycle.rs +++ b/tidal-server/tests/cluster_lifecycle.rs @@ -181,6 +181,13 @@ mod support; +/// This suite choreographs the LEGACY operator drills (manual promote, +/// deliberate split-brain via a severed self-promote, rolling restarts). +/// Auto-election is pinned OFF so the m11p4 failure detector cannot race the +/// drills — the automatic path has its own exit-gate suite +/// (`cluster_election.rs`). +const LEGACY_ELECTION_YAML: &str = "election:\n auto_election: false"; + use std::{ sync::{ Arc, @@ -364,9 +371,13 @@ fn mp_clock_skew_reconciliation_stays_causal() { // Skewed-AHEAD leader, on-time follower, skewed-BEHIND follower. The env var // genuinely offsets each process's HLC (parsed i64 in main.rs → engine), so // this is real ±500ms skew, not a mock. - let (rewrite, proxies) = proxied_rewrite(&["ap-south"]); + // All three regions proxied so ap-south can be isolated in BOTH + // directions (m11p4): an inbound-only sever leaves its outbound votes, + // ships and heartbeats flowing, which turns the intended split-brain + // into a genuine election win. + let (rewrite, proxies) = proxied_rewrite(&["us-east", "eu-west", "ap-south"]); let cluster = MultiProcCluster::start_with( - ClusterOptions::new(3) + ClusterOptions::new(3).with_topology_extra(LEGACY_ELECTION_YAML) .with_env(LEADER, "TIDAL_HLC_SKEW_MS", "500") .with_env(EU_WEST, "TIDAL_HLC_SKEW_MS", "0") .with_env(AP_SOUTH, "TIDAL_HLC_SKEW_MS", "-500") @@ -415,7 +426,7 @@ fn mp_clock_skew_reconciliation_stays_causal() { // ── PHASE B: divergence under a real partition (split-brain hide) ────────── // Sever the skewed-BEHIND follower (ap-south, -500ms) from every peer. - proxies.region("ap-south").sever_all(); + support::partition::isolate_region(&proxies, "ap-south", &["us-east", "eu-west"]); println!("[skew] PHASE B: severed ap-south (-500ms) from all peers"); // AHEAD leader (+500ms, us-east) hides pair A. It is the real leader, so a plain @@ -486,7 +497,7 @@ fn mp_clock_skew_reconciliation_stays_causal() { ); // ── HEAL + collapse the split brain, then reconcile BOTH directions ──────── - proxies.region("ap-south").heal_all(); + support::partition::rejoin_region(&proxies, "ap-south", &["us-east", "eu-west"]); // Re-promote the real leader (us-east) to collapse the split brain — the runbook // step after a partitioned node rejoins. The durable Tag::HardNeg rows survive the // leadership change (store rows, not leader state), so the behind hide is intact. @@ -740,7 +751,7 @@ fn mp_rolling_upgrade_no_loss_no_stall() { // stays owned on THIS thread (restarts need &mut self); the writer thread gets // only the stable node base URLs + its own client, so the two never alias. let mut cluster = MultiProcCluster::start_with( - ClusterOptions::new(3) + ClusterOptions::new(3).with_topology_extra(LEGACY_ELECTION_YAML) .with_env(LEADER, "TIDAL_VERSION_TAG", "N") .with_env(EU_WEST, "TIDAL_VERSION_TAG", "N") .with_env(AP_SOUTH, "TIDAL_VERSION_TAG", "N"), diff --git a/tidal-server/tests/cluster_quorum.rs b/tidal-server/tests/cluster_quorum.rs index c06122e..0640899 100644 --- a/tidal-server/tests/cluster_quorum.rs +++ b/tidal-server/tests/cluster_quorum.rs @@ -40,6 +40,12 @@ mod support; +/// This suite validates the m11p3 quorum mechanics and the MANUAL failover +/// drill (operator promote of the max-applied survivor). Auto-election is +/// pinned OFF so the m11p4 failure detector cannot race the drill — the +/// automatic path has its own exit-gate suite (`cluster_election.rs`). +const LEGACY_ELECTION_YAML: &str = "election:\n auto_election: false"; + use std::sync::{ Arc, atomic::{AtomicBool, Ordering}, @@ -113,7 +119,7 @@ fn post_acked( #[test] fn mp_quorum_writes_gate_and_recover_under_partition() { let (rewrite, proxies) = proxied_rewrite(&["eu-west", "ap-south"]); - let cluster = MultiProcCluster::start_with(ClusterOptions::new(3).with_rewrite(rewrite)); + let cluster = MultiProcCluster::start_with(ClusterOptions::new(3).with_topology_extra(LEGACY_ELECTION_YAML).with_rewrite(rewrite)); let client = reqwest::blocking::Client::builder() .timeout(Duration::from_secs(8)) .build() @@ -249,7 +255,9 @@ fn mp_quorum_ledger_zero_acked_loss_across_killpoints() { println!("[ledger] running {rounds} leader-kill points (TIDAL_QUORUM_KILLPOINTS to widen)"); for round in 0..rounds { - let mut cluster = MultiProcCluster::start(3); + let mut opts = ClusterOptions::new(3).with_topology_extra(LEGACY_ELECTION_YAML); + opts.log = "info".into(); + let mut cluster = MultiProcCluster::start_with(opts); let leader_base = cluster.node(LEADER); let gateway_bases = [ cluster.node(LEADER), @@ -347,13 +355,36 @@ fn mp_quorum_ledger_zero_acked_loss_across_killpoints() { ledger.len() ); - let new_leader = cluster.region_name(chosen).to_string(); + // m11p4: promote is a FENCED transfer — the election's up-to-date + // restriction can refuse a target that fell behind between this + // test's status sample and the vote (in-flight ships keep applying + // for a moment after the kill). The operator drill is to promote the + // OTHER survivor in that case; the zero-acked-loss invariants hold + // for whichever node the election admits. + let mut new_leader = cluster.region_name(chosen).to_string(); let resp = cluster.post( chosen, "/cluster/promote", &serde_json::json!({ "region": new_leader }), ); - assert_eq!(resp.status().as_u16(), 200, "round {round}: promote"); + if resp.status().as_u16() != 200 { + let (other, _) = applied + .iter() + .copied() + .find(|&(idx, _)| idx != chosen) + .expect("two survivors"); + println!( + "[ledger] round {round}: promote of {new_leader} refused (it fell \ + behind the other survivor); promoting the other" + ); + new_leader = cluster.region_name(other).to_string(); + let retry = cluster.post( + other, + "/cluster/promote", + &serde_json::json!({ "region": new_leader }), + ); + assert_eq!(retry.status().as_u16(), 200, "round {round}: promote retry"); + } cluster.wait_leader_agreed(&new_leader, Duration::from_secs(10)); // ── INVARIANT B (content): every acked item is on the new leader ─── @@ -364,7 +395,10 @@ fn mp_quorum_ledger_zero_acked_loss_across_killpoints() { // The text index auto-commits every 2s (engine default), so the FIRST // probe polls past the commit interval; data presence is what is // asserted, not commit timing. - let new_leader_base = cluster.node(chosen); + let winner_idx = (0..3) + .find(|&i| cluster.region_name(i) == new_leader) + .expect("winner index"); + let new_leader_base = cluster.node(winner_idx); let search_deadline = Instant::now() + Duration::from_secs(10); for (entity_id, item_seq, _) in &ledger { let token = item_token(*entity_id); diff --git a/tidal-server/tests/cluster_region.rs b/tidal-server/tests/cluster_region.rs index 3d1f71e..1790623 100644 --- a/tidal-server/tests/cluster_region.rs +++ b/tidal-server/tests/cluster_region.rs @@ -24,7 +24,7 @@ use std::{ }; use tidal_server::cluster::{ - RegionClusterState, RegionSpec, ReplicationSpec, TimeoutsSpec, TopologySpec, WalSpec, + RegionClusterState, ElectionSpec, RegionSpec, ReplicationSpec, TimeoutsSpec, TopologySpec, WalSpec, build_region_router, }; use tidaldb::schema::{DecaySpec, EntityKind, Schema, SchemaBuilder, Window}; @@ -107,6 +107,7 @@ impl Pair { timeouts: TimeoutsSpec::default(), replication: ReplicationSpec::default(), wal: WalSpec::default(), + election: ElectionSpec::default(), } } } @@ -675,6 +676,7 @@ impl Trio { timeouts: TimeoutsSpec::default(), replication: ReplicationSpec::default(), wal: WalSpec::default(), + election: ElectionSpec::default(), } } } diff --git a/tidal-server/tests/cluster_routes.rs b/tidal-server/tests/cluster_routes.rs index bf1a138..b45f3e1 100644 --- a/tidal-server/tests/cluster_routes.rs +++ b/tidal-server/tests/cluster_routes.rs @@ -26,7 +26,7 @@ use std::{ }; use tidal_server::cluster::{ - RegionClusterState, RegionSpec, ReplicationSpec, TimeoutsSpec, TopologySpec, WalSpec, + RegionClusterState, ElectionSpec, RegionSpec, ReplicationSpec, TimeoutsSpec, TopologySpec, WalSpec, build_region_router, }; use tidaldb::schema::{DecaySpec, EntityKind, Schema, SchemaBuilder, Window}; @@ -93,6 +93,7 @@ impl Cluster3 { timeouts: TimeoutsSpec::default(), replication: ReplicationSpec::default(), wal: WalSpec::default(), + election: ElectionSpec::default(), } } diff --git a/tidal-server/tests/support/multiproc.rs b/tidal-server/tests/support/multiproc.rs index 749b8d2..cb1a4b4 100644 --- a/tidal-server/tests/support/multiproc.rs +++ b/tidal-server/tests/support/multiproc.rs @@ -130,6 +130,11 @@ pub struct ClusterOptions { pub rewrite: AddrRewrite, /// Per-node `TIDAL_SERVER_LOG` value (default `"warn"`). pub log: String, + /// Extra topology YAML appended verbatim to every per-process topology + /// file (default none). Used to tune the m11p4 `election:` block — e.g. + /// faster timeouts for the failover gates, or `auto_election: false` for + /// suites that exercise the legacy manual-promote protocol. + pub topology_extra: Option, } impl ClusterOptions { @@ -141,9 +146,18 @@ impl ClusterOptions { extra_env: HashMap::new(), rewrite: identity_rewrite(), log: "warn".into(), + topology_extra: None, } } + /// Append extra YAML (e.g. an `election:` block) to every topology file. + /// Builder-style; chainable. + #[must_use] + pub fn with_topology_extra(mut self, yaml: &str) -> Self { + self.topology_extra = Some(yaml.to_string()); + self + } + /// Add extra env for one region (by index). Builder-style; chainable. #[must_use] pub fn with_env(mut self, region_idx: usize, key: &str, value: &str) -> Self { @@ -302,7 +316,15 @@ impl MultiProcCluster { let schema_path = write_schema(tmp.path()); let topology_paths: Vec = (0..opts.regions) - .map(|i| write_topology_for(tmp.path(), &plans, i, &opts.rewrite)) + .map(|i| { + write_topology_for( + tmp.path(), + &plans, + i, + &opts.rewrite, + opts.topology_extra.as_deref(), + ) + }) .collect(); let mut harness = Self { @@ -489,6 +511,19 @@ impl MultiProcCluster { /// /// Panics if `idx` is out of range or the restarted node does not become /// healthy within [`boot_budget`]. + /// Wipe a DEAD node's data dir (the operator "reseed" drill): the next + /// `restart` boots it as a genuinely fresh node that recovers the full + /// log via the catch-up stream. Panics if the process is still alive. + pub fn wipe_data_dir(&self, idx: usize) { + assert!( + !self.is_alive(idx), + "wipe_data_dir requires the node to be stopped" + ); + let dir = &self.plans[idx].data_dir; + std::fs::remove_dir_all(dir).expect("wipe data dir"); + std::fs::create_dir_all(dir).expect("recreate data dir"); + } + pub fn restart(&mut self, idx: usize, env_overrides: &[(&str, &str)]) { assert!(idx < self.nodes.len(), "restart: node {idx} out of range"); // Ensure any prior process is fully gone (idempotent if already killed). @@ -812,6 +847,7 @@ fn write_topology_for( plans: &[RegionPlan], idx: usize, rewrite: &AddrRewrite, + extra: Option<&str>, ) -> PathBuf { let path = dir.join(format!("topology-{idx}.yaml")); let observer = &plans[idx].name; @@ -832,6 +868,9 @@ fn write_topology_for( } // Region 0 is the initial leader (consistent with cluster_routes.rs). let _ = writeln!(body, "leader: {}", plans[0].name); + if let Some(extra) = extra { + let _ = writeln!(body, "{extra}"); + } let mut f = std::fs::File::create(&path).expect("create topology file"); f.write_all(body.as_bytes()).expect("write topology file"); path diff --git a/tidal-server/tests/support/partition.rs b/tidal-server/tests/support/partition.rs index 02c36ab..d21d17b 100644 --- a/tidal-server/tests/support/partition.rs +++ b/tidal-server/tests/support/partition.rs @@ -448,6 +448,27 @@ pub fn proxied_rewrite(proxied_regions: &[&str]) -> (AddrRewrite, ProxyControlle (rewrite, controller) } +/// Fully isolate `region` from `peers`: BOTH directions, gRPC + HTTP. The +/// inbound edges come from [`ProxyController::region`]; the outbound ones are +/// `region`'s own view of each peer ([`ProxyController::edge`]) — without the +/// outbound cut a "partitioned" node keeps sending heartbeats/ships/votes, +/// which defeats partition scenarios entirely (m11p4). Requires every +/// involved region to have been proxied. +pub fn isolate_region(proxies: &ProxyController, region: &str, peers: &[&str]) { + proxies.region(region).sever_all(); + for peer in peers { + proxies.edge(region, peer).sever_all(); + } +} + +/// Undo [`isolate_region`]. +pub fn rejoin_region(proxies: &ProxyController, region: &str, peers: &[&str]) { + proxies.region(region).heal_all(); + for peer in peers { + proxies.edge(region, peer).heal_all(); + } +} + /// Spawn one detached unidirectional pump `from → to`. Exits when either end errors /// (EOF, reset, or a `sever()`-driven `shutdown`), shutting the write side so the /// partner pump unblocks. Detached: a closed socket always unblocks its blocking diff --git a/tidal/src/db/items.rs b/tidal/src/db/items.rs index be4bce0..0050e98 100644 --- a/tidal/src/db/items.rs +++ b/tidal/src/db/items.rs @@ -162,6 +162,9 @@ impl TidalDb { /// mid-batch halt re-applies safely). On a durability error every append /// staged by this call has still been WAITED — no staged blob is left /// unresolved behind the halt; the first error is returned. + // One linear three-phase pass (validate -> journal -> apply); splitting it + // would scatter the WAL-first ordering invariants across helpers. + #[allow(clippy::too_many_lines)] pub(crate) fn apply_replicated_blobs(&self, records: Vec) -> crate::Result<()> { /// One validated record, parsed exactly once in Phase 1 and applied /// in Phase 3 (item metadata is deserialized here, never re-parsed). @@ -174,6 +177,14 @@ impl TidalDb { id: EntityId, values: &'a [f32], }, + /// A replicated term marker (m11p4): journaled like any blob in + /// Phase 2 (so THIS node's WAL-tail term advances durably), then + /// folded into the term-mark cell in Phase 3 — no storage effect. + TermMark { + term: u64, + leader_region: u16, + seq_slot: usize, + }, } self.require_writeable("apply_replicated_blobs")?; @@ -183,7 +194,7 @@ impl TidalDb { // Phase 1 — validate ALL records before journaling ANY, capturing // each record's parsed form for the Phase 3 upserts. let mut applies: Vec> = Vec::with_capacity(records.len()); - for record in &records { + for (slot, record) in records.iter().enumerate() { match &**record { BlobRecord::ItemMetadata(r) => { let id = EntityId::new(r.entity_id); @@ -198,6 +209,19 @@ impl TidalDb { values: &r.values, }); } + BlobRecord::TermMarker(r) => { + if r.term == 0 { + return Err(TidalError::invalid_input( + "replicated term marker carries term 0 (the topology era never \ + journals one)", + )); + } + applies.push(BlobApply::TermMark { + term: r.term, + leader_region: r.leader_region, + seq_slot: slot, + }); + } } } @@ -205,6 +229,11 @@ impl TidalDb { // fsync (the writer coalesces the staged blobs into shared group // syncs). Skipped outside cluster mode / without a WAL, exactly like // `wal_blob_first`. + // + // `staged_seqs[i]` records the local WAL seqno record `i` landed at + // (m11p4: a replicated term marker's Phase 3 fold wants its own-log + // seqno for the term-mark cell). + let mut staged_seqs: Vec = vec![0; records.len()]; if self.replicate_blobs { let sender = { let wal = self @@ -234,9 +263,10 @@ impl TidalDb { } } } - for p in pending { + for (slot, p) in pending.into_iter().enumerate() { match p.wait() { Ok(seq) => { + staged_seqs[slot] = seq; super::wal_bridge::bump_last_seq_atomic(&self.last_wal_seq, seq); } Err(e) => { @@ -264,6 +294,19 @@ impl TidalDb { BlobApply::Embedding { id, values } => { self.apply_item_embedding_local(id, values)?; } + BlobApply::TermMark { + term, + leader_region, + seq_slot, + } => { + self.wal_term_mark + .advance(term, staged_seqs[seq_slot], leader_region); + tracing::info!( + term, + seq = staged_seqs[seq_slot], + "replicated term marker applied; WAL-tail term advanced" + ); + } } } Ok(()) @@ -666,6 +709,13 @@ impl TidalDb { BlobRecord::Embedding(record) => { self.apply_item_embedding_local(EntityId::new(record.entity_id), &record.values) } + BlobRecord::TermMarker(record) => { + // No storage effect: recovery re-derives the WAL-tail term + // (m11p4) from the last marker in the surviving log. + self.wal_term_mark + .advance(record.term, blob.seq, record.leader_region); + Ok(()) + } }; if let Err(e) = result { tracing::warn!( diff --git a/tidal/src/db/metrics/cluster.rs b/tidal/src/db/metrics/cluster.rs index 17fef0b..bdc5781 100644 --- a/tidal/src/db/metrics/cluster.rs +++ b/tidal/src/db/metrics/cluster.rs @@ -85,6 +85,18 @@ pub struct ClusterMetrics { /// Total `ack=quorum` writes that timed out awaiting the commit index /// (each returned a retryable 503 naming the laggards). quorum_timeouts_total: AtomicU64, + /// This node's current election term (m11p4; 0 = the topology era). + election_term: AtomicU64, + /// This node's election role (m11p4): 0 follower, 1 pre-candidate, + /// 2 candidate, 3 leader. + election_role: AtomicU64, + /// Total elections this node has STARTED (pre-vote rounds; m11p4). + elections_started_total: AtomicU64, + /// Total leadership changes this node has observed (m11p4). + leader_changes_total: AtomicU64, + /// Divergent-suffix quarantine latch (m11p4): 1 = this node is fenced + /// from the data plane until reseeded. + divergence_quarantined: AtomicU64, } impl ClusterMetrics { @@ -101,9 +113,37 @@ impl ClusterMetrics { relay_last_seq: AtomicU64::new(0), relay_durable_seq: AtomicU64::new(0), quorum_timeouts_total: AtomicU64::new(0), + election_term: AtomicU64::new(0), + election_role: AtomicU64::new(0), + elections_started_total: AtomicU64::new(0), + leader_changes_total: AtomicU64::new(0), + divergence_quarantined: AtomicU64::new(0), } } + /// Record this node's election term + role (m11p4): role is 0 follower, + /// 1 pre-candidate, 2 candidate, 3 leader. + pub fn set_election_view(&self, term: u64, role: u64) { + self.election_term.store(term, Ordering::Relaxed); + self.election_role.store(role, Ordering::Relaxed); + } + + /// Count one started election (a pre-vote round; m11p4). + pub fn incr_elections_started(&self) { + self.elections_started_total.fetch_add(1, Ordering::Relaxed); + } + + /// Count one observed leadership change (m11p4). + pub fn incr_leader_changes(&self) { + self.leader_changes_total.fetch_add(1, Ordering::Relaxed); + } + + /// Latch (or clear) the divergent-suffix quarantine (m11p4). + pub fn set_divergence_quarantined(&self, quarantined: bool) { + self.divergence_quarantined + .store(u64::from(quarantined), Ordering::Relaxed); + } + /// Enable rendering of the cluster series. Idempotent; called when a /// cluster surface first takes the metrics handle. pub fn mark_active(&self) { @@ -203,7 +243,9 @@ impl ClusterMetrics { } /// Append the `tidaldb_cluster_*` series to a Prometheus exposition body. + // One linear emit per series; splitting it would scatter the series list. #[allow(clippy::cast_precision_loss)] // monitoring gauges + #[allow(clippy::too_many_lines)] pub(crate) fn render_into(&self, out: &mut String, partition_id: u64) { use std::fmt::Write; @@ -260,6 +302,42 @@ impl ClusterMetrics { self.quorum_timeouts_total.load(Ordering::Relaxed) as f64, ); + super::write_metric_line( + out, + "tidaldb_cluster_election_term", + "This node's current election term (m11p4; 0 = topology era)", + "gauge", + self.election_term.load(Ordering::Relaxed) as f64, + ); + super::write_metric_line( + out, + "tidaldb_cluster_election_role", + "Election role: 0 follower, 1 pre-candidate, 2 candidate, 3 leader", + "gauge", + self.election_role.load(Ordering::Relaxed) as f64, + ); + super::write_metric_line( + out, + "tidaldb_cluster_elections_started_total", + "Elections (pre-vote rounds) this node has started", + "counter", + self.elections_started_total.load(Ordering::Relaxed) as f64, + ); + super::write_metric_line( + out, + "tidaldb_cluster_leader_changes_total", + "Leadership changes this node has observed", + "counter", + self.leader_changes_total.load(Ordering::Relaxed) as f64, + ); + super::write_metric_line( + out, + "tidaldb_cluster_divergence_quarantined", + "Divergent-suffix quarantine latch (1 = fenced from the data plane)", + "gauge", + self.divergence_quarantined.load(Ordering::Relaxed) as f64, + ); + // Per-peer series, labeled by peer shard id + this node's partition. // Snapshot the cells under the read lock, then render lock-free (the // ship sender threads update these cells on their hot path). diff --git a/tidal/src/db/mod.rs b/tidal/src/db/mod.rs index 730c666..0245d7e 100644 --- a/tidal/src/db/mod.rs +++ b/tidal/src/db/mod.rs @@ -113,6 +113,11 @@ pub struct TidalDb { /// records (cluster mode — `peer_shards` non-empty). Standalone nodes /// keep fjall-only item durability and pay zero extra fsyncs. replicate_blobs: bool, + /// m11p4: the WAL-tail term — the highest kind-3 term marker this node's + /// log carries. Fed by recovery, leader marker appends, and replicated + /// marker applies; read by the election machinery (the `lastLogTerm` + /// half of the vote restriction). `(0, 0)` outside cluster mode. + wal_term_mark: Arc, /// m11p2: the flushed-batch ship feed the WAL writer populates (cluster /// mode + persistent only). The replication ship queue's hot-path source. ship_feed: Option>, @@ -437,6 +442,7 @@ impl TidalDb { wal: std::sync::Mutex::new(None), last_wal_seq: Arc::new(AtomicU64::new(0)), replicate_blobs: false, + wal_term_mark: Arc::new(crate::wal::WalTermMark::default()), ship_feed: None, shutdown_checkpoint: Arc::new(AtomicBool::new(false)), checkpoint_thread: std::sync::Mutex::new(None), @@ -983,6 +989,7 @@ impl TidalDb { wal: std::sync::Mutex::new(wal), last_wal_seq: last_seq, replicate_blobs, + wal_term_mark: Arc::new(crate::wal::WalTermMark::default()), ship_feed, shutdown_checkpoint, checkpoint_thread, diff --git a/tidal/src/db/replication_ops.rs b/tidal/src/db/replication_ops.rs index 4c373b7..b269c25 100644 --- a/tidal/src/db/replication_ops.rs +++ b/tidal/src/db/replication_ops.rs @@ -85,6 +85,80 @@ impl TidalDb { self.ship_feed.clone() } + /// The WAL-tail term mark (m11p4): the highest kind-3 term marker this + /// node's log carries, as `(term, seq, leader_region)`. `(0, 0, 0)` = a + /// pre-election log. + /// + /// The term is the `lastLogTerm` half of the vote restriction; the + /// leader region names the STREAM whose seqno numbering the frontier + /// half must be read in (`replication_state().applied_seqno` of that + /// region's shard for a follower; the node's own flushed frontier when + /// it led the term itself). + #[must_use] + pub fn wal_term_mark(&self) -> (u64, u64, u16) { + self.wal_term_mark.snapshot() + } + + /// Journal a kind-3 term marker as this node's next WAL record (m11p4): + /// the FIRST entry an elected leader writes in its term. Waits for the + /// group-commit fsync (the marker ships to followers through the normal + /// feed like any record), folds the term-mark cell, and returns the seqno + /// the marker landed at. + /// + /// # Errors + /// + /// - `TidalError::invalid_input` for `term == 0` (the topology era never + /// journals a marker) or outside cluster mode (no replicated log). + /// - `TidalError::Durability` if the append cannot be staged or its + /// fsync fails — the caller MUST abort the leadership activation: a + /// leader whose term marker is not durable cannot prove its term. + pub fn append_term_marker(&self, term: u64, leader_region: u16) -> crate::Result { + if term == 0 { + return Err(crate::TidalError::invalid_input( + "term markers exist only for elected terms (>= 1)", + )); + } + if !self.replicate_blobs { + return Err(crate::TidalError::invalid_input( + "term markers require cluster mode (the WAL is not a replicated log here)", + )); + } + let sender = { + let wal = self + .wal + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + wal.as_ref().map(crate::wal::WalHandle::sender) + }; + let Some(sender) = sender else { + return Err(crate::TidalError::invalid_input( + "term markers require a durable WAL", + )); + }; + let record = + crate::wal::format::BlobRecord::TermMarker(crate::wal::format::TermMarkerRecord { + term, + leader_region, + }); + let seq = sender + .append_blob_staged(Arc::new(record)) + .map_err(|e| { + crate::TidalError::Durability(crate::schema::DurabilityError { + message: format!("term marker staging failed: {e}"), + }) + })? + .wait() + .map_err(|e| { + crate::TidalError::Durability(crate::schema::DurabilityError { + message: format!("term marker append failed: {e}"), + }) + })?; + super::wal_bridge::bump_last_seq_atomic(&self.last_wal_seq, seq); + self.wal_term_mark.advance(term, seq, leader_region); + tracing::info!(term, seq, "term marker journaled (leadership activation)"); + Ok(seq) + } + /// Read encoded WAL batches covering seqnos `>= from_seq` from the durable /// segments, up to `max_events`/`max_bytes` — the segment read-back behind /// the `StreamSegments` catch-up path (m11p2). diff --git a/tidal/src/replication/commit.rs b/tidal/src/replication/commit.rs index e2e9efa..0d3cba1 100644 --- a/tidal/src/replication/commit.rs +++ b/tidal/src/replication/commit.rs @@ -77,6 +77,12 @@ struct CommitInner { /// leadership change that happened mid-wait (even activate→deactivate→ /// activate cycles that end "active"). epoch: u64, + /// The leadership term this activation is scoped to (m11p4 fencing). + /// A frontier report stamped with any OTHER term never folds — + /// `update_peer_for_term` is the gate — so a deposed leader's followers + /// (or a delayed report from a previous leadership) can never advance + /// this index. 0 = the topology era. + term: u64, /// Per-peer durable marks, monotonic within an epoch. peers: HashMap, /// Cached commit index (recomputed on every mark advance). @@ -123,6 +129,7 @@ impl CommitIndex { inner: Mutex::new(CommitInner { active, epoch: 0, + term: 0, peers: peers.iter().map(|&p| (p, 0)).collect(), commit: 0, scratch: Vec::with_capacity(peers.len()), @@ -167,11 +174,13 @@ impl CommitIndex { /// Activate for a new leadership term starting at `baseline` (the /// promote-time flushed frontier). Marks reset to the baseline — peers /// jump their frontier there via the promote fan-out / catch-up - /// announcements, and nothing above it has been shipped yet. - pub fn activate(&self, baseline: u64) { + /// announcements, and nothing above it has been shipped yet. `term` + /// scopes every subsequent [`update_peer_for_term`] fold (m11p4). + pub fn activate(&self, baseline: u64, term: u64) { let mut inner = self.lock(); inner.active = true; inner.epoch += 1; + inner.term = term; for mark in inner.peers.values_mut() { *mark = baseline; } @@ -180,6 +189,14 @@ impl CommitIndex { self.cv.notify_all(); } + /// The term the index was last activated with (0 = topology era or never + /// activated). For status surfaces; the fold gate reads it under the same + /// lock as the fold. + #[must_use] + pub fn active_term(&self) -> u64 { + self.lock().term + } + /// Deactivate (leadership moved or the queue is shutting down): every /// current and future waiter fails with [`QuorumWaitError::Demoted`]. pub fn deactivate(&self) { @@ -193,11 +210,42 @@ impl CommitIndex { /// Fold a peer's reported durable mark (monotonic; 0 = "unknown" and is /// ignored). Advances the commit index and wakes waiters when the k-th /// largest mark moves. + /// + /// Term-blind: reserved for inputs whose term safety is structural — a + /// ship-ack hint can only exist because the follower ACCEPTED a + /// term-stamped ship (a higher-term follower rejects it), and a heal + /// resume reads the follower's live status on the current leader. + /// Frontier reports from the wire go through + /// [`update_peer_for_term`](Self::update_peer_for_term) instead. pub fn update_peer(&self, peer: ShardId, durable: u64) { if durable == 0 { return; } let mut inner = self.lock(); + self.fold_locked(&mut inner, peer, durable); + } + + /// Fold a peer's durable mark ONLY when `reporter_term` matches the term + /// this index was activated with (m11p4 — design-review C4/C8/C12/C13): + /// a report from a previous leadership, a deposed leader's follower, or a + /// future term this leader has not won never advances the index. Returns + /// whether the report was accepted (refusals are the caller's step-down / + /// logging signal). + pub fn update_peer_for_term(&self, peer: ShardId, durable: u64, reporter_term: u64) -> bool { + if durable == 0 { + return false; + } + let mut inner = self.lock(); + if !inner.active || inner.term != reporter_term { + return false; + } + self.fold_locked(&mut inner, peer, durable); + drop(inner); + true + } + + /// The shared fold body (caller holds the lock). + fn fold_locked(&self, inner: &mut CommitInner, peer: ShardId, durable: u64) { let Some(mark) = inner.peers.get_mut(&peer) else { return; }; @@ -205,10 +253,9 @@ impl CommitIndex { return; } *mark = durable; - let commit = self.compute_commit(&mut inner); + let commit = self.compute_commit(inner); if commit > inner.commit { inner.commit = commit; - drop(inner); self.cv.notify_all(); } } @@ -440,7 +487,7 @@ mod tests { }; std::thread::sleep(Duration::from_millis(20)); idx.deactivate(); - idx.activate(10); + idx.activate(10, 1); assert_eq!(waiter.join().unwrap(), Err(QuorumWaitError::Demoted)); } @@ -449,7 +496,7 @@ mod tests { let idx = CommitIndex::new(&shards(&[1, 2]), true); idx.update_peer(ShardId(1), 50); assert_eq!(idx.committed(), 50); - idx.activate(20); + idx.activate(20, 2); assert_eq!(idx.committed(), 20, "stale pre-term marks must not leak"); // Quorum past the baseline still requires a fresh report. assert!(matches!( @@ -475,7 +522,7 @@ mod tests { idx.wait_for(123, deadline_in(0)), Err(QuorumWaitError::Demoted) ); - idx.activate(0); + idx.activate(0, 1); assert_eq!(idx.wait_for(123, deadline_in(0)), Ok(123)); } } diff --git a/tidal/src/replication/election.rs b/tidal/src/replication/election.rs new file mode 100644 index 0000000..5f7f25d --- /dev/null +++ b/tidal/src/replication/election.rs @@ -0,0 +1,1319 @@ +//! The m11p4 leader-election state machine: a purpose-built, election-only +//! Raft (§5.1–§5.4 + pre-vote + check-quorum + leadership transfer). +//! +//! # Why not a raft crate +//! +//! The replicated log already exists — the WAL is the stream (m11p2) and the +//! quorum commit index gates `ack=quorum` writes over it (m11p3). A raft +//! crate owns its own log and storage contracts; adopting one means running a +//! second log beside the WAL or contorting the crate to delegate. The state +//! that needs consensus here is tiny — `(term, leader)` — and the +//! safety-critical pieces map directly onto primitives the engine already +//! has: the durable WAL frontier ([`flushed_seq`]), the WAL-tail term +//! ([`WalTermMark`]) and the `stream_baseline` persistence discipline. +//! +//! # Purity contract +//! +//! This module does **no I/O and reads no clocks**. Every input carries the +//! caller's `Instant`; randomness is an injected seed (xorshift64*, no `rand` +//! dependency). The driver (tidal-server's election task) owns timers, RPCs, +//! and persistence, and MUST execute [`Action::PersistHardState`] — +//! fsync-complete — **before** any send/reply the same step produced: a vote +//! or term adoption that is not durable does not exist (Raft's hard-state +//! rule; design-review R3/R5). +//! +//! # The up-to-date restriction +//! +//! Votes compare [`LogPosition`] = `(wal_tail_term, durable_frontier)` +//! lexicographically — Raft's `(lastLogTerm, lastLogIndex)`, with both halves +//! derived from the SAME WAL so a crash can only ever understate a node's +//! log, never overstate it (design-review C7; see phase-4.md §1b). +//! +//! [`flushed_seq`]: crate::wal::feed::WalShipFeed::flushed_seq +//! [`WalTermMark`]: crate::wal::WalTermMark + +use std::collections::HashSet; +use std::time::{Duration, Instant}; + +use super::shard::RegionId; + +/// A node's log position for the vote restriction: compared +/// lexicographically, `(wal_tail_term, durable_frontier)`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct LogPosition { + /// Term of the last kind-3 marker in the node's WAL (0 = pre-election log). + pub tail_term: u64, + /// The node's contiguous durable (fsynced) WAL frontier. + pub frontier: u64, +} + +/// Election timing + membership configuration. +#[derive(Debug, Clone)] +pub struct ElectionConfig { + /// This node's region. + pub self_region: RegionId, + /// Every OTHER region in the replica set (the topology's siblings). + pub peers: Vec, + /// Leader → follower heartbeat cadence. + pub heartbeat_interval: Duration, + /// Election timeout range `[min, max)`: a follower that hears no valid + /// leader for a randomized draw from this range starts a pre-vote. + pub election_timeout_min: Duration, + pub election_timeout_max: Duration, + /// Check-quorum window: a leader that cannot reach a majority of peers + /// within this window steps down. MUST satisfy + /// `leader_lease + heartbeat_interval < election_timeout_min` + /// (the C2 bound — the driver validates at config load). + pub leader_lease: Duration, + /// Whether timeouts start elections automatically. When false the + /// detector and fencing still run; elections fire only through the + /// fenced-transfer path (`TimeoutNow`). + pub auto_election: bool, +} + +impl ElectionConfig { + /// Majority of the full replica set (peers + self). + #[must_use] + pub const fn majority(&self) -> usize { + self.peers.len() / 2 + 1 + } +} + +/// The node's current role. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Role { + Follower, + /// Running a pre-vote round (no term has been bumped). + PreCandidate, + /// Running a real election (term bumped + self-vote persisted). + Candidate, + Leader, +} + +/// One outbound vote request the driver must send to every peer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct VoteRpc { + /// The term the candidate asks votes for (for a pre-vote: the PROPOSED + /// term, current+1, without having bumped anything). + pub term: u64, + pub candidate: RegionId, + pub log: LogPosition, + pub prevote: bool, + /// Set on a leadership-transfer election (`TimeoutNow`): voters skip + /// their leader-freshness refusal — the current leader sanctioned this. + pub transfer: bool, +} + +/// What the driver must do after a step. Ordering within one returned `Vec` +/// is the execution order; `PersistHardState` always precedes any send the +/// same step produced. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Action { + /// Fsync `(term, voted_for)` to the election-state file BEFORE executing + /// any later action from this step. + PersistHardState { + term: u64, + voted_for: Option, + }, + /// Fan a vote request out to every peer. + SendVotes(VoteRpc), + /// This node won: run the leadership activation sequence (journal the + /// term marker, activate the ship queue from the flushed frontier, flip + /// the leader view, start heartbeating). The driver MUST step down again + /// ([`ElectionState::on_activation_failed`]) if that sequence fails. + BecomeLeader { term: u64 }, + /// Leadership view change: this node is (now) a follower of `leader` + /// (None = leaderless until contact). The driver flips the leader view + /// FIRST, then deactivates the ship queue (design-review C3 ordering). + BecomeFollower { term: u64, leader: Option }, + /// Send one round of heartbeats to every peer (leader only). + SendHeartbeats { term: u64 }, + /// Tell `target` to start an immediate transfer election at `term + 1`. + SendTimeoutNow { target: RegionId, term: u64 }, + /// The term-0 conflicting-leader misdeployment (two topology files naming + /// different leaders — design-review C5): log ERROR; an election follows. + ConflictingTermZeroLeader { other: RegionId }, +} + +/// The reply to one inbound vote request, returned synchronously to the RPC +/// handler (alongside the step's actions, which execute first). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct VoteReply { + /// The voter's (possibly just-raised) current term. + pub term: u64, + pub granted: bool, +} + +/// xorshift64* — deterministic, dependency-free randomized timeouts. +#[derive(Debug)] +struct XorShift(u64); + +impl XorShift { + const fn next(&mut self) -> u64 { + // A zero state would be a fixed point; map it away. + if self.0 == 0 { + self.0 = 0x9E37_79B9_7F4A_7C15; + } + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_F491_4F6C_DD1D) + } +} + +/// The election state machine. One per cluster node, driven by the +/// tidal-server election task. +/// +/// All methods take `now` (and where relevant the node's own +/// [`LogPosition`]) so the machine itself never reads a clock or the WAL. +#[derive(Debug)] +pub struct ElectionState { + config: ElectionConfig, + role: Role, + /// Mirror of the DURABLE current term (the driver persists before any + /// dependent send; this field assumes that contract). + term: u64, + voted_for: Option, + /// Who leads `term`, as far as this node knows. + leader: Option, + /// Last valid leader contact (heartbeat or term-stamped ship). + last_leader_contact: Option, + /// When the follower/candidate election timer fires next. + election_deadline: Instant, + /// Pre-vote round state: the proposed term and grants collected. + prevote_term: u64, + prevote_grants: HashSet, + /// Real-election grants for `term`. + vote_grants: HashSet, + /// Whether the current candidacy is a sanctioned transfer (skips peer + /// lease refusals). + transfer_candidacy: bool, + /// Leader state: per-peer last successful heartbeat ack. + peer_ack: Vec<(RegionId, Option)>, + /// Leader state: when leadership began / quorum contact was last proven. + lease_anchor: Instant, + /// Leader state: next heartbeat round due. + heartbeat_due: Instant, + rng: XorShift, +} + +impl ElectionState { + /// Build the machine from its durable hard state (term, `voted_for` — as + /// recovered by the election store) and the boot role decision. + /// + /// `boots_as_leader` is true ONLY for the fresh-boot topology-leader case + /// (term 0, no prior state — phase-4.md §1 case 4). Every restart boots + /// follower regardless of what the node was before. + #[must_use] + pub fn new( + config: ElectionConfig, + term: u64, + voted_for: Option, + boots_as_leader: bool, + topology_leader: RegionId, + seed: u64, + now: Instant, + ) -> Self { + let peer_ack = config.peers.iter().map(|&p| (p, None)).collect(); + let mut rng = XorShift(seed ^ 0xA076_1D64_78BD_642F); + let deadline = now + randomized_timeout(&config, &mut rng); + let role = if boots_as_leader { + Role::Leader + } else { + Role::Follower + }; + let leader = if boots_as_leader { + Some(config.self_region) + } else if term == 0 && topology_leader != config.self_region { + // The topology's term-0 leader is the boot belief for a fresh + // follower — but ONLY when it names someone else: a REJOINING + // ex-leader must never re-learn its own leadership from the + // config file (the §1.4-1 amnesia hole). It waits for contact + // or wins an election. + Some(topology_leader) + } else { + None + }; + Self { + config, + role, + term, + voted_for, + leader, + last_leader_contact: None, + election_deadline: deadline, + prevote_term: 0, + prevote_grants: HashSet::new(), + vote_grants: HashSet::new(), + transfer_candidacy: false, + peer_ack, + lease_anchor: now, + heartbeat_due: now, + rng, + } + } + + // ── Read surface ──────────────────────────────────────────────────────── + + #[must_use] + pub const fn role(&self) -> Role { + self.role + } + + #[must_use] + pub const fn term(&self) -> u64 { + self.term + } + + #[must_use] + pub const fn leader(&self) -> Option { + self.leader + } + + #[must_use] + pub const fn config(&self) -> &ElectionConfig { + &self.config + } + + // ── Inputs ────────────────────────────────────────────────────────────── + + /// Drive timers: election timeout (follower/candidate), heartbeat cadence + /// and the check-quorum lease (leader). + pub fn tick(&mut self, now: Instant, my_log: LogPosition) -> Vec { + match self.role { + Role::Leader => self.tick_leader(now), + Role::Follower | Role::PreCandidate | Role::Candidate => { + if now < self.election_deadline { + return Vec::new(); + } + // A quarantined or auto-off node still ticks; it just never + // self-starts an election. + if !self.config.auto_election { + self.reset_election_timer(now); + return Vec::new(); + } + self.start_prevote(now, my_log, false) + } + } + } + + /// A valid leader assertion arrived (heartbeat or term-stamped ship from + /// `leader_region` at `term`). The caller has ALREADY persisted any term + /// adoption this implies? No — this method returns the persist action; + /// the caller executes it before acking the RPC. + pub fn on_leader_contact( + &mut self, + term: u64, + leader_region: RegionId, + now: Instant, + ) -> Vec { + if term < self.term { + // Stale leader; the RPC layer rejects it (the reply carries our + // term, which steps the sender down). + return Vec::new(); + } + let mut actions = Vec::new(); + if term == 0 && self.role == Role::Leader && leader_region != self.config.self_region { + // Term-0 conflicting leadership = mismatched topology files + // (design-review C5). Surface it and arbitrate by election. + actions.push(Action::ConflictingTermZeroLeader { + other: leader_region, + }); + actions.extend(self.start_prevote(now, LogPosition::MAX_SELF, false)); + return actions; + } + if term > self.term { + actions.push(self.adopt_term(term)); + } + // Same-term contact: a leader hearing another same-term leader at + // term >= 1 is impossible (vote uniqueness); a candidate hearing a + // leader at its own term loses the race and follows. + if self.role != Role::Follower { + self.role = Role::Follower; + actions.push(Action::BecomeFollower { + term: self.term, + leader: Some(leader_region), + }); + } else if self.leader != Some(leader_region) { + actions.push(Action::BecomeFollower { + term: self.term, + leader: Some(leader_region), + }); + } + self.leader = Some(leader_region); + self.last_leader_contact = Some(now); + self.reset_election_timer(now); + actions + } + + /// An inbound vote request. Returns the reply (sent AFTER the actions — + /// in particular after the hard-state persist — execute). + pub fn on_vote_request( + &mut self, + rpc: VoteRpc, + my_log: LogPosition, + now: Instant, + ) -> (VoteReply, Vec) { + let mut actions = Vec::new(); + if rpc.prevote { + // Pre-votes change NO state and persist NOTHING. + let lease_fresh = !rpc.transfer + && self + .last_leader_contact + .is_some_and(|t| now.duration_since(t) < self.config.election_timeout_min); + let granted = rpc.term > self.term && !lease_fresh && rpc.log >= my_log; + return ( + VoteReply { + term: self.term, + granted, + }, + actions, + ); + } + + if rpc.term < self.term { + return ( + VoteReply { + term: self.term, + granted: false, + }, + actions, + ); + } + if rpc.term > self.term { + actions.push(self.adopt_term(rpc.term)); + if self.role != Role::Follower { + self.role = Role::Follower; + actions.push(Action::BecomeFollower { + term: self.term, + leader: None, + }); + } + } + let can_vote = self.voted_for.is_none() || self.voted_for == Some(rpc.candidate); + let granted = can_vote && rpc.log >= my_log; + if granted { + self.voted_for = Some(rpc.candidate); + // Persist the vote BEFORE the reply leaves (the driver's + // ordering contract). Re-persisting an unchanged term is cheap + // and keeps one code path. + actions.push(Action::PersistHardState { + term: self.term, + voted_for: self.voted_for, + }); + // Granting resets the timer: the candidate may become leader; + // immediate counter-candidacy would only churn. + self.reset_election_timer(now); + } + ( + VoteReply { + term: self.term, + granted, + }, + actions, + ) + } + + /// A peer's reply to our pre-vote or vote. + pub fn on_vote_response( + &mut self, + from: RegionId, + reply: VoteReply, + prevote: bool, + now: Instant, + my_log: LogPosition, + ) -> Vec { + if reply.term > self.term { + return self.step_down_to(reply.term, None); + } + if prevote { + if self.role != Role::PreCandidate || !reply.granted { + return Vec::new(); + } + self.prevote_grants.insert(from); + if self.prevote_grants.len() >= self.config.majority() { + return self.start_election(now, my_log); + } + return Vec::new(); + } + if self.role != Role::Candidate || !reply.granted { + return Vec::new(); + } + self.vote_grants.insert(from); + if self.vote_grants.len() >= self.config.majority() { + return self.win(now); + } + Vec::new() + } + + /// A follower acked (or refused) our heartbeat. + pub fn on_heartbeat_ack( + &mut self, + from: RegionId, + responder_term: u64, + accepted: bool, + now: Instant, + ) -> Vec { + if responder_term > self.term { + return self.step_down_to(responder_term, None); + } + if self.role == Role::Leader + && accepted + && let Some(slot) = self.peer_ack.iter_mut().find(|(p, _)| *p == from) + { + slot.1 = Some(now); + } + Vec::new() + } + + /// Any RPC response observed a higher term (ship rejection, report + /// refusal, vote reply — the deposed leader's step-down signal). + pub fn on_observed_term(&mut self, term: u64) -> Vec { + if term <= self.term { + return Vec::new(); + } + self.step_down_to(term, None) + } + + /// The current leader told us to take over (fenced transfer): start a + /// real election immediately, skipping pre-vote and the timeout. + pub fn on_timeout_now(&mut self, term: u64, now: Instant, my_log: LogPosition) -> Vec { + if term < self.term || self.role == Role::Leader { + return Vec::new(); + } + let mut actions = Vec::new(); + if term > self.term { + actions.push(self.adopt_term(term)); + } + self.transfer_candidacy = true; + actions.extend(self.start_election(now, my_log)); + actions + } + + /// The driver's leadership-activation sequence failed (term-marker fsync, + /// queue activation): abandon the leadership without serving a write. + pub fn on_activation_failed(&mut self, now: Instant) -> Vec { + if self.role != Role::Leader { + return Vec::new(); + } + self.role = Role::Follower; + self.leader = None; + self.reset_election_timer(now); + vec![Action::BecomeFollower { + term: self.term, + leader: None, + }] + } + + /// Operator-initiated transfer (the leader side): the driver has verified + /// the target's durable mark reached the flushed frontier; emit the + /// `TimeoutNow`. + pub fn transfer_to(&mut self, target: RegionId) -> Vec { + if self.role != Role::Leader || target == self.config.self_region { + return Vec::new(); + } + vec![Action::SendTimeoutNow { + target, + term: self.term, + }] + } + + /// Bridge for the LEGACY term-0 fan-out promote (the pre-m11p4 operator + /// verb, kept for mixed-version clusters): force the machine's + /// leadership view to match an externally applied term-0 promote so it + /// does not fight the verb with stale heartbeats. A no-op at term ≥ 1 — + /// elected leadership is never overridden by the legacy verb. + pub fn force_term0_view(&mut self, leader: RegionId, now: Instant) { + if self.term != 0 { + return; + } + self.leader = Some(leader); + if leader == self.config.self_region { + self.role = Role::Leader; + self.lease_anchor = now; + for (_, ack) in &mut self.peer_ack { + *ack = None; + } + self.heartbeat_due = now + self.config.heartbeat_interval; + } else { + self.role = Role::Follower; + self.last_leader_contact = Some(now); + self.reset_election_timer(now); + } + } + + // ── Internals ─────────────────────────────────────────────────────────── + + fn tick_leader(&mut self, now: Instant) -> Vec { + // Check-quorum: self + every peer whose last heartbeat ack (or the + // leadership start, as the grace anchor) is within the lease window. + // + // Gated on `auto_election`: the step-down exists to bound the + // dual-leader window BEFORE a successor is elected (the C2 lease + // bound) — with automatic elections off the operator owns failover + // and no successor can appear, so stepping down would only convert a + // follower outage into total write unavailability (the pre-m11p4 + // availability posture the knob deliberately preserves). + let mut fresh = 1usize; // self + for (_, ack) in &self.peer_ack { + let anchor = ack.unwrap_or(self.lease_anchor); + if now.duration_since(anchor) < self.config.leader_lease { + fresh += 1; + } + } + if self.config.auto_election && fresh < self.config.majority() { + // Lost quorum contact: stop accepting writes BEFORE any + // successor can be elected (the C2 lease bound). + self.leader = None; + self.role = Role::Follower; + self.reset_election_timer(now); + return vec![Action::BecomeFollower { + term: self.term, + leader: None, + }]; + } + if now >= self.heartbeat_due { + self.heartbeat_due = now + self.config.heartbeat_interval; + return vec![Action::SendHeartbeats { term: self.term }]; + } + Vec::new() + } + + fn start_prevote(&mut self, now: Instant, my_log: LogPosition, transfer: bool) -> Vec { + self.role = Role::PreCandidate; + self.prevote_term = self.term + 1; + self.prevote_grants.clear(); + self.prevote_grants.insert(self.config.self_region); + self.transfer_candidacy = transfer; + self.reset_election_timer(now); + if self.prevote_grants.len() >= self.config.majority() { + // Single-node replica set: the pre-vote is already a majority. + return self.start_election(now, my_log); + } + vec![Action::SendVotes(VoteRpc { + term: self.prevote_term, + candidate: self.config.self_region, + log: my_log, + prevote: true, + transfer, + })] + } + + fn start_election(&mut self, now: Instant, my_log: LogPosition) -> Vec { + self.role = Role::Candidate; + self.term += 1; + self.voted_for = Some(self.config.self_region); + self.leader = None; + self.vote_grants.clear(); + self.vote_grants.insert(self.config.self_region); + self.reset_election_timer(now); + let mut actions = vec![Action::PersistHardState { + term: self.term, + voted_for: self.voted_for, + }]; + if self.vote_grants.len() >= self.config.majority() { + actions.extend(self.win(now)); + return actions; + } + actions.push(Action::SendVotes(VoteRpc { + term: self.term, + candidate: self.config.self_region, + log: my_log, + prevote: false, + transfer: self.transfer_candidacy, + })); + actions + } + + fn win(&mut self, now: Instant) -> Vec { + self.role = Role::Leader; + self.leader = Some(self.config.self_region); + self.transfer_candidacy = false; + self.lease_anchor = now; + for (_, ack) in &mut self.peer_ack { + *ack = None; + } + self.heartbeat_due = now + self.config.heartbeat_interval; + vec![ + Action::BecomeLeader { term: self.term }, + Action::SendHeartbeats { term: self.term }, + ] + } + + fn step_down_to(&mut self, term: u64, leader: Option) -> Vec { + let mut actions = Vec::new(); + if term > self.term { + actions.push(self.adopt_term(term)); + } + self.role = Role::Follower; + self.leader = leader; + actions.push(Action::BecomeFollower { + term: self.term, + leader, + }); + actions + } + + /// Raise the durable term (terms are strictly monotonic — never lowered) + /// and clear the vote. Returns the persist action the driver must execute + /// before any dependent send. + fn adopt_term(&mut self, term: u64) -> Action { + debug_assert!(term > self.term, "terms never move backwards"); + self.term = term; + self.voted_for = None; + Action::PersistHardState { + term, + voted_for: None, + } + } + + fn reset_election_timer(&mut self, now: Instant) { + self.election_deadline = now + randomized_timeout(&self.config, &mut self.rng); + } +} + +impl LogPosition { + /// A position that compares at-or-above anything — used only for the + /// term-0 conflict arbitration where the node's own log is authoritative + /// to itself. + const MAX_SELF: Self = Self { + tail_term: u64::MAX, + frontier: u64::MAX, + }; +} + +/// Draw a timeout uniformly from `[min, max)`. +fn randomized_timeout(config: &ElectionConfig, rng: &mut XorShift) -> Duration { + let min = config.election_timeout_min; + let max = config + .election_timeout_max + .max(min + Duration::from_millis(1)); + let span = max.saturating_sub(min); + let span_ms = u64::try_from(span.as_millis()).unwrap_or(u64::MAX).max(1); + min + Duration::from_millis(rng.next() % span_ms) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + fn region(n: u16) -> RegionId { + RegionId(n) + } + + fn config(self_n: u16, peers: &[u16]) -> ElectionConfig { + ElectionConfig { + self_region: region(self_n), + peers: peers.iter().map(|&p| region(p)).collect(), + heartbeat_interval: Duration::from_millis(300), + election_timeout_min: Duration::from_millis(1500), + election_timeout_max: Duration::from_millis(3000), + leader_lease: Duration::from_millis(900), + auto_election: true, + } + } + + fn log(term: u64, frontier: u64) -> LogPosition { + LogPosition { + tail_term: term, + frontier, + } + } + + fn follower(self_n: u16, peers: &[u16], term: u64, now: Instant) -> ElectionState { + ElectionState::new(config(self_n, peers), term, None, false, region(0), 7, now) + } + + /// Advance past the worst-case election timeout. + fn past_timeout(now: Instant) -> Instant { + now + Duration::from_millis(3001) + } + + #[test] + fn follower_times_out_into_prevote_then_election_then_leadership() { + let t0 = Instant::now(); + let mut m = follower(0, &[1, 2], 0, t0); + let t1 = past_timeout(t0); + let actions = m.tick(t1, log(0, 10)); + assert_eq!(m.role(), Role::PreCandidate); + assert!( + matches!(&actions[..], [Action::SendVotes(rpc)] if rpc.prevote && rpc.term == 1), + "pre-vote proposes term+1 without bumping: {actions:?}" + ); + assert_eq!(m.term(), 0, "pre-vote must not inflate the term"); + + // One pre-vote grant (self + 1 = majority of 3) → real election. + let actions = m.on_vote_response( + region(1), + VoteReply { + term: 0, + granted: true, + }, + true, + t1, + log(0, 10), + ); + assert_eq!(m.role(), Role::Candidate); + assert_eq!(m.term(), 1); + assert!( + matches!( + &actions[..], + [ + Action::PersistHardState { + term: 1, + voted_for: Some(v) + }, + Action::SendVotes(rpc) + ] if *v == region(0) && !rpc.prevote && rpc.term == 1 + ), + "candidacy persists hard state BEFORE sending votes: {actions:?}" + ); + + // One real grant → leader. + let actions = m.on_vote_response( + region(2), + VoteReply { + term: 1, + granted: true, + }, + false, + t1, + log(0, 10), + ); + assert_eq!(m.role(), Role::Leader); + assert!( + matches!( + &actions[..], + [ + Action::BecomeLeader { term: 1 }, + Action::SendHeartbeats { term: 1 } + ] + ), + "{actions:?}" + ); + } + + #[test] + fn vote_restriction_refuses_stale_logs_lexicographically() { + let t0 = Instant::now(); + let mut m = follower(1, &[0, 2], 5, t0); + // Candidate with a LOWER tail term loses even with a higher frontier. + let (reply, _) = m.on_vote_request( + VoteRpc { + term: 6, + candidate: region(2), + log: log(1, 1_000_000), + prevote: false, + transfer: false, + }, + log(2, 10), + t0, + ); + assert!(!reply.granted, "lexicographic: (1, 1M) < (2, 10)"); + // Term adopted even when the vote is refused. + assert_eq!(m.term(), 6); + + // Same tail term, higher frontier wins. + let (reply, actions) = m.on_vote_request( + VoteRpc { + term: 7, + candidate: region(0), + log: log(2, 11), + prevote: false, + transfer: false, + }, + log(2, 10), + t0, + ); + assert!(reply.granted); + assert!( + actions + .iter() + .any(|a| matches!(a, Action::PersistHardState { term: 7, voted_for: Some(v) } if *v == region(0))), + "the granted vote persists before the reply: {actions:?}" + ); + } + + #[test] + fn at_most_one_vote_per_term() { + let t0 = Instant::now(); + let mut m = follower(1, &[0, 2], 0, t0); + let rpc = |cand: u16| VoteRpc { + term: 3, + candidate: region(cand), + log: log(0, 100), + prevote: false, + transfer: false, + }; + let (first, _) = m.on_vote_request(rpc(0), log(0, 10), t0); + assert!(first.granted); + let (second, _) = m.on_vote_request(rpc(2), log(0, 10), t0); + assert!(!second.granted, "already voted for region 0 in term 3"); + // Idempotent re-grant to the SAME candidate (a retried RPC). + let (again, _) = m.on_vote_request(rpc(0), log(0, 10), t0); + assert!(again.granted); + } + + #[test] + fn prevote_lease_refusal_protects_a_live_leader() { + let t0 = Instant::now(); + let mut m = follower(1, &[0, 2], 4, t0); + let _ = m.on_leader_contact(4, region(0), t0); + // A partitioned node probes with a pre-vote shortly after. + let t1 = t0 + Duration::from_millis(200); + let (reply, _) = m.on_vote_request( + VoteRpc { + term: 5, + candidate: region(2), + log: log(4, 1_000), + prevote: true, + transfer: false, + }, + log(4, 500), + t1, + ); + assert!(!reply.granted, "leader heard {:?} ago; lease refusal", 200); + assert_eq!(m.term(), 4, "pre-vote never changes state"); + + // The same probe as a sanctioned TRANSFER bypasses the lease. + let (reply, _) = m.on_vote_request( + VoteRpc { + term: 5, + candidate: region(2), + log: log(4, 1_000), + prevote: true, + transfer: true, + }, + log(4, 500), + t1, + ); + assert!(reply.granted); + } + + #[test] + fn leader_contact_resets_timer_and_demotes_candidates() { + let t0 = Instant::now(); + let mut m = follower(0, &[1, 2], 0, t0); + let t1 = past_timeout(t0); + let _ = m.tick(t1, log(0, 0)); + assert_eq!(m.role(), Role::PreCandidate); + let actions = m.on_leader_contact(1, region(1), t1); + assert_eq!(m.role(), Role::Follower); + assert_eq!(m.leader(), Some(region(1))); + assert!( + actions + .iter() + .any(|a| matches!(a, Action::PersistHardState { term: 1, .. })), + "adopting term 1 persists: {actions:?}" + ); + } + + #[test] + fn check_quorum_steps_down_without_majority_acks() { + let t0 = Instant::now(); + let mut m = follower(0, &[1, 2], 0, t0); + // Become leader directly via timeout + grants. + let t1 = past_timeout(t0); + let _ = m.tick(t1, log(0, 0)); + let _ = m.on_vote_response( + region(1), + VoteReply { + term: 0, + granted: true, + }, + true, + t1, + log(0, 0), + ); + let _ = m.on_vote_response( + region(1), + VoteReply { + term: 1, + granted: true, + }, + false, + t1, + log(0, 0), + ); + assert_eq!(m.role(), Role::Leader); + + // Within the lease (grace anchor) the leader holds. + let t2 = t1 + Duration::from_millis(500); + assert!( + m.tick(t2, log(1, 0)) + .iter() + .all(|a| !matches!(a, Action::BecomeFollower { .. })), + "inside the lease the leader must not step down" + ); + + // Past the lease with zero acks: step down. + let t3 = t1 + Duration::from_millis(901); + let actions = m.tick(t3, log(1, 0)); + assert_eq!(m.role(), Role::Follower); + assert!( + matches!( + &actions[..], + [Action::BecomeFollower { + term: 1, + leader: None + }] + ), + "{actions:?}" + ); + + // Acks keep a leader alive: re-win and feed acks. + let _ = m.tick(past_timeout(t3), log(1, 0)); + let _ = m.on_vote_response( + region(1), + VoteReply { + term: 1, + granted: true, + }, + true, + past_timeout(t3), + log(1, 0), + ); + let t4 = past_timeout(t3); + let _ = m.on_vote_response( + region(1), + VoteReply { + term: 2, + granted: true, + }, + false, + t4, + log(1, 0), + ); + assert_eq!(m.role(), Role::Leader); + let t5 = t4 + Duration::from_millis(800); + let _ = m.on_heartbeat_ack(region(1), 2, true, t5); + let t6 = t4 + Duration::from_millis(1_600); + assert!( + m.tick(t6, log(2, 0)) + .iter() + .all(|a| !matches!(a, Action::BecomeFollower { .. })), + "one fresh peer ack (+self) = majority of 3; the lease holds" + ); + } + + #[test] + fn higher_term_from_any_response_steps_a_leader_down() { + let t0 = Instant::now(); + let mut m = follower(0, &[1, 2], 0, t0); + let t1 = past_timeout(t0); + let _ = m.tick(t1, log(0, 0)); + let _ = m.on_vote_response( + region(1), + VoteReply { + term: 0, + granted: true, + }, + true, + t1, + log(0, 0), + ); + let _ = m.on_vote_response( + region(1), + VoteReply { + term: 1, + granted: true, + }, + false, + t1, + log(0, 0), + ); + assert_eq!(m.role(), Role::Leader); + let actions = m.on_observed_term(9); + assert_eq!(m.role(), Role::Follower); + assert_eq!(m.term(), 9); + assert!( + matches!( + &actions[..], + [ + Action::PersistHardState { + term: 9, + voted_for: None + }, + Action::BecomeFollower { term: 9, .. } + ] + ), + "persist precedes the role change: {actions:?}" + ); + } + + #[test] + fn timeout_now_skips_prevote_and_timeout() { + let t0 = Instant::now(); + let mut m = follower(1, &[0, 2], 3, t0); + let actions = m.on_timeout_now(3, t0, log(3, 50)); + assert_eq!(m.role(), Role::Candidate); + assert_eq!(m.term(), 4); + assert!( + actions.iter().any( + |a| matches!(a, Action::SendVotes(rpc) if !rpc.prevote && rpc.transfer && rpc.term == 4) + ), + "{actions:?}" + ); + } + + #[test] + fn auto_election_off_never_self_starts() { + let t0 = Instant::now(); + let mut cfg = config(0, &[1, 2]); + cfg.auto_election = false; + let mut m = ElectionState::new(cfg, 0, None, false, region(1), 7, t0); + let actions = m.tick(past_timeout(t0), log(0, 0)); + assert!(actions.is_empty()); + assert_eq!(m.role(), Role::Follower); + // TimeoutNow (operator promote) still works. + let actions = m.on_timeout_now(0, past_timeout(t0), log(0, 0)); + assert_eq!(m.role(), Role::Candidate); + assert!(!actions.is_empty()); + } + + #[test] + fn rejoining_term0_ex_leader_never_self_leads_from_topology() { + let t0 = Instant::now(); + // boots_as_leader=false (a rejoin) with the topology naming SELF: + // the machine must come up leaderless, not self-leading (§1.4-1). + let m = ElectionState::new(config(0, &[1, 2]), 0, None, false, region(0), 7, t0); + assert_eq!(m.role(), Role::Follower); + assert_eq!( + m.leader(), + None, + "a rejoin never trusts the topology about ITSELF" + ); + // A rejoin where the topology names ANOTHER node keeps the hint. + let m = ElectionState::new(config(1, &[0, 2]), 0, None, false, region(0), 7, t0); + assert_eq!(m.leader(), Some(region(0))); + } + + #[test] + fn fresh_boot_topology_leader_leads_term_zero() { + let t0 = Instant::now(); + let m = ElectionState::new(config(0, &[1, 2]), 0, None, true, region(0), 7, t0); + assert_eq!(m.role(), Role::Leader); + assert_eq!(m.term(), 0); + assert_eq!(m.leader(), Some(region(0))); + } + + #[test] + fn term_zero_conflicting_leader_triggers_election() { + let t0 = Instant::now(); + let mut m = ElectionState::new(config(0, &[1, 2]), 0, None, true, region(0), 7, t0); + let actions = m.on_leader_contact(0, region(1), t0); + assert!( + actions.iter().any( + |a| matches!(a, Action::ConflictingTermZeroLeader { other } if *other == region(1)) + ), + "{actions:?}" + ); + assert!( + matches!( + m.role(), + Role::PreCandidate | Role::Candidate | Role::Leader + ), + "arbitration by election, not silent dual leadership" + ); + } + + #[test] + fn candidate_retries_via_new_prevote_after_timeout() { + let t0 = Instant::now(); + let mut m = follower(0, &[1, 2], 0, t0); + let t1 = past_timeout(t0); + let _ = m.tick(t1, log(0, 0)); + let _ = m.on_vote_response( + region(1), + VoteReply { + term: 0, + granted: true, + }, + true, + t1, + log(0, 0), + ); + assert_eq!(m.role(), Role::Candidate); + assert_eq!(m.term(), 1); + // No votes arrive; the next timeout re-enters pre-vote (no term churn + // without a pre-vote majority — bounded churn under flapping links). + let t2 = past_timeout(t1); + let actions = m.tick(t2, log(0, 0)); + assert_eq!(m.role(), Role::PreCandidate); + assert!( + matches!(&actions[..], [Action::SendVotes(rpc)] if rpc.prevote && rpc.term == 2), + "{actions:?}" + ); + assert_eq!( + m.term(), + 1, + "the failed candidacy's term stands; no inflation" + ); + } + + #[test] + fn activation_failure_steps_down_without_a_vote_or_term_change() { + let t0 = Instant::now(); + let mut m = follower(0, &[1, 2], 0, t0); + let t1 = past_timeout(t0); + let _ = m.tick(t1, log(0, 0)); + let _ = m.on_vote_response( + region(1), + VoteReply { + term: 0, + granted: true, + }, + true, + t1, + log(0, 0), + ); + let _ = m.on_vote_response( + region(1), + VoteReply { + term: 1, + granted: true, + }, + false, + t1, + log(0, 0), + ); + assert_eq!(m.role(), Role::Leader); + // The driver's activation sequence failed (term-marker fsync): the + // won leadership is abandoned WITHOUT touching the durable term — + // the candidacy's persist already covers it — and the node returns + // to a leaderless follower with a fresh election timer. + let actions = m.on_activation_failed(t1); + assert_eq!(m.role(), Role::Follower); + assert_eq!(m.leader(), None); + assert_eq!(m.term(), 1, "the term stands; only the role rolls back"); + assert!( + matches!( + &actions[..], + [Action::BecomeFollower { + term: 1, + leader: None + }] + ), + "{actions:?}" + ); + // Idempotent / no-op when not leading. + assert!(m.on_activation_failed(t1).is_empty()); + } + + #[test] + fn transfer_to_emits_timeout_now_only_from_a_leader_to_a_peer() { + let t0 = Instant::now(); + let mut m = follower(0, &[1, 2], 0, t0); + // Not a leader: no action. + assert!(m.transfer_to(region(1)).is_empty()); + let t1 = past_timeout(t0); + let _ = m.tick(t1, log(0, 0)); + let _ = m.on_vote_response( + region(1), + VoteReply { + term: 0, + granted: true, + }, + true, + t1, + log(0, 0), + ); + let _ = m.on_vote_response( + region(1), + VoteReply { + term: 1, + granted: true, + }, + false, + t1, + log(0, 0), + ); + assert_eq!(m.role(), Role::Leader); + // Self-transfer is a no-op. + assert!(m.transfer_to(region(0)).is_empty()); + // A peer transfer emits the sanctioned TimeoutNow at the CURRENT term. + let actions = m.transfer_to(region(2)); + assert!( + matches!( + &actions[..], + [Action::SendTimeoutNow { + target, + term: 1 + }] if *target == region(2) + ), + "{actions:?}" + ); + assert_eq!( + m.role(), + Role::Leader, + "the transfer itself changes nothing" + ); + } + + #[test] + fn stale_vote_reply_terms_are_ignored() { + let t0 = Instant::now(); + let mut m = follower(0, &[1, 2, 3, 4], 0, t0); + let t1 = past_timeout(t0); + let _ = m.tick(t1, log(0, 0)); + // majority of 5 = 3: self + 2 pre-vote grants needed. + let _ = m.on_vote_response( + region(1), + VoteReply { + term: 0, + granted: true, + }, + true, + t1, + log(0, 0), + ); + assert_eq!(m.role(), Role::PreCandidate, "2 of 3 needed grants so far"); + let _ = m.on_vote_response( + region(2), + VoteReply { + term: 0, + granted: true, + }, + true, + t1, + log(0, 0), + ); + assert_eq!(m.role(), Role::Candidate); + // A duplicate grant from the same region must not double-count. + let _ = m.on_vote_response( + region(1), + VoteReply { + term: 1, + granted: true, + }, + false, + t1, + log(0, 0), + ); + let _ = m.on_vote_response( + region(1), + VoteReply { + term: 1, + granted: true, + }, + false, + t1, + log(0, 0), + ); + assert_eq!(m.role(), Role::Candidate, "1 unique grant + self < 3"); + let _ = m.on_vote_response( + region(3), + VoteReply { + term: 1, + granted: true, + }, + false, + t1, + log(0, 0), + ); + assert_eq!(m.role(), Role::Leader); + } +} diff --git a/tidal/src/replication/election_store.rs b/tidal/src/replication/election_store.rs new file mode 100644 index 0000000..5d0d65d --- /dev/null +++ b/tidal/src/replication/election_store.rs @@ -0,0 +1,277 @@ +//! Durable election hard state (m11p4): `(current_term, voted_for)`. +//! +//! One small file, `data_dir/election_state`, written with the exact +//! `stream_baseline` discipline — write `.tmp`, [`sync_file_durable`] +//! (`F_FULLFSYNC` on macOS), atomic rename, [`sync_dir_durable`] — and framed +//! with a magic + version + BLAKE3 checksum so corruption is *detected*, +//! never guessed around. +//! +//! # Persistence is a fence +//! +//! [`ElectionStore::persist`] MUST complete before any externally visible +//! action that depends on the state it writes: before a vote reply leaves, +//! before a higher term is acknowledged, before leadership is assumed. Votes +//! and term changes are rare (never per-write), so the fsync cost is +//! irrelevant. +//! +//! # Boot classification (phase-4.md §1; design-review C1) +//! +//! | File | Data dir | Outcome | +//! |---|---|---| +//! | reads clean | — | [`BootState::Rejoin`] — always a follower | +//! | corrupt | — | **error** — refuse to boot; a node that cannot prove its term must not guess term 0 | +//! | absent | has a WAL | [`BootState::StateFileLost`] — follower at term 0, loud ERROR (the file was deleted out from under a node that has run before) | +//! | absent | fresh | [`BootState::Fresh`] — the topology's term-0 roles apply; persist `{term: 0}` before serving | +//! +//! [`sync_file_durable`]: crate::wal::sync_file_durable +//! [`sync_dir_durable`]: crate::wal::sync_dir_durable + +use std::path::{Path, PathBuf}; + +use super::shard::RegionId; +use crate::wal::error::WalError; + +/// File name inside the node's data dir. +const ELECTION_STATE_FILE: &str = "election_state"; + +/// File magic: "TELE" (tidalDB election). +const MAGIC: [u8; 4] = *b"TELE"; + +/// Format version 1. +const VERSION: u8 = 1; + +/// On-disk size: magic 4 + version 1 + voted-flag 1 + voted u16 + term u64 + +/// checksum 16 (truncated BLAKE3 over bytes 0..16). +const FILE_SIZE: usize = 4 + 1 + 1 + 2 + 8 + 16; + +/// The durable hard state. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HardState { + /// Highest term this node has seen. Strictly monotonic. + pub current_term: u64, + /// The vote cast in `current_term` (at most one). + pub voted_for: Option, +} + +/// What [`ElectionStore::load`] found at boot. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BootState { + /// The state file read clean: the node ALWAYS boots as a follower — + /// leadership is learned from heartbeats or won by election, never + /// assumed from a restart (the §1.4-1 fix). + Rejoin(HardState), + /// No state file and the data dir is fresh (no WAL): a genuinely new + /// node — the topology's term-0 roles apply exactly as before m11p4. + Fresh, + /// No state file but a WAL exists: the file was deleted out from under a + /// node that has run before. The node boots as a FOLLOWER at term 0 (it + /// must never re-assume topology leadership on a guess) and the caller + /// logs ERROR. Terms recover upward from the first contact. + StateFileLost, +} + +/// Reader/writer for the election-state file. +#[derive(Debug, Clone)] +pub struct ElectionStore { + path: PathBuf, + dir: PathBuf, +} + +impl ElectionStore { + /// Bind to `data_dir/election_state`. + #[must_use] + pub fn new(data_dir: &Path) -> Self { + Self { + path: data_dir.join(ELECTION_STATE_FILE), + dir: data_dir.to_path_buf(), + } + } + + /// Classify boot per the table in the module docs. `wal_exists` is the + /// caller's "data dir is not fresh" signal (any WAL segment present). + /// + /// # Errors + /// + /// [`WalError::Corruption`] when the file exists but cannot be read or + /// fails its magic/version/checksum — the node must refuse to boot + /// rather than guess term 0 (design-review C1 case 2). + pub fn load(&self, wal_exists: bool) -> Result { + let bytes = match std::fs::read(&self.path) { + Ok(b) => b, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return Ok(if wal_exists { + BootState::StateFileLost + } else { + BootState::Fresh + }); + } + Err(e) => { + return Err(WalError::Corruption { + message: format!( + "election_state at {} exists but cannot be read ({e}); a node that \ + cannot prove its term must not guess — repair or reseed the data dir", + self.path.display() + ), + }); + } + }; + let state = decode(&bytes).map_err(|why| WalError::Corruption { + message: format!( + "election_state at {} is corrupt ({why}); booting at term 0 would re-open \ + the restart-amnesia split-brain — repair or reseed the data dir", + self.path.display() + ), + })?; + Ok(BootState::Rejoin(state)) + } + + /// Durably persist `state`: write `.tmp`, fsync, rename, fsync dir. + /// MUST complete before any action that depends on the new state. + /// + /// # Errors + /// + /// [`WalError::Io`] on any filesystem failure — the caller must treat + /// the state transition as NOT having happened. + pub fn persist(&self, state: HardState) -> Result<(), WalError> { + let bytes = encode(state); + let tmp = self.path.with_extension("tmp"); + { + use std::io::Write; + let mut f = std::fs::File::create(&tmp)?; + f.write_all(&bytes)?; + crate::wal::sync_file_durable(&f)?; + } + std::fs::rename(&tmp, &self.path)?; + crate::wal::sync_dir_durable(&self.dir)?; + Ok(()) + } +} + +fn encode(state: HardState) -> [u8; FILE_SIZE] { + let mut buf = [0u8; FILE_SIZE]; + buf[0..4].copy_from_slice(&MAGIC); + buf[4] = VERSION; + match state.voted_for { + Some(region) => { + buf[5] = 1; + buf[6..8].copy_from_slice(®ion.0.to_le_bytes()); + } + None => { + buf[5] = 0; + } + } + buf[8..16].copy_from_slice(&state.current_term.to_le_bytes()); + let hash = blake3::hash(&buf[0..16]); + buf[16..32].copy_from_slice(&hash.as_bytes()[0..16]); + buf +} + +fn decode(bytes: &[u8]) -> Result { + if bytes.len() != FILE_SIZE { + return Err(format!("{} bytes, expected {FILE_SIZE}", bytes.len())); + } + if bytes[0..4] != MAGIC { + return Err("bad magic".into()); + } + if bytes[4] != VERSION { + return Err(format!("unknown version {}", bytes[4])); + } + let hash = blake3::hash(&bytes[0..16]); + if bytes[16..32] != hash.as_bytes()[0..16] { + return Err("checksum mismatch".into()); + } + let voted_for = match bytes[5] { + 0 => None, + 1 => Some(RegionId(u16::from_le_bytes([bytes[6], bytes[7]]))), + other => return Err(format!("bad voted_for flag {other}")), + }; + let current_term = u64::from_le_bytes( + bytes[8..16] + .try_into() + .map_err(|_| "bad term bytes".to_string())?, + ); + Ok(HardState { + current_term, + voted_for, + }) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn roundtrip_and_boot_classification() { + let dir = tempfile::tempdir().unwrap(); + let store = ElectionStore::new(dir.path()); + + // Absent + fresh dir = Fresh. + assert_eq!(store.load(false).unwrap(), BootState::Fresh); + // Absent + WAL present = StateFileLost (never topology leadership). + assert_eq!(store.load(true).unwrap(), BootState::StateFileLost); + + let state = HardState { + current_term: 7, + voted_for: Some(RegionId(2)), + }; + store.persist(state).unwrap(); + assert_eq!(store.load(true).unwrap(), BootState::Rejoin(state)); + assert_eq!(store.load(false).unwrap(), BootState::Rejoin(state)); + + // Overwrite moves forward. + let newer = HardState { + current_term: 9, + voted_for: None, + }; + store.persist(newer).unwrap(); + assert_eq!(store.load(true).unwrap(), BootState::Rejoin(newer)); + } + + #[test] + fn corruption_refuses_to_classify() { + let dir = tempfile::tempdir().unwrap(); + let store = ElectionStore::new(dir.path()); + store + .persist(HardState { + current_term: 3, + voted_for: None, + }) + .unwrap(); + // Flip a byte in the term field. + let path = dir.path().join("election_state"); + let mut bytes = std::fs::read(&path).unwrap(); + bytes[9] ^= 0xFF; + std::fs::write(&path, &bytes).unwrap(); + let err = store.load(true).expect_err("corruption must refuse boot"); + assert!( + err.to_string().contains("corrupt"), + "actionable error, got: {err}" + ); + + // Truncation refuses too. + std::fs::write(&path, &bytes[0..10]).unwrap(); + assert!(store.load(true).is_err()); + } + + #[test] + fn tmp_file_never_shadows_the_real_state() { + let dir = tempfile::tempdir().unwrap(); + let store = ElectionStore::new(dir.path()); + store + .persist(HardState { + current_term: 5, + voted_for: Some(RegionId(1)), + }) + .unwrap(); + // A crash can orphan a .tmp; load must read only the real file. + std::fs::write(dir.path().join("election_state.tmp"), b"garbage").unwrap(); + assert_eq!( + store.load(true).unwrap(), + BootState::Rejoin(HardState { + current_term: 5, + voted_for: Some(RegionId(1)), + }) + ); + } +} diff --git a/tidal/src/replication/in_process.rs b/tidal/src/replication/in_process.rs index 6d8e0b3..02d304c 100644 --- a/tidal/src/replication/in_process.rs +++ b/tidal/src/replication/in_process.rs @@ -152,6 +152,8 @@ mod tests { event_count: 5, leader_last_seq: seqno, stream_baseline: 0, + term: 0, + leader_region: 0, } } @@ -212,6 +214,8 @@ mod tests { event_count: 0, leader_last_seq: 0, stream_baseline: 0, + term: 0, + leader_region: 0, }; let result = t0.send_segment(ShardId(1), payload); assert!(result.is_err()); diff --git a/tidal/src/replication/mod.rs b/tidal/src/replication/mod.rs index 07e1ad5..f58a73e 100644 --- a/tidal/src/replication/mod.rs +++ b/tidal/src/replication/mod.rs @@ -6,6 +6,8 @@ pub mod commit; pub mod control; pub mod crdt; +pub mod election; +pub mod election_store; pub mod idempotency; pub mod in_process; pub mod lag; @@ -26,6 +28,10 @@ pub mod upgrade; pub use commit::{CommitIndex, QuorumWaitError}; pub use control::{ClusterHealth, ControlPlane, RegionHealth, ShardStats}; pub use crdt::{Hlc, HlcTimestamp}; +pub use election::{ + Action as ElectionAction, ElectionConfig, ElectionState, LogPosition, Role, VoteReply, VoteRpc, +}; +pub use election_store::{BootState, ElectionStore, HardState}; pub use idempotency::{IdempotencyKey, IdempotencyStore}; pub use in_process::{InProcessTransport, InProcessTransportFactory}; pub use lag::ReplicationLagGauge; diff --git a/tidal/src/replication/receiver.rs b/tidal/src/replication/receiver.rs index eda7381..91a080f 100644 --- a/tidal/src/replication/receiver.rs +++ b/tidal/src/replication/receiver.rs @@ -617,6 +617,10 @@ fn prepare_segment( } BatchPayload::ItemMetadata(record) => blobs.push(BlobRecord::ItemMetadata(record)), BatchPayload::Embedding(record) => blobs.push(BlobRecord::Embedding(record)), + // m11p4: a replicated term marker rides the blob path — the + // applier journals it into THIS node's WAL (advancing its durable + // tail term) and folds the term-mark cell instead of storage. + BatchPayload::TermMarker(record) => blobs.push(BlobRecord::TermMarker(record)), } let batch_size = HEADER_SIZE + header.payload_len as usize; @@ -1471,6 +1475,8 @@ mod tests { event_count: entities.len() as u64, leader_last_seq: last, stream_baseline: 0, + term: 0, + leader_region: 0, } }; @@ -1527,6 +1533,8 @@ mod tests { event_count: 1, leader_last_seq: 1, stream_baseline: 0, + term: 0, + leader_region: 0, }) .unwrap(); @@ -1575,6 +1583,8 @@ mod tests { event_count: 1, leader_last_seq: 1, stream_baseline: 0, + term: 0, + leader_region: 0, }) .unwrap(); @@ -1628,6 +1638,8 @@ mod tests { event_count: 1, leader_last_seq: 1, stream_baseline: 0, + term: 0, + leader_region: 0, }) .unwrap(); @@ -1769,6 +1781,7 @@ mod tests { BlobRecord::Embedding(r) => { format!("emb:{}:{}", r.entity_id, r.values.len()) } + BlobRecord::TermMarker(r) => format!("term:{}", r.term), }) .collect(); self.applied.lock().unwrap().extend(lines); @@ -1817,6 +1830,8 @@ mod tests { event_count: 4, leader_last_seq: 4, stream_baseline: 0, + term: 0, + leader_region: 0, }; apply_drained(vec![payload], &ledger, &state, None, Some(&applier)).unwrap(); @@ -1867,6 +1882,8 @@ mod tests { event_count: 1, leader_last_seq: 1, stream_baseline: 0, + term: 0, + leader_region: 0, }; let err = apply_drained(vec![payload], &ledger, &state, None, None) .expect_err("a blob with no applier must halt"); @@ -1901,6 +1918,8 @@ mod tests { event_count: 2, leader_last_seq: 12, stream_baseline: 10, + term: 0, + leader_region: 0, }; apply_drained(vec![payload], &ledger, &state, None, None).unwrap(); assert_eq!( diff --git a/tidal/src/replication/relay.rs b/tidal/src/replication/relay.rs index 4e315bb..7fabfd9 100644 --- a/tidal/src/replication/relay.rs +++ b/tidal/src/replication/relay.rs @@ -109,6 +109,37 @@ pub const fn range_payload( event_count, leader_last_seq: last_seq, stream_baseline: 0, + // Term-0 default: relay-path callers (in-process fabric, sim cluster) + // never run elections. The cluster ship queue stamps the real term + // (`range_payload_with_term`). + term: 0, + leader_region: 0, + } +} + +/// [`range_payload`] with the m11p4 term stamp. +/// +/// The cluster ship queue's payload shape: every batch carries the +/// leadership `(term, leader_region)` that shipped it, so followers can +/// fence a deposed leader's traffic. +#[must_use] +pub const fn range_payload_with_term( + source_shard: ShardId, + first_seq: u64, + last_seq: u64, + event_count: u64, + bytes: Vec, + term: u64, + leader_region: u16, +) -> WalSegmentPayload { + WalSegmentPayload { + id: WalSegmentId::new(RegionId::SINGLE, source_shard, first_seq), + bytes, + event_count, + leader_last_seq: last_seq, + stream_baseline: 0, + term, + leader_region, } } diff --git a/tidal/src/replication/ship.rs b/tidal/src/replication/ship.rs index a856468..170d9c3 100644 --- a/tidal/src/replication/ship.rs +++ b/tidal/src/replication/ship.rs @@ -68,7 +68,7 @@ use std::{ use super::{ commit::CommitIndex, - relay::{SignalRelay, encode_run, range_payload}, + relay::{SignalRelay, encode_run}, shard::ShardId, transport::{Transport, TransportError}, }; @@ -258,6 +258,9 @@ pub struct ShipQueueConfig { pub window: usize, /// Delay before a transiently-failed run is retried. pub retry_backoff: Duration, + /// This node's region id, stamped as `leader_region` on every outbound + /// payload (m11p4 fencing). 0 for relay-path/test users without elections. + pub leader_region: u16, } impl Default for ShipQueueConfig { @@ -267,6 +270,7 @@ impl Default for ShipQueueConfig { max_batch_bytes: 16 * 1024 * 1024, window: 4, retry_backoff: Duration::from_millis(100), + leader_region: 0, } } } @@ -324,6 +328,19 @@ struct PeerShip { acked_atomic: AtomicU64, } +/// The region every outbound batch names as its leader (m11p4). Lives in the +/// queue config so [`ShipQueue::spawn`]'s signature stays stable; defaults to +/// 0 for relay-path/test users that never run elections. +impl ShipQueueConfig { + /// This node's region id, stamped as `leader_region` on every shipped + /// payload alongside the term. + #[must_use] + pub const fn with_leader_region(mut self, region: u16) -> Self { + self.leader_region = region; + self + } +} + impl PeerShip { fn lock(&self) -> std::sync::MutexGuard<'_, PeerState> { self.state.lock().unwrap_or_else(PoisonError::into_inner) @@ -339,6 +356,10 @@ struct ShipShared { shutdown: AtomicBool, /// Leadership gate: only an active queue dispatches (see module docs). active: AtomicBool, + /// The leadership term every outbound batch is stamped with (m11p4 + /// fencing). 0 = the topology era; set by [`ShipQueue::activate_from`] + /// when this node leads an elected term. + term: AtomicU64, /// Quorum commit index over the peers' durable marks (m11p3). Follows /// the queue's leadership gate: activated/deactivated with it. commit: Arc, @@ -424,6 +445,9 @@ impl ShipQueue { config, shutdown: AtomicBool::new(false), active: AtomicBool::new(active), + // Boot is the topology era (term 0); an elected leadership stamps + // its term through activate_from. + term: AtomicU64::new(0), commit, #[cfg(feature = "metrics")] metrics, @@ -482,10 +506,12 @@ impl ShipQueue { /// Activate dispatch with a fresh stream baseline: every peer's cursor /// jumps to `baseline + 1`, parked retries clear, and acked frontiers - /// reset to the baseline. Called when this node becomes leader (promote): - /// `baseline` is its WAL flushed frontier at promotion — everything at or - /// below it is pre-stream history that must NOT push to peers. - pub fn activate_from(&self, baseline: u64) { + /// reset to the baseline. Called when this node becomes leader (promote + /// or election win): `baseline` is its WAL flushed frontier at promotion — + /// everything at or below it is pre-stream history that must NOT push to + /// peers. `term` is the leadership term every outbound batch is stamped + /// with and the commit index is scoped to (m11p4; 0 = the topology era). + pub fn activate_from(&self, baseline: u64, term: u64) { for cell in &self.shared.peers { { let mut state = cell.lock(); @@ -497,10 +523,11 @@ impl ShipQueue { } cell.acked_atomic.store(baseline, Ordering::Release); } - self.shared.commit.activate(baseline); + self.shared.term.store(term, Ordering::Release); + self.shared.commit.activate(baseline, term); self.shared.active.store(true, Ordering::Release); self.shared.wake_all(); - tracing::info!(baseline, "ship queue activated (leadership)"); + tracing::info!(baseline, term, "ship queue activated (leadership)"); } /// Deactivate dispatch (this node stopped leading). In-flight sends @@ -826,12 +853,14 @@ fn sender_loop(shared: &ShipShared, cell: &PeerShip) { while let Some(run) = claim_and_collect(shared, cell) { #[cfg(feature = "metrics")] let started = Instant::now(); - let payload = range_payload( + let payload = crate::replication::relay::range_payload_with_term( shared.source.source_shard(), run.first, run.last, run.event_count, run.bytes.clone(), + shared.term.load(Ordering::Acquire), + shared.config.leader_region, ); match shared.transport.send_segment(cell.peer, payload) { Ok(()) => { @@ -1057,6 +1086,7 @@ mod tests { max_batch_bytes: 1 << 20, window: 1, retry_backoff: Duration::from_millis(10), + leader_region: 0, }, ); @@ -1098,6 +1128,7 @@ mod tests { max_batch_bytes: 1 << 20, window: 4, retry_backoff: Duration::from_millis(20), + leader_region: 0, }, ); @@ -1202,6 +1233,7 @@ mod tests { max_batch_bytes: 1 << 20, window: 1, retry_backoff: Duration::from_millis(10), + leader_region: 0, }, ); @@ -1253,7 +1285,7 @@ mod tests { assert!(!queue.is_active()); // Promote with baseline 5: pre-promote history must NOT push. - queue.activate_from(5); + queue.activate_from(5, 1); write_n(&relay, &db, 3); // seqnos 6..=8 queue.publish(); assert!( @@ -1304,6 +1336,7 @@ mod tests { max_batch_bytes: 1 << 20, window: 1, retry_backoff: Duration::from_millis(10), + leader_region: 0, }, true, #[cfg(feature = "metrics")] diff --git a/tidal/src/replication/shipper.rs b/tidal/src/replication/shipper.rs index b94117b..a622c46 100644 --- a/tidal/src/replication/shipper.rs +++ b/tidal/src/replication/shipper.rs @@ -330,6 +330,8 @@ pub fn spawn_shipper( event_count, leader_last_seq: *leader_last_seq, stream_baseline: 0, + term: 0, + leader_region: 0, }; match transport.send_segment(peer, payload) { Ok(()) => { @@ -470,7 +472,8 @@ pub fn filter_segment_drop_local(bytes: &[u8]) -> Vec { let events = match payload { crate::wal::format::BatchPayload::Signals(events) => events, crate::wal::format::BatchPayload::ItemMetadata(_) - | crate::wal::format::BatchPayload::Embedding(_) => { + | crate::wal::format::BatchPayload::Embedding(_) + | crate::wal::format::BatchPayload::TermMarker(_) => { out.extend_from_slice(&remaining[..batch_size]); continue; } diff --git a/tidal/src/replication/transport.rs b/tidal/src/replication/transport.rs index 473f215..6f35ee7 100644 --- a/tidal/src/replication/transport.rs +++ b/tidal/src/replication/transport.rs @@ -38,6 +38,15 @@ pub struct WalSegmentPayload { /// the previous leader's stream), so the receiver jumps its frontier to /// the baseline instead of waiting for a gap that will never close. pub stream_baseline: u64, + /// The sender's leadership term (m11p4 fencing). `0` = the topology era + /// (no election has ever happened) or a pre-m11p4 sender; a receiver at + /// term ≥ 1 rejects term-0 traffic, and any receiver rejects a term below + /// its own — a deposed leader's ships can never apply. + pub term: u64, + /// The region claiming leadership of `term` (m11p4). Carried so a + /// receiver can attribute leader contact and detect the term-0 + /// conflicting-leader misdeployment. + pub leader_region: u16, } /// Errors that can occur during WAL segment transport. @@ -276,6 +285,8 @@ mod tests { event_count: 1, leader_last_seq: 42, stream_baseline: 0, + term: 0, + leader_region: 0, }; assert_eq!(payload.id.seqno, 42); assert_eq!(payload.bytes.len(), 3); diff --git a/tidal/src/wal/diagnostics.rs b/tidal/src/wal/diagnostics.rs index 45e7bb1..57d9c91 100644 --- a/tidal/src/wal/diagnostics.rs +++ b/tidal/src/wal/diagnostics.rs @@ -157,7 +157,8 @@ pub fn diagnose_wal(data_dir: &Path) -> Result { let record_count = match payload { crate::wal::format::BatchPayload::Signals(events) => events.len(), crate::wal::format::BatchPayload::ItemMetadata(_) - | crate::wal::format::BatchPayload::Embedding(_) => 1, + | crate::wal::format::BatchPayload::Embedding(_) + | crate::wal::format::BatchPayload::TermMarker(_) => 1, }; let n = record_count as u64; event_count += n; diff --git a/tidal/src/wal/format/batch.rs b/tidal/src/wal/format/batch.rs index 7f9cb58..3c4aac8 100644 --- a/tidal/src/wal/format/batch.rs +++ b/tidal/src/wal/format/batch.rs @@ -64,6 +64,17 @@ pub const BATCH_KIND_SIGNALS: u8 = 0; pub const BATCH_KIND_ITEM_METADATA: u8 = 1; /// Batch kind: one item-embedding blob record. pub const BATCH_KIND_EMBEDDING: u8 = 2; +/// Batch kind: one term-marker record (m11p4 leader election). +/// +/// Journaled by a leader as its FIRST log entry of an elected term (≥ 1) and +/// replicated like any record, so every replica's WAL carries the term of the +/// leadership that wrote its tail — the `(lastLogTerm, lastLogIndex)` half of +/// Raft's vote restriction, crash-atomic with the frontier by construction +/// (they live in the same fsync stream). Never written at term 0 (the +/// topology-file era), so a rolling m11p3→m11p4 upgrade ships zero unknown +/// record kinds until the first election — which itself requires a vote +/// quorum of upgraded binaries. +pub const BATCH_KIND_TERM_MARKER: u8 = 3; /// Size of the batch header in bytes (one cache line). pub const HEADER_SIZE: usize = 64; @@ -442,6 +453,17 @@ pub struct EmbeddingRecord { pub values: Vec, } +/// One term-marker record (batch kind 3, m11p4): the first log entry an +/// elected leader journals in its term. Folding it has no storage effect — +/// replicas record `(term, seq)` as their WAL-tail term. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TermMarkerRecord { + /// The elected term this marker opens (always ≥ 1). + pub term: u64, + /// The region that leads `term`. + pub leader_region: u16, +} + /// A decoded batch payload, by kind. #[derive(Debug, Clone, PartialEq)] pub enum BatchPayload { @@ -451,6 +473,8 @@ pub enum BatchPayload { ItemMetadata(ItemMetadataRecord), /// Kind 2: one item-embedding blob. Embedding(EmbeddingRecord), + /// Kind 3: one term marker (m11p4 election). + TermMarker(TermMarkerRecord), } /// One blob (kind-1/2) record submitted for a WAL append. @@ -464,6 +488,8 @@ pub enum BlobRecord { ItemMetadata(ItemMetadataRecord), /// An item-embedding mutation (batch kind 2). Embedding(EmbeddingRecord), + /// A term marker (batch kind 3, m11p4): no entity, no storage effect. + TermMarker(TermMarkerRecord), } impl BlobRecord { @@ -473,15 +499,17 @@ impl BlobRecord { match self { Self::ItemMetadata(_) => BATCH_KIND_ITEM_METADATA, Self::Embedding(_) => BATCH_KIND_EMBEDDING, + Self::TermMarker(_) => BATCH_KIND_TERM_MARKER, } } - /// The entity this mutation targets. + /// The entity this mutation targets (0 for a term marker — it has none). #[must_use] pub const fn entity_id(&self) -> u64 { match self { Self::ItemMetadata(r) => r.entity_id, Self::Embedding(r) => r.entity_id, + Self::TermMarker(_) => 0, } } @@ -503,6 +531,7 @@ impl BlobRecord { encode_item_metadata_batch(r, seq, batch_ts, shard_id, region_id) } Self::Embedding(r) => encode_embedding_batch(r, seq, batch_ts, shard_id, region_id), + Self::TermMarker(r) => encode_term_marker_batch(r, seq, batch_ts, shard_id, region_id), } } } @@ -567,6 +596,34 @@ pub fn encode_embedding_batch( ) } +/// Encode a kind-3 term-marker batch (one record, one seqno): +/// `term (u64 LE) | leader_region (u16 LE)`. +/// +/// # Errors +/// +/// Returns `WalError::Corruption` only through the shared frame encoder +/// (the 10-byte payload can never exceed the blob ceiling). +pub fn encode_term_marker_batch( + record: &TermMarkerRecord, + seq: u64, + batch_ts: u64, + shard_id: ShardId, + region_id: RegionId, +) -> Result, WalError> { + let mut payload = Vec::with_capacity(10); + payload.extend_from_slice(&record.term.to_le_bytes()); + payload.extend_from_slice(&record.leader_region.to_le_bytes()); + encode_frame( + BATCH_KIND_TERM_MARKER, + 1, + &payload, + seq, + batch_ts, + shard_id, + region_id, + ) +} + /// Shared frame encoder for every batch kind: 64-byte header + payload, BLAKE3 /// over `header[0..32] || payload`. The single owner of the header layout — /// every kind goes through here so the framing cannot drift. @@ -628,7 +685,9 @@ pub fn decode_batch(bytes: &[u8]) -> Result<(BatchHeader, Vec), Wal let (header, payload) = decode_batch_payload(bytes)?; match payload { BatchPayload::Signals(events) => Ok((header, events)), - BatchPayload::ItemMetadata(_) | BatchPayload::Embedding(_) => Err(WalError::Corruption { + BatchPayload::ItemMetadata(_) + | BatchPayload::Embedding(_) + | BatchPayload::TermMarker(_) => Err(WalError::Corruption { message: format!( "kind-{} batch decoded through the signals-only decode_batch; \ use decode_batch_payload", @@ -683,6 +742,7 @@ pub fn decode_batch_payload(bytes: &[u8]) -> Result<(BatchHeader, BatchPayload), if flags != BATCH_KIND_SIGNALS && flags != BATCH_KIND_ITEM_METADATA && flags != BATCH_KIND_EMBEDDING + && flags != BATCH_KIND_TERM_MARKER { return Err(WalError::Corruption { message: format!("unsupported batch kind: {flags}"), @@ -787,6 +847,7 @@ pub fn decode_batch_payload(bytes: &[u8]) -> Result<(BatchHeader, BatchPayload), BatchPayload::Signals(events) } BATCH_KIND_ITEM_METADATA => BatchPayload::ItemMetadata(decode_item_blob(payload_bytes)?), + BATCH_KIND_TERM_MARKER => BatchPayload::TermMarker(decode_term_marker_blob(payload_bytes)?), // The kind set was validated in phase 1; only embedding remains. _ => BatchPayload::Embedding(decode_embedding_blob(payload_bytes)?), }; @@ -826,6 +887,39 @@ fn decode_item_blob(payload: &[u8]) -> Result { }) } +/// Parse a kind-3 blob: `term (u64 LE) | leader_region (u16 LE)`. +fn decode_term_marker_blob(payload: &[u8]) -> Result { + if payload.len() != 10 { + return Err(WalError::Corruption { + message: format!( + "term-marker blob length {} != 10 (term u64 + leader_region u16)", + payload.len() + ), + }); + } + let term = u64::from_le_bytes(payload[0..8].try_into().map_err(|_| WalError::Corruption { + message: "term-marker blob: invalid term bytes".into(), + })?); + let leader_region = + u16::from_le_bytes( + payload[8..10] + .try_into() + .map_err(|_| WalError::Corruption { + message: "term-marker blob: invalid leader_region bytes".into(), + })?, + ); + if term == 0 { + return Err(WalError::Corruption { + message: "term-marker blob: term 0 is the topology era and never journals a marker" + .into(), + }); + } + Ok(TermMarkerRecord { + term, + leader_region, + }) +} + /// Parse a kind-2 blob: `entity_id (u64 LE) | dim (u32 LE) | dim × f32 LE`. fn decode_embedding_blob(payload: &[u8]) -> Result { if payload.len() < 12 { diff --git a/tidal/src/wal/format/mod.rs b/tidal/src/wal/format/mod.rs index 191028b..20d0e7b 100644 --- a/tidal/src/wal/format/mod.rs +++ b/tidal/src/wal/format/mod.rs @@ -14,13 +14,13 @@ pub mod session; // so that existing `use crate::wal::format::{...}` paths continue to resolve. pub use batch::{ - BATCH_KIND_EMBEDDING, BATCH_KIND_ITEM_METADATA, BATCH_KIND_SIGNALS, BatchHeader, BatchPayload, - BlobRecord, EVENT_SIZE, EVENT_SIZE_V3, EmbeddingRecord, EventRecord, FORMAT_VERSION, - FORMAT_VERSION_V1, FORMAT_VERSION_V2, FORMAT_VERSION_V3, HEADER_PAYLOAD_LEN_OFFSET, - HEADER_SIZE, ItemMetadataRecord, MAGIC, MAX_BLOB_PAYLOAD_BYTES, MAX_EVENTS_PER_BATCH, - RECORD_TYPE_SIGNAL, decode_batch, decode_batch_payload, encode_batch, encode_batch_with_shard, - encode_embedding_batch, encode_item_metadata_batch, event_content_hash, event_size_for_version, - payload_len_from_header, + BATCH_KIND_EMBEDDING, BATCH_KIND_ITEM_METADATA, BATCH_KIND_SIGNALS, BATCH_KIND_TERM_MARKER, + BatchHeader, BatchPayload, BlobRecord, EVENT_SIZE, EVENT_SIZE_V3, EmbeddingRecord, EventRecord, + FORMAT_VERSION, FORMAT_VERSION_V1, FORMAT_VERSION_V2, FORMAT_VERSION_V3, + HEADER_PAYLOAD_LEN_OFFSET, HEADER_SIZE, ItemMetadataRecord, MAGIC, MAX_BLOB_PAYLOAD_BYTES, + MAX_EVENTS_PER_BATCH, RECORD_TYPE_SIGNAL, TermMarkerRecord, decode_batch, decode_batch_payload, + encode_batch, encode_batch_with_shard, encode_embedding_batch, encode_item_metadata_batch, + encode_term_marker_batch, event_content_hash, event_size_for_version, payload_len_from_header, }; pub use session::{ SESSION_RECORD_CLOSE, SESSION_RECORD_SIGNAL, SESSION_RECORD_START, SESSION_RECORD_VERSION_V2, diff --git a/tidal/src/wal/mod.rs b/tidal/src/wal/mod.rs index 8453636..2ba40d7 100644 --- a/tidal/src/wal/mod.rs +++ b/tidal/src/wal/mod.rs @@ -113,6 +113,51 @@ pub(crate) fn sync_dir_durable(dir: &std::path::Path) -> Result<(), WalError> { } } +/// The WAL-tail term: the highest term marker (m11p4, kind-3 record) this +/// node's log carries, with the seqno it occupies. +/// +/// One cell per node (the WAL is one log). Fed by recovery (the scan finds +/// the last marker), by a leader journaling its term's marker, and by a +/// follower applying a replicated one. Read by the election machinery as the +/// `lastLogTerm` half of Raft's up-to-date restriction — atomic with the +/// frontier by construction, because both live in the same fsync stream. +/// `(0, 0)` = a pre-election log (the topology era never writes markers). +#[derive(Debug, Default)] +pub struct WalTermMark { + /// `(term, seq, leader_region)` under ONE lock: the three fields are a + /// single logical value — the election machinery picks the STREAM whose + /// numbering the frontier is read in from `leader_region`, so a torn + /// read (new term with the previous marker's region) would compare + /// frontiers across numberings. Updates happen once per term and reads + /// a few times per second; a mutex is the correct tool, not split + /// atomics. + inner: std::sync::Mutex<(u64, u64, u16)>, +} + +impl WalTermMark { + /// Fold a marker, monotonically by term (a re-applied or recovered older + /// marker never regresses the mark). + pub fn advance(&self, term: u64, seq: u64, leader_region: u16) { + let mut inner = self + .inner + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if term > inner.0 { + *inner = (term, seq, leader_region); + } + } + + /// The current `(term, seq, leader_region)` mark. `(0, 0, 0)` = no + /// marker in the log (the topology era). + #[must_use] + pub fn snapshot(&self) -> (u64, u64, u16) { + *self + .inner + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + /// A signal event to be appended to the WAL. /// /// This is the public write type. It maps 1:1 to the internal diff --git a/tidal/src/wal/reader.rs b/tidal/src/wal/reader.rs index 4b59c52..b8d8dc4 100644 --- a/tidal/src/wal/reader.rs +++ b/tidal/src/wal/reader.rs @@ -226,6 +226,18 @@ pub fn recover(dir: &Path) -> Result { } expected_first_seq = Some(candidate); } + BatchPayload::TermMarker(record) => { + let seq = header.first_seq; + all_blobs.push(RecoveredBlob { + seq, + record: BlobRecord::TermMarker(record), + }); + let candidate = seq.saturating_add(1); + if candidate > next_seq { + next_seq = candidate; + } + expected_first_seq = Some(candidate); + } } } } diff --git a/tidal/tests/m8p10_reconcile_idempotence.rs b/tidal/tests/m8p10_reconcile_idempotence.rs index 87118b9..eb4ad4b 100644 --- a/tidal/tests/m8p10_reconcile_idempotence.rs +++ b/tidal/tests/m8p10_reconcile_idempotence.rs @@ -151,6 +151,8 @@ fn reconcile_of_converged_nodes_is_a_fixpoint() { event_count: 1, leader_last_seq: 1, stream_baseline: 0, + term: 0, + leader_region: 0, }) .unwrap(); wait_for_applied(&follower_state, ShardId::SINGLE, 1); diff --git a/tidal/tests/m8p2_replication.rs b/tidal/tests/m8p2_replication.rs index 76ca918..d4c43c2 100644 --- a/tidal/tests/m8p2_replication.rs +++ b/tidal/tests/m8p2_replication.rs @@ -223,6 +223,8 @@ fn payload_injection_updates_follower_ledger() { event_count: 1, leader_last_seq: 1, stream_baseline: 0, + term: 0, + leader_region: 0, }) .unwrap(); @@ -281,6 +283,8 @@ fn replay_is_idempotent() { event_count: 1, leader_last_seq: 1, stream_baseline: 0, + term: 0, + leader_region: 0, }) .unwrap(); } @@ -325,6 +329,8 @@ fn in_process_transport_delivers_segment() { // first_seq=1, 1 event → last WAL seq = 1 (the file seqno 42 differs). leader_last_seq: 1, stream_baseline: 0, + term: 0, + leader_region: 0, }, ) .unwrap(); @@ -404,6 +410,8 @@ fn full_pipeline_leader_to_follower() { // first_seq=1, 2 events → last WAL seq = 2. leader_last_seq: 2, stream_baseline: 0, + term: 0, + leader_region: 0, }) .unwrap(); @@ -512,6 +520,8 @@ fn replication_decay_scores_match() { // Every batch is first_seq=1 with 200 events → last WAL seq = 200. leader_last_seq: 1 + chunk.len() as u64 - 1, stream_baseline: 0, + term: 0, + leader_region: 0, }) .unwrap(); } @@ -582,6 +592,8 @@ fn follower_serves_retrieve_queries() { event_count: 2, leader_last_seq: 2, stream_baseline: 0, + term: 0, + leader_region: 0, }) .unwrap(); @@ -643,6 +655,8 @@ fn corrupted_segment_is_rejected() { event_count: 1, leader_last_seq: 1, stream_baseline: 0, + term: 0, + leader_region: 0, }) .unwrap(); @@ -664,6 +678,8 @@ fn corrupted_segment_is_rejected() { // encode_batch(events, first_seq=1, ts=2) → 1 event → last WAL seq = 1. leader_last_seq: 1, stream_baseline: 0, + term: 0, + leader_region: 0, }) .unwrap(); @@ -746,6 +762,8 @@ fn with_transport_auto_wires_follower_receiver() { event_count: 1, leader_last_seq: 1, stream_baseline: 0, + term: 0, + leader_region: 0, }) .unwrap(); diff --git a/tidal/tests/m8p2_replication_durability.rs b/tidal/tests/m8p2_replication_durability.rs index 7e3e1a4..dcdda92 100644 --- a/tidal/tests/m8p2_replication_durability.rs +++ b/tidal/tests/m8p2_replication_durability.rs @@ -130,6 +130,8 @@ fn build_segment( // Last WAL seq = first_seq + N - 1 (matches the doc comment above). leader_last_seq: first_seq + entities.len() as u64 - 1, stream_baseline: 0, + term: 0, + leader_region: 0, } }