From fab5467b8f0b15379c41ef4635aade1ce84aaa8a Mon Sep 17 00:00:00 2001 From: jordan Date: Fri, 21 Aug 2026 02:15:35 -0600 Subject: [PATCH] test(cluster): reproduce the multi-group reseed silent hole The served-evidence marker fix (afdda7c) closes the SINGLE-group case, proven by mp_follower_reseeds_via_snapshot_after_compaction passing with its content probe. It does not close the multi-group case, and nothing in the suite covered that: the one reseed gate was single-group, and the harness leaves reseed_self_restart at false, so a per-group self-restart that never reaches a fixpoint was invisible. New mp_multi_group_node_converges_after_reseeding_several_groups reproduces the production shape from k8s/cluster/topology-configmap.yaml: 3 nodes x 3 groups, full placement, production election timers, reseed_self_restart TRUE. It stops one node so its group leadership moves and a survivor ends up leading two groups (the live tidaldb-1 arrangement), writes past WAL_RETENTION_SEGMENTS, gracefully restarts the survivors to compact, then brings the node back. The test also stands in for the ORCHESTRATOR. reseed_self_restart drains and exits(0) expecting a reboot; the harness has no supervisor and `is_alive` only checks that the handle is retained, so an exited node just stays down. Sustained HTTP unreachability is the exit signal and `restart` is the reboot, counted against a finite ceiling. The content probe stays supervised too, because the first run settled, then re-latched and exited, and an unsupervised probe merely panicked on a connection error and hid it. Observed failure, the local twin of the production incident: [multi] node 2 settled after 0 orchestrator restart(s) [multi] node 2 exited AFTER settling; orchestrator reboot #1 missing item 500 (reboots=1) ... reseed_required: false, lag_events: 0, applied_events: 3798, election_tail_term: 2 The node reports no marker and zero lag while an item written before its outage is absent. That is the same silent hole tidaldb-0 showed at lag_events: 0. Marked #[ignore] with the reason and the invocation, so the nightly chaos gate keeps its signal instead of going permanently red on a known-open defect. Removing the attribute is the gate for the fix. Also adds write_heavy_item_retrying: which survivor inherits a stopped node's groups varies per run, so a write may be local for one group and a cross-group forward for another, and a forward inside an election window legitimately answers a retryable 503. Retrying keeps the fixture deterministic without masking a hard failure. --- tidal-server/tests/cluster_reseed.rs | 281 +++++++++++++++++++++++++++ 1 file changed, 281 insertions(+) diff --git a/tidal-server/tests/cluster_reseed.rs b/tidal-server/tests/cluster_reseed.rs index 085fc3f..362ca95 100644 --- a/tidal-server/tests/cluster_reseed.rs +++ b/tidal-server/tests/cluster_reseed.rs @@ -190,6 +190,57 @@ fn write_heavy_item( assert_eq!(resp.status().as_u16(), 204, "leader /signals must 204"); } +/// [`write_heavy_item`] with a bounded retry, for the multi-group fixture. +/// +/// Which survivor inherits a stopped node's groups varies per run, so a write +/// issued at any one gateway may be local for one group and a cross-group FORWARD +/// for another, and a forward issued inside an election window legitimately answers +/// a retryable 503. Retrying keeps the FIXTURE deterministic without masking a hard +/// failure: after the budget it still fails, and it reports the last status. +fn write_heavy_item_retrying( + cluster: &MultiProcCluster, + idx: usize, + entity_id: u64, + blob: &str, + heavy: bool, +) { + let deadline = Instant::now() + Duration::from_secs(30); + loop { + let mut metadata = serde_json::Map::new(); + metadata.insert("title".into(), item_token(entity_id).into()); + if heavy { + for k in 0..BLOB_KEYS { + metadata.insert(format!("blob{k}"), blob.into()); + } + } + let status = cluster + .post( + idx, + "/items", + &serde_json::json!({ "entity_id": entity_id, "metadata": metadata }), + ) + .status() + .as_u16(); + if status == 201 { + // Embedding + signal are best-effort here: the gate is WAL volume on the + // groups, which the item blob already provides. + let v = entity_id as f32; + let _ = cluster.post( + idx, + "/embeddings", + &serde_json::json!({ "entity_id": entity_id, "values": [v, v + 1.0, v + 2.0, v + 3.0] }), + ); + return; + } + assert!( + Instant::now() < deadline, + "item {entity_id} never accepted (last status {status}); a cross-group forward is \ + failing beyond the election window, which is a different defect than this gate covers" + ); + std::thread::sleep(Duration::from_millis(250)); + } +} + /// Whether `/search?query=` on node `idx` returns `entity_id` (an exact /// item-presence probe — the content invariant). fn item_searchable(cluster: &MultiProcCluster, idx: usize, entity_id: u64) -> bool { @@ -555,6 +606,236 @@ fn mp_follower_reseeds_via_snapshot_after_compaction() { println!("[reseed] ap-south reseeded clean: reseed_required=false, quarantined=false"); } +/// The PRODUCTION shape (2026-08-20 incident): a node that hosts SEVERAL shard +/// groups and must reseed MORE THAN ONE of them must converge — not heal one +/// group per process restart forever. +/// +/// Live tidaldb-0 restarted 8 times in 22 minutes on the served-evidence fix with +/// the exiting group alternating 2 -> 1 -> 2. Each restart streamed a fresh +/// snapshot of a 33k x 1536-dim corpus off the healthy leaders, so the loop is +/// expensive as well as non-terminating. Nothing in the suite covered a +/// multi-group reseed: `mp_follower_reseeds_via_snapshot_after_compaction` is +/// single-group, so a per-group `reseed_self_restart` that never reaches a +/// fixpoint was invisible. +/// +/// Shape, matching `k8s/cluster/topology-configmap.yaml`: 3 nodes x 3 groups, +/// full placement, balanced term-0 leaders (group `s` led by node `s`), and the +/// PRODUCTION election timers. Node 2 goes down, so its group-2 leadership moves +/// and one surviving node ends up leading two groups — the live arrangement. +/// +/// The gate is a FIXPOINT: node 2 must end Ready, reporting no reseed marker, with +/// content readable from every group, within a bounded number of restarts. +/// +/// # CURRENTLY REPRODUCES AN OPEN DEFECT — `#[ignore]`d, not broken +/// +/// This test FAILS today, on purpose: it is the reproduction for a real bug that +/// the served-evidence marker fix does NOT close. Observed run: +/// +/// ```text +/// [multi] node 2 settled after 0 orchestrator restart(s) +/// [multi] node 2 exited AFTER settling; orchestrator reboot #1 +/// missing item 500 (reboots=1) ... reseed_required: false, lag_events: 0, +/// applied_events: 3798, election_tail_term: 2 +/// ``` +/// +/// A multi-group node reports NO reseed marker and ZERO lag while an item written +/// before its outage is absent — the silent-hole shape, and the local twin of the +/// 2026-08-20 production failure where tidaldb-0 reported `lag_events: 0` on a +/// replica missing history. Single-group reseed is correct +/// (`mp_follower_reseeds_via_snapshot_after_compaction` passes with the same content +/// probe); the multi-group path is not. +/// +/// It is `#[ignore]`d so the nightly chaos gate keeps its signal rather than going +/// permanently red on a known-open defect. Remove the attribute as the fix's gate: +/// +/// ```bash +/// TIDAL_TEST_BOOT_BUDGET_SECS=300 TIDAL_TEST_CONVERGENCE_BUDGET_SECS=180 \ +/// cargo test -p tidal-server --features cluster-e2e --test cluster_reseed \ +/// mp_multi_group_node_converges_after_reseeding_several_groups -- --nocapture +/// ``` +#[test] +#[ignore = "reproduces an OPEN multi-group reseed defect: the node reports \ + reseed_required=false and lag_events=0 while missing items. Un-ignore as the \ + fix's gate; see the doc comment for the invocation."] +fn mp_multi_group_node_converges_after_reseeding_several_groups() { + const GROUPS: usize = 3; + /// Generous but FINITE. One restart per group that needs a reseed is the + /// designed cost (the exit is process-wide); anything beyond that is the + /// non-terminating loop this gate exists to catch. + const RESTART_CEILING: u32 = GROUPS as u32 + 2; + + // `reseed_self_restart: true` is what PRODUCTION sets + // (k8s/cluster/topology-configmap.yaml) and what makes the loop possible: a + // latched marker drains and exits(0) so the orchestrator's next boot installs. + // The harness default is false, which is precisely why no existing test could + // ever observe a non-terminating multi-group reseed. + let topology = format!("{PROD_ELECTION_YAML}\nreplication:\n reseed_self_restart: true"); + let mut cluster = MultiProcCluster::start_sharded(3, GROUPS, Some(&topology)); + let leaders = cluster.wait_shard_leaders_agreed(convergence_budget()); + println!("[multi] initial group leaders: {leaders:?}"); + + // Baseline across every group. Entity ids are hash-routed, so a spread of ids + // lands writes in all three groups. + for entity in 1..=30u64 { + write_heavy_item_retrying(&cluster, LEADER, entity, "", false); + } + cluster.wait_converged_all(convergence_budget()); + println!("[multi] baseline converged on all three nodes"); + + // Node 2 leaves. Its group-2 leadership moves, so a survivor now leads two + // groups — exactly the live tidaldb-1 arrangement. + // + // `wait_shard_leaders_agreed` only requires the LIVE nodes to AGREE, and they + // agree on the dead node until the election timeout elapses. Writes are + // hash-routed, so until group 2's leadership actually moves, every entity + // landing in that group has no live leader and `/items` is not a 201. Wait for + // the handoff itself, which is also the arrangement being reproduced. + cluster.stop_graceful(AP_SOUTH); + let stopped = cluster.region_name(AP_SOUTH).to_string(); + let handoff_deadline = Instant::now() + convergence_budget(); + let after = loop { + let map = cluster.wait_shard_leaders_agreed(convergence_budget()); + if map.values().all(|l| *l != stopped) { + break map; + } + assert!( + Instant::now() < handoff_deadline, + "group leadership never vacated the stopped node {stopped}: {map:?}" + ); + std::thread::sleep(Duration::from_millis(250)); + }; + assert!( + after + .values() + .any(|l| after.values().filter(|x| *x == l).count() > 1), + "expected a survivor to lead TWO groups after the handoff (the live arrangement), \ + got {after:?}" + ); + println!("[multi] node 2 down; group leaders now {after:?} (a survivor leads two groups)"); + + // Enough heavy writes to rotate past WAL_RETENTION_SEGMENTS on the groups the + // survivors lead, so node 2's resume points are genuinely compacted away. + let blob = blob_value(); + for entity in 31..=(30 + OFFLINE_ITEMS) { + write_heavy_item_retrying(&cluster, LEADER, entity, &blob, true); + } + println!("[multi] wrote {OFFLINE_ITEMS} heavy items while node 2 was down"); + + // Graceful restarts of the live nodes run the clean-shutdown compaction. + cluster.restart_graceful(LEADER, &[]); + cluster.restart_graceful(EU_WEST, &[]); + let post = cluster.wait_shard_leaders_agreed(convergence_budget()); + println!("[multi] survivors gracefully restarted (WAL compacted); leaders {post:?}"); + + // Node 2 returns needing a reseed on more than one group. + cluster.restart(AP_SOUTH, &[]); + + // FIXPOINT, with the test standing in for the ORCHESTRATOR. + // + // `reseed_self_restart` drains and exits(0) expecting something to boot the + // process again — in production that is the kubelet's restart policy. The + // harness has no supervisor: an exited node simply stays down and + // `is_alive` still reports true (it only checks that the handle is retained). + // So this loop IS the supervisor: sustained HTTP unreachability is the exit + // signal, and `restart` (which sigkills idempotently first) is the reboot. + // Counting those reboots is the gate — one per group needing a reseed is the + // designed cost; unbounded is the live loop. + let deadline = Instant::now() + convergence_budget() + Duration::from_secs(240); + let mut restarts: u32 = 0; + let mut unreachable_polls = 0u32; + loop { + match cluster.local_status(AP_SOUTH) { + Some(s) => { + unreachable_polls = 0; + if s["reseed_required"].as_bool() == Some(false) + && s["reseeding"].as_bool() == Some(false) + && s["quarantined"].as_bool() == Some(false) + { + println!("[multi] node 2 settled after {restarts} orchestrator restart(s)"); + break; + } + } + None => { + unreachable_polls += 1; + // ~2s unreachable = it drained and exited (a self-restart), not a + // momentary blip. Reboot it exactly as the kubelet would. + if unreachable_polls >= 4 { + unreachable_polls = 0; + restarts += 1; + assert!( + restarts <= RESTART_CEILING, + "node 2 has been rebooted {restarts} times (ceiling {RESTART_CEILING}) \ + without settling: a multi-group node is healing at most one group per \ + boot and never reaching a fixpoint. This is the 2026-08-20 production \ + loop, where the exiting group alternated 2 -> 1 -> 2 across 8 restarts \ + in 22 minutes while re-streaming a full snapshot each cycle." + ); + println!( + "[multi] node 2 exited (self-restart); orchestrator reboot #{restarts}" + ); + cluster.restart(AP_SOUTH, &[]); + } + } + } + assert!( + Instant::now() < deadline, + "node 2 never settled; reboots={restarts}, last status: {:?}", + cluster.local_status(AP_SOUTH) + ); + std::thread::sleep(Duration::from_millis(500)); + } + + // CONTENT across every group: a settled marker means nothing if the log has a + // hole. Probe ids spanning the hash space so all three groups are exercised. + // + // STILL SUPERVISED. A node can settle and then re-latch and exit again — the + // first run of this fixture did exactly that, and an unsupervised probe just + // panicked on a connection error, hiding the interesting behaviour. Keep + // rebooting against the SAME ceiling here, so a node that only appears to + // converge is still caught, and use `local_status` (an Option) rather than a + // bare GET so unreachability is data instead of a panic. + let probe_deadline = Instant::now() + convergence_budget() + Duration::from_secs(120); + for entity in [31u64, 100, 500, 1500, 30 + OFFLINE_ITEMS] { + loop { + if cluster.local_status(AP_SOUTH).is_some() { + unreachable_polls = 0; + if item_searchable(&cluster, AP_SOUTH, entity) { + break; + } + } else { + unreachable_polls += 1; + if unreachable_polls >= 4 { + unreachable_polls = 0; + restarts += 1; + assert!( + restarts <= RESTART_CEILING, + "node 2 settled and then RE-LATCHED and exited again, reboot {restarts} \ + of ceiling {RESTART_CEILING}: convergence was not a fixpoint. This is \ + the production shape — apparent convergence followed by another \ + self-restart cycle." + ); + println!( + "[multi] node 2 exited AFTER settling; orchestrator reboot #{restarts}" + ); + cluster.restart(AP_SOUTH, &[]); + } + } + assert!( + Instant::now() < probe_deadline, + "reseeded multi-group node is missing item {entity} (reboots={restarts}): the \ + groups settled but the data did not arrive, which is the silent-hole shape this \ + suite must never pass. status: {:?}", + cluster.local_status(AP_SOUTH) + ); + std::thread::sleep(Duration::from_millis(250)); + } + } + println!( + "[multi] every probed item is readable on the reseeded multi-group node \ + ({restarts} reboot(s) total)" + ); +} + /// POST with the `x-tidal-ack` header through a dedicated client. `Some(seq)` /// only for a 2xx carrying `x-tidal-seq` (the ledger's "acknowledged"). fn post_acked(