diff --git a/tidal-server/src/cluster/node.rs b/tidal-server/src/cluster/node.rs index d15c2f7..bd625db 100644 --- a/tidal-server/src/cluster/node.rs +++ b/tidal-server/src/cluster/node.rs @@ -6050,6 +6050,7 @@ pub struct AggregatedRegionStatus { #[allow(clippy::significant_drop_tightening)] pub async fn cluster_status( State(node): State>, + headers: HeaderMap, ) -> std::result::Result, ClusterAppError> { let state = node.replica_for(None)?; let leader_name = state.leader_name(); @@ -6059,9 +6060,24 @@ pub async fn cluster_status( // in-process (no HTTP hop); peers over the async client with the per-peer // budget. Each entry: (RegionId, name, Option). let client = state.client.clone(); + // A relayed operator hop: pass the caller's credential straight through, the + // same way `/cluster/promote` does. Without it every peer answers 401 and + // this aggregate reported `reachable: false`, `partitioned: true`, + // `applied_events: null` and `version: ""` for EVERY peer on an + // authenticated cluster — a total-partition reading taken off a perfectly + // healthy fleet, from the one surface an operator reads to clear the N/N+1 + // version skew before a rolling upgrade (and the runbook's own deploy + // step 4). `security::bearer_from_env` documents this exact trap: "node then + // dials an authenticated peer with NO credential". + // + // FORWARDING the caller's header, rather than reaching for + // `creds.bearer()`, keeps a weakly-authenticated caller from borrowing this + // node's own credential to read peers it could not read directly. + let auth = forwarded_auth(&headers); let own_status = state.local_status().ok(); let futures = regions.into_iter().map(|(rid, name, http)| { let client = client.clone(); + let auth = auth.clone(); let own_status = if rid == state.region { own_status.clone() } else { @@ -6082,12 +6098,11 @@ pub async fn cluster_status( return (rid, name, None); // unreachable: no declared addr }; let url = peer_url(&http_addr, "/cluster/status/local"); - let body = client - .get(&url) - .timeout(forward::STATUS_PEER_TIMEOUT) - .send() - .await - .ok(); + let mut req = client.get(&url).timeout(forward::STATUS_PEER_TIMEOUT); + if let Some(auth) = auth { + req = req.header(axum::http::header::AUTHORIZATION, auth); + } + let body = req.send().await.ok(); let json = match body { Some(resp) if resp.status().is_success() => { resp.json::().await.ok() diff --git a/tidal-server/tests/cluster_runbook.rs b/tidal-server/tests/cluster_runbook.rs index d33467b..cc19fcf 100644 --- a/tidal-server/tests/cluster_runbook.rs +++ b/tidal-server/tests/cluster_runbook.rs @@ -1145,6 +1145,89 @@ fn runbook_auth_protected_routes_401_probes_open() { println!("[auth] protected routes 401 bare, probes + /openapi.json open, valid bearer 204"); } +/// The runbook's deploy step 4 — `GET /cluster/status | jq '.regions[] | {name, +/// lag_events, reachable}'` — must tell the TRUTH on an authenticated cluster. +/// +/// The aggregate probes every peer's `/cluster/status/local` over HTTP, and that +/// route is token-gated (asserted by +/// `runbook_auth_protected_routes_401_probes_open` above). A probe carrying no +/// credential therefore reads 401 from every peer, and each peer row degrades to +/// the "honest unknown": `reachable: false`, `partitioned: true`, +/// `applied_events: null`, `version: ""`. The result is a TOTAL-PARTITION +/// reading taken off a perfectly healthy fleet, on the one surface an operator +/// reads to clear the N/N+1 version skew before a rolling upgrade. +/// `security::bearer_from_env` documents this exact trap: a node that "dials an +/// authenticated peer with NO credential". +/// +/// Every OTHER multi-process test — including `cluster_multiproc`'s +/// all-reachable assertion — runs with no `TIDAL_API_KEY`, where a +/// credential-less probe succeeds. That is precisely why this was invisible for +/// so long, so this test carries the key. +#[test] +fn runbook_cluster_status_aggregate_reaches_peers_under_auth() { + const KEY: &str = "runbook-status-aggregate-key"; + let opts = (0..3).fold(ClusterOptions::new(3), |opts, i| { + opts.with_env(i, "TIDAL_API_KEY", KEY) + }); + let cluster = MultiProcCluster::start_with(opts); + + let status_url = format!("{}/cluster/status", cluster.node(LEADER)); + let fetch = || -> serde_json::Value { + let resp = cluster + .client() + .get(&status_url) + .header("Authorization", format!("Bearer {KEY}")) + .send() + .expect("GET /cluster/status"); + assert_eq!( + resp.status().as_u16(), + 200, + "an operator read of /cluster/status must still serve with the bearer" + ); + resp.json().expect("/cluster/status body is JSON") + }; + + // The fan-out gets the convergence budget to report every peer. + let deadline = Instant::now() + convergence_budget(); + let settled = loop { + let snap = fetch(); + let all_reachable = snap["regions"].as_array().is_some_and(|rows| { + rows.len() == 3 && rows.iter().all(|r| r["reachable"].as_bool() == Some(true)) + }); + if all_reachable { + break snap; + } + assert!( + Instant::now() < deadline, + "/cluster/status never reported all 3 regions reachable on an \ + AUTHENTICATED cluster — a peer probe that carries no credential \ + reads its 401 as `unreachable`. Last snapshot: {snap}" + ); + std::thread::sleep(Duration::from_millis(250)); + }; + + // Each peer row must carry REAL values, never the unreachable placeholder. + for row in settled["regions"].as_array().expect("regions array") { + let name = row["name"].as_str().unwrap_or(""); + assert_eq!( + row["partitioned"].as_bool(), + Some(false), + "{name}: a reachable region is not partitioned" + ); + assert!( + !row["applied_events"].is_null(), + "{name}: applied_events must be a known number, not the \ + unreachable-peer null" + ); + assert!( + !row["version"].as_str().unwrap_or_default().is_empty(), + "{name}: must report its build version — the pre-upgrade N/N+1 skew \ + check reads this field, and an unreachable peer reports it empty" + ); + } + println!("[auth] /cluster/status aggregated all 3 regions through the forwarded bearer"); +} + // ── Auth: the admin key separates operator authority from data-plane access ──── /// With BOTH `TIDAL_API_KEY` and `TIDAL_ADMIN_KEY` set, a data-plane bearer is