//! HTTP routes for the single-process cluster ([`ClusterState`]): the data //! surface (items/embeddings/signals/feed/search), the cluster-management //! surface (status/promote/partition/heal), and the sharded scatter-gather //! routes. The multi-process region node's routes live in `cluster::node`. use std::{sync::Arc, time::Duration}; use axum::{ Json, Router, extract::{Query, Request, State}, http::{HeaderMap, StatusCode}, middleware::{self, Next}, response::{IntoResponse, Response}, routing::{get, post}, }; use serde::{Deserialize, Serialize}; use tidaldb::{ query::{retrieve::Retrieve, search::Search}, schema::EntityId, }; use tower::{ServiceBuilder, limit::ConcurrencyLimitLayer}; use tower_http::timeout::TimeoutLayer; use utoipa::ToSchema; use super::{node::require_local_ack, state::ClusterState}; use crate::{ dto::{ EmbeddingRequest, FeedItem, FeedQuery, FeedResponse, ItemRequest, MAX_LIMIT, SearchItem, SearchQueryParams, SearchResponse, SignalRequest, default_limit, default_profile, feed_items, search_items, }, error::{Result, ServerError}, offload::offload_read, }; /// Build the cluster-mode router. /// /// Exposes the same data routes as standalone (items, embeddings, signals, /// feed, search) plus cluster management endpoints. /// /// Protected routes carry the same load-shedding middleware as the standalone /// router, consuming the shared [`crate::router::REQUEST_TIMEOUT_SECS`] / /// [`crate::router::MAX_CONCURRENCY`] / [`crate::router::BODY_LIMIT_BYTES`] /// constants: /// - [`TimeoutLayer`] — 408 for requests exceeding the shared timeout; /// - [`ConcurrencyLimitLayer`] — queues beyond the shared in-flight cap; /// - body limit — 413 for oversized bodies before any deserialization. /// /// Health/status probes (`public`) are intentionally outside this stack so they /// are never queued or timed out under saturation. pub fn build_cluster_router( state: Arc, creds: Arc, ) -> Router { let public = Router::new() .route("/health", get(cluster_health)) // Shared with the standalone router via [`crate::health`] so the two // modes can never advertise a different probe contract. .route("/health/startup", get(crate::health::health_startup)) .route("/health/live", get(crate::health::health_live)) // `/cluster/status` deliberately NOT here - see the region router: it // reports leader, membership and seqnos, so it is protected. // Cluster-superset OpenAPI document (adds the /cluster/* and /sharded/* // routes). Unauthenticated, like the probes — contract only, no data. .route("/openapi.json", get(crate::openapi::serve_cluster)) .with_state(Arc::clone(&state)); // The destructive operator verbs, gated on the cluster-admin credential when // one is configured. Same rationale as the multi-process region router: these // used to share the data-plane bearer, so any client key could promote, // partition, or heal. let admin_creds = Arc::clone(&creds); let admin = Router::new() .route("/cluster/promote", post(cluster_promote)) .route("/cluster/partition", post(cluster_partition)) .route("/cluster/heal", post(cluster_heal)) .layer(middleware::from_fn(move |req: Request, next: Next| { let creds = Arc::clone(&admin_creds); async move { if creds.admin_ok(req.headers()) { return next.run(req).await; } crate::router::admin_forbidden_response() } })) .with_state(Arc::clone(&state)); let protected = Router::new() .route("/items", post(create_item)) .route("/embeddings", post(write_embedding)) .route("/signals", post(write_signal)) .route("/feed", get(feed)) .route("/search", get(search)) .route("/cluster/status", get(cluster_status)) // Sharded (scatter-gather) routes. .route("/sharded/items", post(sharded_create_item)) .route("/sharded/embeddings", post(sharded_write_embedding)) .route("/sharded/signals", post(sharded_write_signal)) .route("/sharded/feed", get(sharded_feed)) .route("/sharded/search", get(sharded_search)) .with_state(state) .merge(admin) // Shared with the standalone router so the body cap can never drift // (raise one, forget the other). See [`crate::router::BODY_LIMIT_BYTES`]. .layer(axum::extract::DefaultBodyLimit::max( crate::router::BODY_LIMIT_BYTES, )); // Credentials read PER REQUEST from `creds` so a rotation takes effect with no // restart (m11p7). No bearer configured ⇒ pass through (open). The admin key // also authenticates here (it is a superset credential) so an operator // presenting it is not rejected before the admin gate runs. let protected = protected.layer(middleware::from_fn(move |req: Request, next: Next| { let creds = Arc::clone(&creds); async move { if !creds.authenticated(req.headers()) { return crate::router::unauthorized_response(req.headers()); } let principal = creds.principal(req.headers()); if let Err((retry_after_ms, limit)) = creds.check_rate(&principal) { return crate::router::too_many_requests(retry_after_ms, limit); } next.run(req).await } })); // Mirror the standalone router's load-shedding stack on the cluster // protected routes: a request-timeout (408) and a hard in-flight cap (429) // using the SAME constants as standalone so the two surfaces can never // advertise different overload behavior. Health probes (the `public` // routes) are deliberately left outside this stack so liveness/readiness are // never queued or timed out under saturation. let protected = protected.layer( ServiceBuilder::new() .layer(TimeoutLayer::with_status_code( StatusCode::REQUEST_TIMEOUT, Duration::from_secs(crate::router::REQUEST_TIMEOUT_SECS), )) .layer(ConcurrencyLimitLayer::new(crate::router::MAX_CONCURRENCY)), ); // m11p8: the same request-id + tracing stack as the standalone router — // assigns/echoes `x-request-id` and opens a per-request span. The cluster // routers previously skipped it; with it, a write forwarded to the leader // carries the originating gateway's id (see `cluster::forward`). crate::router::with_request_id_tracing(public.merge(protected)) } // ── Health ────────────────────────────────────────────────────────────────── // // The unconditional startup/live probes are shared with the standalone router // via [`crate::health`]; only the mode-specific readiness probe (`cluster_health`) // lives here. async fn cluster_health( State(state): State>, ) -> std::result::Result<(StatusCode, Json), ClusterAppError> { if state.is_shutting_down() { return Ok(( StatusCode::SERVICE_UNAVAILABLE, Json(serde_json::json!({ "ok": false, "service": "tidaldb", "cause": "shutting down" })), )); } let cluster = state.cluster().map_err(ClusterAppError)?; let leader = cluster.leader_region(); let items = cluster.item_count(leader); Ok(( StatusCode::OK, Json(serde_json::json!({ "ok": true, "service": "tidaldb", "mode": "cluster", "leader": state.region_name(leader), "items": items, })), )) } // ── Cluster management ───────────────────────────────────────────────────── /// `GET /cluster/status` response body. #[derive(Serialize, ToSchema)] pub struct ClusterStatusResponse { /// Current leader region name. leader: String, /// Length of the leader's replication relay log. relay_log_len: u64, /// Per-region replication status. regions: Vec, } /// One region's replication status within a [`ClusterStatusResponse`]. #[derive(Serialize, ToSchema)] pub struct RegionStatus { /// Region name. name: String, /// Replication events applied on this region. applied_events: u64, /// Events the region lags the leader by. lag_events: u64, /// Whether this region is currently partitioned from the leader. partitioned: bool, } #[utoipa::path( get, path = "/cluster/status", tag = "cluster", responses( (status = 200, description = "Cluster replication status", body = ClusterStatusResponse), (status = 401, description = "Missing or invalid credential"), ), security(("bearerAuth" = [])), )] pub async fn cluster_status( State(state): State>, ) -> std::result::Result, ClusterAppError> { let cluster = state.cluster().map_err(ClusterAppError)?; let leader_region = cluster.leader_region(); let relay_log_len = cluster.relay_log_len(); let regions: Vec = cluster .regions() .into_iter() .map(|region| { let applied = cluster.applied_count(region); let lag = relay_log_len.saturating_sub(applied); RegionStatus { name: state.region_name(region).to_string(), applied_events: applied, lag_events: lag, partitioned: cluster.is_partitioned(region), } }) .collect(); Ok(Json(ClusterStatusResponse { leader: state.region_name(leader_region).to_string(), relay_log_len, regions, })) } /// Body for the cluster region-management routes (`/cluster/promote`, /// `/cluster/partition`, `/cluster/heal`). #[derive(Deserialize, ToSchema)] pub struct RegionRequest { /// Target region name. #[schema(example = "us-east")] region: String, } #[utoipa::path( post, path = "/cluster/promote", tag = "cluster", request_body = RegionRequest, responses( (status = 200, description = "Region promoted to leader"), (status = 400, description = "Unknown region"), (status = 401, description = "Missing or invalid API key"), ), security(("bearerAuth" = [])), )] pub async fn cluster_promote( State(state): State>, Json(req): Json, ) -> std::result::Result, ClusterAppError> { let region_id = state.resolve_region(&req.region).map_err(ClusterAppError)?; state .cluster() .map_err(ClusterAppError)? .promote_leader(region_id); Ok(Json(serde_json::json!({ "ok": true, "leader": req.region, }))) } #[utoipa::path( post, path = "/cluster/partition", tag = "cluster", request_body = RegionRequest, responses( (status = 200, description = "Region partitioned from the leader"), (status = 400, description = "Unknown region"), (status = 401, description = "Missing or invalid API key"), ), security(("bearerAuth" = [])), )] pub async fn cluster_partition( State(state): State>, Json(req): Json, ) -> std::result::Result, ClusterAppError> { let region_id = state.resolve_region(&req.region).map_err(ClusterAppError)?; state .cluster() .map_err(ClusterAppError)? .partition_region(region_id); Ok(Json(serde_json::json!({ "ok": true, "partitioned": req.region, }))) } #[utoipa::path( post, path = "/cluster/heal", tag = "cluster", request_body = RegionRequest, responses( (status = 200, description = "Region healed (missed segments re-shipped)"), (status = 400, description = "Unknown region"), (status = 401, description = "Missing or invalid API key"), (status = 429, description = "Write pool saturated"), ), security(("bearerAuth" = [])), )] pub async fn cluster_heal( State(state): State>, Json(req): Json, ) -> std::result::Result, ClusterAppError> { let region_id = state.resolve_region(&req.region).map_err(ClusterAppError)?; // heal_region re-ships missed segments over gRPC (blocking), so offload it to // the runtime-free write pool. A saturated pool degrades to 429, not a 500. let cluster = state.cluster_arc().map_err(ClusterAppError)?; state .write_pool() .submit(move || { cluster.heal_region(region_id); Ok(()) }) .await .map_err(ClusterAppError)?; Ok(Json(serde_json::json!({ "ok": true, "healed": req.region, }))) } // ── Data routes (cluster mode) ───────────────────────────────────────────── #[utoipa::path( post, path = "/items", tag = "data", request_body = ItemRequest, responses( (status = 201, description = "Item created on the leader"), (status = 400, description = "Invalid request"), (status = 401, description = "Missing or invalid API key"), ), security(("bearerAuth" = [])), )] pub async fn create_item( State(state): State>, Json(req): Json, ) -> std::result::Result { state .cluster() .map_err(ClusterAppError)? .write_item_with_metadata(EntityId::new(req.entity_id), &req.metadata) .map_err(|e| ClusterAppError(ServerError::from(e)))?; Ok(StatusCode::CREATED) } #[utoipa::path( post, path = "/embeddings", tag = "data", request_body = EmbeddingRequest, responses( (status = 204, description = "Embedding written on the leader"), (status = 400, description = "Invalid request"), (status = 401, description = "Missing or invalid API key"), ), security(("bearerAuth" = [])), )] pub async fn write_embedding( State(state): State>, Json(req): Json, ) -> std::result::Result { state .cluster() .map_err(ClusterAppError)? .write_item_embedding(EntityId::new(req.entity_id), &req.values) .map_err(|e| ClusterAppError(ServerError::from(e)))?; Ok(StatusCode::NO_CONTENT) } /// Refuse a signal whose originating context this router cannot honour. /// /// `SimulatedCluster::write_signal` and `scatter_gather::sharded_write_signal` /// both apply `(signal, entity, weight)` and nothing else, so `user_id` / /// `creator_id` would be accepted and thrown away — no hard negatives, no seen /// tracking, no interaction weight, no preference vector, and a 204 claiming /// success. Fail closed instead, naming the route that does support it. /// /// # Errors /// /// `ServerError::BadRequest` when either context id is present. fn reject_unsupported_signal_context(req: &SignalRequest) -> Result<()> { if req.user_id.is_some() || req.creator_id.is_some() { return Err(ServerError::BadRequest( "single-process cluster mode cannot record signal context: \ user_id/creator_id are unsupported on this router. Run the \ multi-process cluster (`--region`, the deployed RF3 topology), \ which applies context via signal_with_context_staged." .to_owned(), )); } Ok(()) } /// Record a signal on the leader region and eagerly ship it to followers. /// /// Returns `204 No Content` once the signal is **durably applied on the leader** /// (storage + WAL fsync). The follower ship is BEST-EFFORT: a ship that fails is /// logged (engine side) and queued for re-delivery via `await_convergence` / /// `heal_region`, but the 204 does NOT assert quorum or follower acknowledgement /// — it asserts leader durability only. A quorum-ack write contract (a 204 that /// blocks for N-follower acks) is a post-M8 follow-up (ROADMAP M8 Known Gaps G4), /// NOT part of m8p10 — the same contract the multi-process `/signals` route holds. #[utoipa::path( post, path = "/signals", tag = "data", request_body = SignalRequest, responses( (status = 204, description = "Signal durably applied on the leader"), (status = 400, description = "Invalid request"), (status = 401, description = "Missing or invalid API key"), (status = 429, description = "Write pool saturated"), ), security(("bearerAuth" = [])), )] pub async fn write_signal( State(state): State>, Json(req): Json, ) -> std::result::Result { // The simulated relay behind single-process mode applies (signal, entity, // weight) only, so it CANNOT honour originating context. Refuse rather than // accept-and-discard: a 204 on a dropped `user_id` is the failure mode that // made a clustered deployment look healthy while learning nothing. The // production multi-process path (`build_region_router`) carries context // properly — see `ClusterNode::stage_signal_local`. reject_unsupported_signal_context(&req).map_err(ClusterAppError)?; // write_signal ships to followers over gRPC (a blocking `runtime.block_on`), // so it must run off the async reactor AND off any thread carrying a runtime // handle — hand it to the runtime-free write pool. A saturated pool yields // 429 (backpressure), not a 500. See [`crate::offload::ClusterWritePool`]. let cluster = state.cluster_arc().map_err(ClusterAppError)?; let signal = req.signal; let entity = EntityId::new(req.entity_id); let weight = req.weight; state .write_pool() .submit(move || { cluster .write_signal(&signal, entity, weight) .map_err(ServerError::from) }) .await .map_err(ClusterAppError)?; Ok(StatusCode::NO_CONTENT) } #[utoipa::path( get, path = "/feed", tag = "data", params(FeedQuery), responses( (status = 200, description = "Ranked feed from the target region", body = FeedResponse), (status = 400, description = "Unknown region or invalid request"), (status = 401, description = "Missing or invalid API key"), ), security(("bearerAuth" = [])), )] pub async fn feed( State(state): State>, Query(query): Query, ) -> std::result::Result, ClusterAppError> { let region = state .read_region(query.region.as_deref()) .map_err(ClusterAppError)?; // Clamp the client-supplied limit at the network trust boundary (the same // memory-amplification guard the standalone /feed handler applies) — the // cluster read path previously passed the raw value straight to the engine. let mut builder = Retrieve::builder() .profile(&query.profile) .limit(query.clamped_limit() as usize); if let Some(user_id) = query.user_id { builder = builder.for_user(user_id); } // m12p2: "more like this" seed for `profile=related` ANN candidate-gen. if let Some(seed) = query.similar_to { builder = builder.similar_to(EntityId::new(seed)); } let retrieve = builder .build() .map_err(|e| ClusterAppError(ServerError::Tidal(e.into())))?; // `retrieve` is a blocking `TidalDb` query; running it directly on the axum // reactor would stall every other in-flight request behind one slow shard. // Offload it to the blocking pool — see [`offload_cluster_read`]. let cluster = state.cluster_arc().map_err(ClusterAppError)?; let result = offload_cluster_read(move || { cluster .retrieve(region, &retrieve) .map_err(ServerError::from) }) .await?; Ok(Json(FeedResponse { items: feed_items(&result.items), total_candidates: result.total_candidates, region: query.region, unavailable_shards: None, // single-process region serve: complete })) } #[utoipa::path( get, path = "/search", tag = "data", params(SearchQueryParams), responses( (status = 200, description = "Ranked search from the target region", body = SearchResponse), (status = 400, description = "Unknown region or invalid request"), (status = 401, description = "Missing or invalid API key"), ), security(("bearerAuth" = [])), )] pub async fn search( State(state): State>, Query(query): Query, ) -> std::result::Result, ClusterAppError> { let region = state .read_region(query.region.as_deref()) .map_err(ClusterAppError)?; // Clamp the client-supplied limit at the trust boundary (matches the // standalone /search handler); the cluster path passed the raw value before. let mut builder = Search::builder() .query(&query.query) .limit(query.clamped_limit()); if let Some(user_id) = query.user_id { builder = builder.for_user(user_id); } let search_query = builder .build() .map_err(|e| ClusterAppError(ServerError::Tidal(e.into())))?; // The text-index reload and the search are both blocking `TidalDb` calls; // running them on the axum reactor would stall the whole node. Offload the // pair to the blocking pool so the reactor stays free — see // [`offload_cluster_read`]. let cluster = state.cluster_arc().map_err(ClusterAppError)?; let result = offload_cluster_read(move || { // Reload text index on the target region before searching. cluster .node(region) .db .reload_text_index() .map_err(ServerError::from)?; cluster .search(region, &search_query) .map_err(ServerError::from) }) .await?; Ok(Json(SearchResponse { items: search_items(&result.items), total_candidates: result.total_candidates, region: query.region, unavailable_shards: None, // single-process region serve: complete })) } // ── Sharded (scatter-gather) routes ───────────────────────────────────────── /// `POST /sharded/items` — write to the entity's owning shard. /// /// **SINGLE-COPY.** The write lands on the owning shard's store only; it does not /// ride the leader relay the non-sharded `/items` surface uses, so it has /// redundancy 1 regardless of the replication factor. That is by design (parallel /// write throughput across shard owners), which is why the surface requires /// `x-tidal-ack: local` as an explicit acknowledgement of the tradeoff. For a /// replicated write use `POST /items`. #[utoipa::path( post, path = "/sharded/items", tag = "sharded", request_body = ItemRequest, responses( (status = 201, description = "Item written SINGLE-COPY to its owning shard (not replicated)"), (status = 400, description = "Invalid request, or the required `x-tidal-ack: local` single-copy opt-in is missing (use POST /items for a replicated write)"), (status = 401, description = "Missing or invalid API key"), ), security(("bearerAuth" = [])), )] pub async fn sharded_create_item( State(state): State>, headers: HeaderMap, Json(req): Json, ) -> std::result::Result { // The same gate, from the same definition, as the multi-process region router // (`cluster::node`). One URL, one durability contract. require_local_ack(&headers, "/sharded/items").map_err(ClusterAppError)?; let shards = state.shard_ids().map_err(ClusterAppError)?; // The single-shard write does a blocking storage + WAL-fsync `TidalDb` call; // running it inline would pin this reactor worker for the whole write (the // exact hazard `offload` exists to prevent). It touches only the local store // (no gRPC ship), so `spawn_blocking` is sufficient — same treatment the // non-sharded cluster reads/writes already get. See [`crate::offload`]. let cluster = state.cluster_arc().map_err(ClusterAppError)?; let entity = EntityId::new(req.entity_id); let metadata = req.metadata; offload_cluster_read(move || { crate::scatter_gather::sharded_write_item(&cluster, entity, &metadata, &shards) }) .await?; Ok(StatusCode::CREATED) } /// `POST /sharded/embeddings` — write to the entity's owning shard. /// /// **SINGLE-COPY** — see [`sharded_create_item`]. Requires `x-tidal-ack: local`; /// for a replicated write use `POST /embeddings`. #[utoipa::path( post, path = "/sharded/embeddings", tag = "sharded", request_body = EmbeddingRequest, responses( (status = 204, description = "Embedding written SINGLE-COPY to its owning shard (not replicated)"), (status = 400, description = "Invalid request, or the required `x-tidal-ack: local` single-copy opt-in is missing (use POST /embeddings for a replicated write)"), (status = 401, description = "Missing or invalid API key"), ), security(("bearerAuth" = [])), )] pub async fn sharded_write_embedding( State(state): State>, headers: HeaderMap, Json(req): Json, ) -> std::result::Result { require_local_ack(&headers, "/sharded/embeddings").map_err(ClusterAppError)?; let shards = state.shard_ids().map_err(ClusterAppError)?; // Offload the blocking single-shard embedding write off the reactor (see // [`sharded_create_item`]). let cluster = state.cluster_arc().map_err(ClusterAppError)?; let entity = EntityId::new(req.entity_id); let values = req.values; offload_cluster_read(move || { crate::scatter_gather::sharded_write_embedding(&cluster, entity, &values, &shards) }) .await?; Ok(StatusCode::NO_CONTENT) } /// `POST /sharded/signals` — write to the entity's owning shard. /// /// **SINGLE-COPY** — see [`sharded_create_item`]. Requires `x-tidal-ack: local`; /// for a replicated write use `POST /signals`. #[utoipa::path( post, path = "/sharded/signals", tag = "sharded", request_body = SignalRequest, responses( (status = 204, description = "Signal written SINGLE-COPY to its owning shard (not replicated)"), (status = 400, description = "Invalid request, or the required `x-tidal-ack: local` single-copy opt-in is missing (use POST /signals for a replicated write)"), (status = 401, description = "Missing or invalid API key"), ), security(("bearerAuth" = [])), )] pub async fn sharded_write_signal( State(state): State>, headers: HeaderMap, Json(req): Json, ) -> std::result::Result { require_local_ack(&headers, "/sharded/signals").map_err(ClusterAppError)?; // Same reason as `write_signal`: the scatter-gather write applies // (signal, entity, weight) only and would discard context behind a 204. reject_unsupported_signal_context(&req).map_err(ClusterAppError)?; let shards = state.shard_ids().map_err(ClusterAppError)?; // Offload the blocking single-shard signal write off the reactor (see // [`sharded_create_item`]). let cluster = state.cluster_arc().map_err(ClusterAppError)?; let signal = req.signal; let entity = EntityId::new(req.entity_id); let weight = req.weight; offload_cluster_read(move || { crate::scatter_gather::sharded_write_signal(&cluster, &signal, entity, weight, &shards) }) .await?; Ok(StatusCode::NO_CONTENT) } /// `GET /sharded/feed` query parameters. #[derive(Deserialize, ToSchema, utoipa::IntoParams)] #[into_params(parameter_in = Query)] pub struct ShardedFeedQuery { /// Optional user to personalize ranking for. #[serde(default)] user_id: Option, /// Named ranking profile; defaults to `for_you`. #[serde(default = "default_profile")] #[param(example = "for_you")] profile: String, /// Page size; clamped to `MAX_LIMIT` (1000). #[serde(default = "default_limit")] #[param(example = 20)] limit: u32, /// Per-shard deadline in milliseconds for the scatter-gather. #[serde(default)] deadline_ms: Option, } impl ShardedFeedQuery { /// Ranking profile name. #[must_use] pub fn profile(&self) -> &str { &self.profile } /// Personalization user, if any. #[must_use] pub const fn user_id(&self) -> Option { self.user_id } /// Page size clamped to [`MAX_LIMIT`] (trust-boundary guard). #[must_use] pub const fn clamped_limit(&self) -> usize { if self.limit > MAX_LIMIT { MAX_LIMIT as usize } else { self.limit as usize } } /// Client-supplied per-shard deadline budget. #[must_use] pub const fn deadline_ms(&self) -> Option { self.deadline_ms } } /// `GET /sharded/feed` response body. #[derive(Serialize, ToSchema)] pub struct ShardedFeedResponse { /// Merged ranked items across all shards. pub(crate) items: Vec, /// Total candidates considered across shards. pub(crate) total_candidates: usize, /// Scatter-gather execution metadata. pub(crate) scatter_gather: ScatterGatherInfo, } /// Scatter-gather execution metadata returned with sharded responses. #[derive(Serialize, ToSchema)] pub struct ScatterGatherInfo { /// True when one or more shards were unavailable and results are partial. degraded: bool, /// Names of shards that did not respond within the deadline. #[serde(skip_serializing_if = "Vec::is_empty")] unavailable_shards: Vec, /// Number of shards queried. shards_queried: usize, /// Wall-clock time the scatter-gather took, in milliseconds. elapsed_ms: u64, /// Per-shard deadline that was applied, in milliseconds. shard_deadline_ms: u64, } impl From for ScatterGatherInfo { /// Project the engine's scatter-gather execution metadata into the HTTP /// response shape. Single source for the five-field mapping so the /// `/sharded/feed` and `/sharded/search` handlers cannot drift. fn from(meta: crate::scatter_gather::ScatterGatherMeta) -> Self { Self { degraded: meta.degraded, unavailable_shards: meta.unavailable_shards, shards_queried: meta.shards_queried, elapsed_ms: meta.elapsed_ms, shard_deadline_ms: meta.shard_deadline_ms, } } } #[utoipa::path( get, path = "/sharded/feed", tag = "sharded", params(ShardedFeedQuery), responses( (status = 200, description = "Scatter-gather ranked feed", body = ShardedFeedResponse), (status = 400, description = "Invalid request"), (status = 401, description = "Missing or invalid API key"), ), security(("bearerAuth" = [])), )] pub async fn sharded_feed( State(state): State>, Query(query): Query, ) -> std::result::Result, ClusterAppError> { // Clamp the client-supplied limit at the trust boundary before fanning the // scatter-gather out to every shard (the local query struct has no // clamped_limit helper, so clamp inline against the shared MAX_LIMIT). let mut builder = Retrieve::builder() .profile(&query.profile) .limit(query.limit.min(MAX_LIMIT) as usize); if let Some(user_id) = query.user_id { builder = builder.for_user(user_id); } let retrieve = builder .build() .map_err(|e| ClusterAppError(ServerError::Tidal(e.into())))?; // The scatter-gather coordinator merges shard results and reads creator // metadata for coordinator-level diversity — all blocking. Offload the // whole coordination off the reactor so a slow shard never stalls it. let cluster = state.cluster_arc().map_err(ClusterAppError)?; let shards = state.shard_ids().map_err(ClusterAppError)?; let region_names = state.id_to_name_map().clone(); let deadline_ms = query.deadline_ms; let (result, meta) = offload_cluster_read(move || { crate::scatter_gather::scatter_gather_retrieve( &cluster, &retrieve, &shards, ®ion_names, deadline_ms, ) }) .await?; Ok(Json(ShardedFeedResponse { items: feed_items(&result.items), total_candidates: result.total_candidates, scatter_gather: meta.into(), })) } /// `GET /sharded/search` query parameters. #[derive(Deserialize, ToSchema, utoipa::IntoParams)] #[into_params(parameter_in = Query)] pub struct ShardedSearchQuery { /// Free-text query string. #[param(example = "jazz piano")] query: String, /// Optional user to personalize ranking for. #[serde(default)] user_id: Option, /// Page size; clamped to `MAX_LIMIT` (1000). #[serde(default = "default_limit")] #[param(example = 20)] limit: u32, /// Per-shard deadline in milliseconds for the scatter-gather. #[serde(default)] deadline_ms: Option, } impl ShardedSearchQuery { /// Free-text query string. #[must_use] pub fn query_text(&self) -> &str { &self.query } /// Personalization user, if any. #[must_use] pub const fn user_id(&self) -> Option { self.user_id } /// Page size clamped to [`MAX_LIMIT`] as a `u32` (SEARCH limit is `u32`). #[must_use] pub const fn clamped_limit_u32(&self) -> u32 { if self.limit > MAX_LIMIT { MAX_LIMIT } else { self.limit } } /// Client-supplied per-shard deadline budget. #[must_use] pub const fn deadline_ms(&self) -> Option { self.deadline_ms } } /// `GET /sharded/search` response body. #[derive(Serialize, ToSchema)] pub struct ShardedSearchResponse { /// Merged ranked search results across all shards. pub(crate) items: Vec, /// Total candidates considered across shards. pub(crate) total_candidates: usize, /// Scatter-gather execution metadata. pub(crate) scatter_gather: ScatterGatherInfo, } #[utoipa::path( get, path = "/sharded/search", tag = "sharded", params(ShardedSearchQuery), responses( (status = 200, description = "Scatter-gather ranked search", body = ShardedSearchResponse), (status = 400, description = "Invalid request"), (status = 401, description = "Missing or invalid API key"), ), security(("bearerAuth" = [])), )] pub async fn sharded_search( State(state): State>, Query(query): Query, ) -> std::result::Result, ClusterAppError> { // Clamp at the trust boundary before scatter-gather (inline against the // shared MAX_LIMIT — the local sharded query struct has no helper). let mut builder = Search::builder() .query(&query.query) .limit(query.limit.min(MAX_LIMIT)); if let Some(user_id) = query.user_id { builder = builder.for_user(user_id); } let search_query = builder .build() .map_err(|e| ClusterAppError(ServerError::Tidal(e.into())))?; // Offload the blocking scatter-gather coordination off the reactor (see // [`sharded_feed`] / [`offload_cluster_read`]). let cluster = state.cluster_arc().map_err(ClusterAppError)?; let shards = state.shard_ids().map_err(ClusterAppError)?; let region_names = state.id_to_name_map().clone(); let deadline_ms = query.deadline_ms; let (result, meta) = offload_cluster_read(move || { crate::scatter_gather::scatter_gather_search( &cluster, &search_query, &shards, ®ion_names, deadline_ms, ) }) .await?; Ok(Json(ShardedSearchResponse { items: search_items(&result.items), total_candidates: result.total_candidates, scatter_gather: meta.into(), })) } // ── Blocking-work offload ──────────────────────────────────────────────────── /// Run a blocking READ-only cluster query (RETRIEVE / SEARCH) on the tokio /// blocking pool, off the async reactor, and await its result. /// /// Thin adapter over the shared [`offload_read`] that maps its [`ServerError`] /// into a [`ClusterAppError`] for the cluster handlers. Both the standalone and /// cluster read paths route through the same helper so they cannot drift. async fn offload_cluster_read(f: F) -> std::result::Result where F: FnOnce() -> Result + Send + 'static, T: Send + 'static, { offload_read(f).await.map_err(ClusterAppError) } // ── Error handling ───────────────────────────────────────────────────────── pub struct ClusterAppError(pub ServerError); impl IntoResponse for ClusterAppError { fn into_response(self) -> Response { let status = crate::router::status_from_error(&self.0); // For the multi-process routing errors, name the responsible node in the // body so a client can re-target the write/read itself (task 03 turns // these into transparent forwarding). Other errors keep the flat shape. let body = match &self.0 { ServerError::NotLeader { leader, http_addr, term, shard, } => serde_json::json!({ "error": self.0.to_string(), "leader": leader, "leader_http_addr": http_addr, "term": term, "shard": shard, }), ServerError::LeaderUnreachable { leader, http_addr, cause, } => serde_json::json!({ "error": self.0.to_string(), "leader": leader, "leader_http_addr": http_addr, "cause": cause, }), ServerError::NotLocal { region } => serde_json::json!({ "error": self.0.to_string(), "region": region, }), ServerError::RegionUnreachable { region, cause } => serde_json::json!({ "error": self.0.to_string(), "region": region, "cause": cause, }), ServerError::QuorumTimeout { seq, needed, confirmed, committed, laggards, } => serde_json::json!({ "error": self.0.to_string(), "retryable": true, "seq": seq, "needed": needed, "confirmed": confirmed, "commit_index": committed, "laggards": laggards, }), _ => serde_json::json!({ "error": self.0.to_string() }), }; (status, Json(body)).into_response() } }