tidaldb/tidal-server/src/cluster/routes.rs

//! HTTP routes for the single-process cluster ([`ClusterState`]): the data
//! surface (items/embeddings/signals/feed/search), the cluster-management
//! surface (status/promote/partition/heal), and the sharded scatter-gather
//! routes. The multi-process region node's routes live in `cluster::node`.

use std::{sync::Arc, time::Duration};

use axum::{
    Json, Router,
    extract::{Query, Request, State},
    http::{HeaderMap, StatusCode},
    middleware::{self, Next},
    response::{IntoResponse, Response},
    routing::{get, post},
};
use serde::{Deserialize, Serialize};
use tidaldb::{
    query::{retrieve::Retrieve, search::Search},
    schema::EntityId,
};
use tower::{ServiceBuilder, limit::ConcurrencyLimitLayer};
use tower_http::timeout::TimeoutLayer;
use utoipa::ToSchema;

use super::{node::require_local_ack, state::ClusterState};
use crate::{
    dto::{
        EmbeddingRequest, FeedItem, FeedQuery, FeedResponse, ItemRequest, MAX_LIMIT, SearchItem,
        SearchQueryParams, SearchResponse, SignalRequest, default_limit, default_profile,
        feed_items, search_items,
    },
    error::{Result, ServerError},
    offload::offload_read,
};

/// Build the cluster-mode router.
///
/// Exposes the same data routes as standalone (items, embeddings, signals,
/// feed, search) plus cluster management endpoints.
///
/// Protected routes carry the same load-shedding middleware as the standalone
/// router, consuming the shared [`crate::router::REQUEST_TIMEOUT_SECS`] /
/// [`crate::router::MAX_CONCURRENCY`] / [`crate::router::BODY_LIMIT_BYTES`]
/// constants:
/// - [`TimeoutLayer`] — 408 for requests exceeding the shared timeout;
/// - [`ConcurrencyLimitLayer`] — queues beyond the shared in-flight cap;
/// - body limit — 413 for oversized bodies before any deserialization.
///
/// Health/status probes (`public`) are intentionally outside this stack so they
/// are never queued or timed out under saturation.
pub fn build_cluster_router(
    state: Arc<ClusterState>,
    creds: Arc<crate::cluster::security::ClusterCreds>,
) -> Router {
    let public = Router::new()
        .route("/health", get(cluster_health))
        // Shared with the standalone router via [`crate::health`] so the two
        // modes can never advertise a different probe contract.
        .route("/health/startup", get(crate::health::health_startup))
        .route("/health/live", get(crate::health::health_live))
        // `/cluster/status` deliberately NOT here - see the region router: it
        // reports leader, membership and seqnos, so it is protected.
        // Cluster-superset OpenAPI document (adds the /cluster/* and /sharded/*
        // routes). Unauthenticated, like the probes — contract only, no data.
        .route("/openapi.json", get(crate::openapi::serve_cluster))
        .with_state(Arc::clone(&state));

    // The destructive operator verbs, gated on the cluster-admin credential when
    // one is configured. Same rationale as the multi-process region router: these
    // used to share the data-plane bearer, so any client key could promote,
    // partition, or heal.
    let admin_creds = Arc::clone(&creds);
    let admin = Router::new()
        .route("/cluster/promote", post(cluster_promote))
        .route("/cluster/partition", post(cluster_partition))
        .route("/cluster/heal", post(cluster_heal))
        .layer(middleware::from_fn(move |req: Request, next: Next| {
            let creds = Arc::clone(&admin_creds);
            async move {
                if creds.admin_ok(req.headers()) {
                    return next.run(req).await;
                }
                crate::router::admin_forbidden_response()
            }
        }))
        .with_state(Arc::clone(&state));

    let protected = Router::new()
        .route("/items", post(create_item))
        .route("/embeddings", post(write_embedding))
        .route("/signals", post(write_signal))
        .route("/feed", get(feed))
        .route("/search", get(search))
        .route("/cluster/status", get(cluster_status))
        // Sharded (scatter-gather) routes.
        .route("/sharded/items", post(sharded_create_item))
        .route("/sharded/embeddings", post(sharded_write_embedding))
        .route("/sharded/signals", post(sharded_write_signal))
        .route("/sharded/feed", get(sharded_feed))
        .route("/sharded/search", get(sharded_search))
        .with_state(state)
        .merge(admin)
        // Shared with the standalone router so the body cap can never drift
        // (raise one, forget the other). See [`crate::router::BODY_LIMIT_BYTES`].
        .layer(axum::extract::DefaultBodyLimit::max(
            crate::router::BODY_LIMIT_BYTES,
        ));

    // Credentials read PER REQUEST from `creds` so a rotation takes effect with no
    // restart (m11p7). No bearer configured ⇒ pass through (open). The admin key
    // also authenticates here (it is a superset credential) so an operator
    // presenting it is not rejected before the admin gate runs.
    let protected = protected.layer(middleware::from_fn(move |req: Request, next: Next| {
        let creds = Arc::clone(&creds);
        async move {
            if !creds.authenticated(req.headers()) {
                return crate::router::unauthorized_response(req.headers());
            }
            let principal = creds.principal(req.headers());
            if let Err((retry_after_ms, limit)) = creds.check_rate(&principal) {
                return crate::router::too_many_requests(retry_after_ms, limit);
            }
            next.run(req).await
        }
    }));

    // Mirror the standalone router's load-shedding stack on the cluster
    // protected routes: a request-timeout (408) and a hard in-flight cap (429)
    // using the SAME constants as standalone so the two surfaces can never
    // advertise different overload behavior. Health probes (the `public`
    // routes) are deliberately left outside this stack so liveness/readiness are
    // never queued or timed out under saturation.
    let protected = protected.layer(
        ServiceBuilder::new()
            .layer(TimeoutLayer::with_status_code(
                StatusCode::REQUEST_TIMEOUT,
                Duration::from_secs(crate::router::REQUEST_TIMEOUT_SECS),
            ))
            .layer(ConcurrencyLimitLayer::new(crate::router::MAX_CONCURRENCY)),
    );

    // m11p8: the same request-id + tracing stack as the standalone router —
    // assigns/echoes `x-request-id` and opens a per-request span. The cluster
    // routers previously skipped it; with it, a write forwarded to the leader
    // carries the originating gateway's id (see `cluster::forward`).
    crate::router::with_request_id_tracing(public.merge(protected))
}

// ── Health ──────────────────────────────────────────────────────────────────
//
// The unconditional startup/live probes are shared with the standalone router
// via [`crate::health`]; only the mode-specific readiness probe (`cluster_health`)
// lives here.

async fn cluster_health(
    State(state): State<Arc<ClusterState>>,
) -> std::result::Result<(StatusCode, Json<serde_json::Value>), ClusterAppError> {
    if state.is_shutting_down() {
        return Ok((
            StatusCode::SERVICE_UNAVAILABLE,
            Json(serde_json::json!({
                "ok": false,
                "service": "tidaldb",
                "cause": "shutting down"
            })),
        ));
    }

    let cluster = state.cluster().map_err(ClusterAppError)?;
    let leader = cluster.leader_region();
    let items = cluster.item_count(leader);

    Ok((
        StatusCode::OK,
        Json(serde_json::json!({
            "ok": true,
            "service": "tidaldb",
            "mode": "cluster",
            "leader": state.region_name(leader),
            "items": items,
        })),
    ))
}

// ── Cluster management ─────────────────────────────────────────────────────

/// `GET /cluster/status` response body.
#[derive(Serialize, ToSchema)]
pub struct ClusterStatusResponse {
    /// Current leader region name.
    leader: String,
    /// Length of the leader's replication relay log.
    relay_log_len: u64,
    /// Per-region replication status.
    regions: Vec<RegionStatus>,
}

/// One region's replication status within a [`ClusterStatusResponse`].
#[derive(Serialize, ToSchema)]
pub struct RegionStatus {
    /// Region name.
    name: String,
    /// Replication events applied on this region.
    applied_events: u64,
    /// Events the region lags the leader by.
    lag_events: u64,
    /// Whether this region is currently partitioned from the leader.
    partitioned: bool,
}

#[utoipa::path(
    get,
    path = "/cluster/status",
    tag = "cluster",
    responses(
        (status = 200, description = "Cluster replication status", body = ClusterStatusResponse),
        (status = 401, description = "Missing or invalid credential"),
    ),
    security(("bearerAuth" = [])),
)]
pub async fn cluster_status(
    State(state): State<Arc<ClusterState>>,
) -> std::result::Result<Json<ClusterStatusResponse>, ClusterAppError> {
    let cluster = state.cluster().map_err(ClusterAppError)?;
    let leader_region = cluster.leader_region();
    let relay_log_len = cluster.relay_log_len();

    let regions: Vec<RegionStatus> = cluster
        .regions()
        .into_iter()
        .map(|region| {
            let applied = cluster.applied_count(region);
            let lag = relay_log_len.saturating_sub(applied);
            RegionStatus {
                name: state.region_name(region).to_string(),
                applied_events: applied,
                lag_events: lag,
                partitioned: cluster.is_partitioned(region),
            }
        })
        .collect();

    Ok(Json(ClusterStatusResponse {
        leader: state.region_name(leader_region).to_string(),
        relay_log_len,
        regions,
    }))
}

/// Body for the cluster region-management routes (`/cluster/promote`,
/// `/cluster/partition`, `/cluster/heal`).
#[derive(Deserialize, ToSchema)]
pub struct RegionRequest {
    /// Target region name.
    #[schema(example = "us-east")]
    region: String,
}

#[utoipa::path(
    post,
    path = "/cluster/promote",
    tag = "cluster",
    request_body = RegionRequest,
    responses(
        (status = 200, description = "Region promoted to leader"),
        (status = 400, description = "Unknown region"),
        (status = 401, description = "Missing or invalid API key"),
    ),
    security(("bearerAuth" = [])),
)]
pub async fn cluster_promote(
    State(state): State<Arc<ClusterState>>,
    Json(req): Json<RegionRequest>,
) -> std::result::Result<Json<serde_json::Value>, ClusterAppError> {
    let region_id = state.resolve_region(&req.region).map_err(ClusterAppError)?;
    state
        .cluster()
        .map_err(ClusterAppError)?
        .promote_leader(region_id);
    Ok(Json(serde_json::json!({
        "ok": true,
        "leader": req.region,
    })))
}

#[utoipa::path(
    post,
    path = "/cluster/partition",
    tag = "cluster",
    request_body = RegionRequest,
    responses(
        (status = 200, description = "Region partitioned from the leader"),
        (status = 400, description = "Unknown region"),
        (status = 401, description = "Missing or invalid API key"),
    ),
    security(("bearerAuth" = [])),
)]
pub async fn cluster_partition(
    State(state): State<Arc<ClusterState>>,
    Json(req): Json<RegionRequest>,
) -> std::result::Result<Json<serde_json::Value>, ClusterAppError> {
    let region_id = state.resolve_region(&req.region).map_err(ClusterAppError)?;
    state
        .cluster()
        .map_err(ClusterAppError)?
        .partition_region(region_id);
    Ok(Json(serde_json::json!({
        "ok": true,
        "partitioned": req.region,
    })))
}

#[utoipa::path(
    post,
    path = "/cluster/heal",
    tag = "cluster",
    request_body = RegionRequest,
    responses(
        (status = 200, description = "Region healed (missed segments re-shipped)"),
        (status = 400, description = "Unknown region"),
        (status = 401, description = "Missing or invalid API key"),
        (status = 429, description = "Write pool saturated"),
    ),
    security(("bearerAuth" = [])),
)]
pub async fn cluster_heal(
    State(state): State<Arc<ClusterState>>,
    Json(req): Json<RegionRequest>,
) -> std::result::Result<Json<serde_json::Value>, ClusterAppError> {
    let region_id = state.resolve_region(&req.region).map_err(ClusterAppError)?;
    // heal_region re-ships missed segments over gRPC (blocking), so offload it to
    // the runtime-free write pool. A saturated pool degrades to 429, not a 500.
    let cluster = state.cluster_arc().map_err(ClusterAppError)?;
    state
        .write_pool()
        .submit(move || {
            cluster.heal_region(region_id);
            Ok(())
        })
        .await
        .map_err(ClusterAppError)?;
    Ok(Json(serde_json::json!({
        "ok": true,
        "healed": req.region,
    })))
}

// ── Data routes (cluster mode) ─────────────────────────────────────────────

#[utoipa::path(
    post,
    path = "/items",
    tag = "data",
    request_body = ItemRequest,
    responses(
        (status = 201, description = "Item created on the leader"),
        (status = 400, description = "Invalid request"),
        (status = 401, description = "Missing or invalid API key"),
    ),
    security(("bearerAuth" = [])),
)]
pub async fn create_item(
    State(state): State<Arc<ClusterState>>,
    Json(req): Json<ItemRequest>,
) -> std::result::Result<StatusCode, ClusterAppError> {
    state
        .cluster()
        .map_err(ClusterAppError)?
        .write_item_with_metadata(EntityId::new(req.entity_id), &req.metadata)
        .map_err(|e| ClusterAppError(ServerError::from(e)))?;
    Ok(StatusCode::CREATED)
}

#[utoipa::path(
    post,
    path = "/embeddings",
    tag = "data",
    request_body = EmbeddingRequest,
    responses(
        (status = 204, description = "Embedding written on the leader"),
        (status = 400, description = "Invalid request"),
        (status = 401, description = "Missing or invalid API key"),
    ),
    security(("bearerAuth" = [])),
)]
pub async fn write_embedding(
    State(state): State<Arc<ClusterState>>,
    Json(req): Json<EmbeddingRequest>,
) -> std::result::Result<StatusCode, ClusterAppError> {
    state
        .cluster()
        .map_err(ClusterAppError)?
        .write_item_embedding(EntityId::new(req.entity_id), &req.values)
        .map_err(|e| ClusterAppError(ServerError::from(e)))?;
    Ok(StatusCode::NO_CONTENT)
}

/// Refuse a signal whose originating context this router cannot honour.
///
/// `SimulatedCluster::write_signal` and `scatter_gather::sharded_write_signal`
/// both apply `(signal, entity, weight)` and nothing else, so `user_id` /
/// `creator_id` would be accepted and thrown away — no hard negatives, no seen
/// tracking, no interaction weight, no preference vector, and a 204 claiming
/// success. Fail closed instead, naming the route that does support it.
///
/// # Errors
///
/// `ServerError::BadRequest` when either context id is present.
fn reject_unsupported_signal_context(req: &SignalRequest) -> Result<()> {
    if req.user_id.is_some() || req.creator_id.is_some() {
        return Err(ServerError::BadRequest(
            "single-process cluster mode cannot record signal context: \
             user_id/creator_id are unsupported on this router. Run the \
             multi-process cluster (`--region`, the deployed RF3 topology), \
             which applies context via signal_with_context_staged."
                .to_owned(),
        ));
    }
    Ok(())
}

/// Record a signal on the leader region and eagerly ship it to followers.
///
/// Returns `204 No Content` once the signal is **durably applied on the leader**
/// (storage + WAL fsync). The follower ship is BEST-EFFORT: a ship that fails is
/// logged (engine side) and queued for re-delivery via `await_convergence` /
/// `heal_region`, but the 204 does NOT assert quorum or follower acknowledgement
/// — it asserts leader durability only. A quorum-ack write contract (a 204 that
/// blocks for N-follower acks) is a post-M8 follow-up (ROADMAP M8 Known Gaps G4),
/// NOT part of m8p10 — the same contract the multi-process `/signals` route holds.
#[utoipa::path(
    post,
    path = "/signals",
    tag = "data",
    request_body = SignalRequest,
    responses(
        (status = 204, description = "Signal durably applied on the leader"),
        (status = 400, description = "Invalid request"),
        (status = 401, description = "Missing or invalid API key"),
        (status = 429, description = "Write pool saturated"),
    ),
    security(("bearerAuth" = [])),
)]
pub async fn write_signal(
    State(state): State<Arc<ClusterState>>,
    Json(req): Json<SignalRequest>,
) -> std::result::Result<StatusCode, ClusterAppError> {
    // The simulated relay behind single-process mode applies (signal, entity,
    // weight) only, so it CANNOT honour originating context. Refuse rather than
    // accept-and-discard: a 204 on a dropped `user_id` is the failure mode that
    // made a clustered deployment look healthy while learning nothing. The
    // production multi-process path (`build_region_router`) carries context
    // properly — see `ClusterNode::stage_signal_local`.
    reject_unsupported_signal_context(&req).map_err(ClusterAppError)?;
    // write_signal ships to followers over gRPC (a blocking `runtime.block_on`),
    // so it must run off the async reactor AND off any thread carrying a runtime
    // handle — hand it to the runtime-free write pool. A saturated pool yields
    // 429 (backpressure), not a 500. See [`crate::offload::ClusterWritePool`].
    let cluster = state.cluster_arc().map_err(ClusterAppError)?;
    let signal = req.signal;
    let entity = EntityId::new(req.entity_id);
    let weight = req.weight;
    state
        .write_pool()
        .submit(move || {
            cluster
                .write_signal(&signal, entity, weight)
                .map_err(ServerError::from)
        })
        .await
        .map_err(ClusterAppError)?;
    Ok(StatusCode::NO_CONTENT)
}

#[utoipa::path(
    get,
    path = "/feed",
    tag = "data",
    params(FeedQuery),
    responses(
        (status = 200, description = "Ranked feed from the target region", body = FeedResponse),
        (status = 400, description = "Unknown region or invalid request"),
        (status = 401, description = "Missing or invalid API key"),
    ),
    security(("bearerAuth" = [])),
)]
pub async fn feed(
    State(state): State<Arc<ClusterState>>,
    Query(query): Query<FeedQuery>,
) -> std::result::Result<Json<FeedResponse>, ClusterAppError> {
    let region = state
        .read_region(query.region.as_deref())
        .map_err(ClusterAppError)?;

    // Clamp the client-supplied limit at the network trust boundary (the same
    // memory-amplification guard the standalone /feed handler applies) — the
    // cluster read path previously passed the raw value straight to the engine.
    let mut builder = Retrieve::builder()
        .profile(&query.profile)
        .limit(query.clamped_limit() as usize);
    if let Some(user_id) = query.user_id {
        builder = builder.for_user(user_id);
    }
    // m12p2: "more like this" seed for `profile=related` ANN candidate-gen.
    if let Some(seed) = query.similar_to {
        builder = builder.similar_to(EntityId::new(seed));
    }
    let retrieve = builder
        .build()
        .map_err(|e| ClusterAppError(ServerError::Tidal(e.into())))?;

    // `retrieve` is a blocking `TidalDb` query; running it directly on the axum
    // reactor would stall every other in-flight request behind one slow shard.
    // Offload it to the blocking pool — see [`offload_cluster_read`].
    let cluster = state.cluster_arc().map_err(ClusterAppError)?;
    let result = offload_cluster_read(move || {
        cluster
            .retrieve(region, &retrieve)
            .map_err(ServerError::from)
    })
    .await?;

    Ok(Json(FeedResponse {
        items: feed_items(&result.items),
        total_candidates: result.total_candidates,
        region: query.region,
        unavailable_shards: None, // single-process region serve: complete
    }))
}

#[utoipa::path(
    get,
    path = "/search",
    tag = "data",
    params(SearchQueryParams),
    responses(
        (status = 200, description = "Ranked search from the target region", body = SearchResponse),
        (status = 400, description = "Unknown region or invalid request"),
        (status = 401, description = "Missing or invalid API key"),
    ),
    security(("bearerAuth" = [])),
)]
pub async fn search(
    State(state): State<Arc<ClusterState>>,
    Query(query): Query<SearchQueryParams>,
) -> std::result::Result<Json<SearchResponse>, ClusterAppError> {
    let region = state
        .read_region(query.region.as_deref())
        .map_err(ClusterAppError)?;

    // Clamp the client-supplied limit at the trust boundary (matches the
    // standalone /search handler); the cluster path passed the raw value before.
    let mut builder = Search::builder()
        .query(&query.query)
        .limit(query.clamped_limit());
    if let Some(user_id) = query.user_id {
        builder = builder.for_user(user_id);
    }
    let search_query = builder
        .build()
        .map_err(|e| ClusterAppError(ServerError::Tidal(e.into())))?;

    // The text-index reload and the search are both blocking `TidalDb` calls;
    // running them on the axum reactor would stall the whole node. Offload the
    // pair to the blocking pool so the reactor stays free — see
    // [`offload_cluster_read`].
    let cluster = state.cluster_arc().map_err(ClusterAppError)?;
    let result = offload_cluster_read(move || {
        // Reload text index on the target region before searching.
        cluster
            .node(region)
            .db
            .reload_text_index()
            .map_err(ServerError::from)?;
        cluster
            .search(region, &search_query)
            .map_err(ServerError::from)
    })
    .await?;

    Ok(Json(SearchResponse {
        items: search_items(&result.items),
        total_candidates: result.total_candidates,
        region: query.region,
        unavailable_shards: None, // single-process region serve: complete
    }))
}

// ── Sharded (scatter-gather) routes ─────────────────────────────────────────

/// `POST /sharded/items` — write to the entity's owning shard.
///
/// **SINGLE-COPY.** The write lands on the owning shard's store only; it does not
/// ride the leader relay the non-sharded `/items` surface uses, so it has
/// redundancy 1 regardless of the replication factor. That is by design (parallel
/// write throughput across shard owners), which is why the surface requires
/// `x-tidal-ack: local` as an explicit acknowledgement of the tradeoff. For a
/// replicated write use `POST /items`.
#[utoipa::path(
    post,
    path = "/sharded/items",
    tag = "sharded",
    request_body = ItemRequest,
    responses(
        (status = 201, description = "Item written SINGLE-COPY to its owning shard (not replicated)"),
        (status = 400, description = "Invalid request, or the required `x-tidal-ack: local` single-copy opt-in is missing (use POST /items for a replicated write)"),
        (status = 401, description = "Missing or invalid API key"),
    ),
    security(("bearerAuth" = [])),
)]
pub async fn sharded_create_item(
    State(state): State<Arc<ClusterState>>,
    headers: HeaderMap,
    Json(req): Json<ItemRequest>,
) -> std::result::Result<StatusCode, ClusterAppError> {
    // The same gate, from the same definition, as the multi-process region router
    // (`cluster::node`). One URL, one durability contract.
    require_local_ack(&headers, "/sharded/items").map_err(ClusterAppError)?;
    let shards = state.shard_ids().map_err(ClusterAppError)?;
    // The single-shard write does a blocking storage + WAL-fsync `TidalDb` call;
    // running it inline would pin this reactor worker for the whole write (the
    // exact hazard `offload` exists to prevent). It touches only the local store
    // (no gRPC ship), so `spawn_blocking` is sufficient — same treatment the
    // non-sharded cluster reads/writes already get. See [`crate::offload`].
    let cluster = state.cluster_arc().map_err(ClusterAppError)?;
    let entity = EntityId::new(req.entity_id);
    let metadata = req.metadata;
    offload_cluster_read(move || {
        crate::scatter_gather::sharded_write_item(&cluster, entity, &metadata, &shards)
    })
    .await?;
    Ok(StatusCode::CREATED)
}

/// `POST /sharded/embeddings` — write to the entity's owning shard.
///
/// **SINGLE-COPY** — see [`sharded_create_item`]. Requires `x-tidal-ack: local`;
/// for a replicated write use `POST /embeddings`.
#[utoipa::path(
    post,
    path = "/sharded/embeddings",
    tag = "sharded",
    request_body = EmbeddingRequest,
    responses(
        (status = 204, description = "Embedding written SINGLE-COPY to its owning shard (not replicated)"),
        (status = 400, description = "Invalid request, or the required `x-tidal-ack: local` single-copy opt-in is missing (use POST /embeddings for a replicated write)"),
        (status = 401, description = "Missing or invalid API key"),
    ),
    security(("bearerAuth" = [])),
)]
pub async fn sharded_write_embedding(
    State(state): State<Arc<ClusterState>>,
    headers: HeaderMap,
    Json(req): Json<EmbeddingRequest>,
) -> std::result::Result<StatusCode, ClusterAppError> {
    require_local_ack(&headers, "/sharded/embeddings").map_err(ClusterAppError)?;
    let shards = state.shard_ids().map_err(ClusterAppError)?;
    // Offload the blocking single-shard embedding write off the reactor (see
    // [`sharded_create_item`]).
    let cluster = state.cluster_arc().map_err(ClusterAppError)?;
    let entity = EntityId::new(req.entity_id);
    let values = req.values;
    offload_cluster_read(move || {
        crate::scatter_gather::sharded_write_embedding(&cluster, entity, &values, &shards)
    })
    .await?;
    Ok(StatusCode::NO_CONTENT)
}

/// `POST /sharded/signals` — write to the entity's owning shard.
///
/// **SINGLE-COPY** — see [`sharded_create_item`]. Requires `x-tidal-ack: local`;
/// for a replicated write use `POST /signals`.
#[utoipa::path(
    post,
    path = "/sharded/signals",
    tag = "sharded",
    request_body = SignalRequest,
    responses(
        (status = 204, description = "Signal written SINGLE-COPY to its owning shard (not replicated)"),
        (status = 400, description = "Invalid request, or the required `x-tidal-ack: local` single-copy opt-in is missing (use POST /signals for a replicated write)"),
        (status = 401, description = "Missing or invalid API key"),
    ),
    security(("bearerAuth" = [])),
)]
pub async fn sharded_write_signal(
    State(state): State<Arc<ClusterState>>,
    headers: HeaderMap,
    Json(req): Json<SignalRequest>,
) -> std::result::Result<StatusCode, ClusterAppError> {
    require_local_ack(&headers, "/sharded/signals").map_err(ClusterAppError)?;
    // Same reason as `write_signal`: the scatter-gather write applies
    // (signal, entity, weight) only and would discard context behind a 204.
    reject_unsupported_signal_context(&req).map_err(ClusterAppError)?;
    let shards = state.shard_ids().map_err(ClusterAppError)?;
    // Offload the blocking single-shard signal write off the reactor (see
    // [`sharded_create_item`]).
    let cluster = state.cluster_arc().map_err(ClusterAppError)?;
    let signal = req.signal;
    let entity = EntityId::new(req.entity_id);
    let weight = req.weight;
    offload_cluster_read(move || {
        crate::scatter_gather::sharded_write_signal(&cluster, &signal, entity, weight, &shards)
    })
    .await?;
    Ok(StatusCode::NO_CONTENT)
}

/// `GET /sharded/feed` query parameters.
#[derive(Deserialize, ToSchema, utoipa::IntoParams)]
#[into_params(parameter_in = Query)]
pub struct ShardedFeedQuery {
    /// Optional user to personalize ranking for.
    #[serde(default)]
    user_id: Option<u64>,
    /// Named ranking profile; defaults to `for_you`.
    #[serde(default = "default_profile")]
    #[param(example = "for_you")]
    profile: String,
    /// Page size; clamped to `MAX_LIMIT` (1000).
    #[serde(default = "default_limit")]
    #[param(example = 20)]
    limit: u32,
    /// Per-shard deadline in milliseconds for the scatter-gather.
    #[serde(default)]
    deadline_ms: Option<u64>,
}

impl ShardedFeedQuery {
    /// Ranking profile name.
    #[must_use]
    pub fn profile(&self) -> &str {
        &self.profile
    }
    /// Personalization user, if any.
    #[must_use]
    pub const fn user_id(&self) -> Option<u64> {
        self.user_id
    }
    /// Page size clamped to [`MAX_LIMIT`] (trust-boundary guard).
    #[must_use]
    pub const fn clamped_limit(&self) -> usize {
        if self.limit > MAX_LIMIT {
            MAX_LIMIT as usize
        } else {
            self.limit as usize
        }
    }
    /// Client-supplied per-shard deadline budget.
    #[must_use]
    pub const fn deadline_ms(&self) -> Option<u64> {
        self.deadline_ms
    }
}

/// `GET /sharded/feed` response body.
#[derive(Serialize, ToSchema)]
pub struct ShardedFeedResponse {
    /// Merged ranked items across all shards.
    pub(crate) items: Vec<FeedItem>,
    /// Total candidates considered across shards.
    pub(crate) total_candidates: usize,
    /// Scatter-gather execution metadata.
    pub(crate) scatter_gather: ScatterGatherInfo,
}

/// Scatter-gather execution metadata returned with sharded responses.
#[derive(Serialize, ToSchema)]
pub struct ScatterGatherInfo {
    /// True when one or more shards were unavailable and results are partial.
    degraded: bool,
    /// Names of shards that did not respond within the deadline.
    #[serde(skip_serializing_if = "Vec::is_empty")]
    unavailable_shards: Vec<String>,
    /// Number of shards queried.
    shards_queried: usize,
    /// Wall-clock time the scatter-gather took, in milliseconds.
    elapsed_ms: u64,
    /// Per-shard deadline that was applied, in milliseconds.
    shard_deadline_ms: u64,
}

impl From<crate::scatter_gather::ScatterGatherMeta> for ScatterGatherInfo {
    /// Project the engine's scatter-gather execution metadata into the HTTP
    /// response shape. Single source for the five-field mapping so the
    /// `/sharded/feed` and `/sharded/search` handlers cannot drift.
    fn from(meta: crate::scatter_gather::ScatterGatherMeta) -> Self {
        Self {
            degraded: meta.degraded,
            unavailable_shards: meta.unavailable_shards,
            shards_queried: meta.shards_queried,
            elapsed_ms: meta.elapsed_ms,
            shard_deadline_ms: meta.shard_deadline_ms,
        }
    }
}

#[utoipa::path(
    get,
    path = "/sharded/feed",
    tag = "sharded",
    params(ShardedFeedQuery),
    responses(
        (status = 200, description = "Scatter-gather ranked feed", body = ShardedFeedResponse),
        (status = 400, description = "Invalid request"),
        (status = 401, description = "Missing or invalid API key"),
    ),
    security(("bearerAuth" = [])),
)]
pub async fn sharded_feed(
    State(state): State<Arc<ClusterState>>,
    Query(query): Query<ShardedFeedQuery>,
) -> std::result::Result<Json<ShardedFeedResponse>, ClusterAppError> {
    // Clamp the client-supplied limit at the trust boundary before fanning the
    // scatter-gather out to every shard (the local query struct has no
    // clamped_limit helper, so clamp inline against the shared MAX_LIMIT).
    let mut builder = Retrieve::builder()
        .profile(&query.profile)
        .limit(query.limit.min(MAX_LIMIT) as usize);
    if let Some(user_id) = query.user_id {
        builder = builder.for_user(user_id);
    }
    let retrieve = builder
        .build()
        .map_err(|e| ClusterAppError(ServerError::Tidal(e.into())))?;

    // The scatter-gather coordinator merges shard results and reads creator
    // metadata for coordinator-level diversity — all blocking. Offload the
    // whole coordination off the reactor so a slow shard never stalls it.
    let cluster = state.cluster_arc().map_err(ClusterAppError)?;
    let shards = state.shard_ids().map_err(ClusterAppError)?;
    let region_names = state.id_to_name_map().clone();
    let deadline_ms = query.deadline_ms;
    let (result, meta) = offload_cluster_read(move || {
        crate::scatter_gather::scatter_gather_retrieve(
            &cluster,
            &retrieve,
            &shards,
            &region_names,
            deadline_ms,
        )
    })
    .await?;

    Ok(Json(ShardedFeedResponse {
        items: feed_items(&result.items),
        total_candidates: result.total_candidates,
        scatter_gather: meta.into(),
    }))
}

/// `GET /sharded/search` query parameters.
#[derive(Deserialize, ToSchema, utoipa::IntoParams)]
#[into_params(parameter_in = Query)]
pub struct ShardedSearchQuery {
    /// Free-text query string.
    #[param(example = "jazz piano")]
    query: String,
    /// Optional user to personalize ranking for.
    #[serde(default)]
    user_id: Option<u64>,
    /// Page size; clamped to `MAX_LIMIT` (1000).
    #[serde(default = "default_limit")]
    #[param(example = 20)]
    limit: u32,
    /// Per-shard deadline in milliseconds for the scatter-gather.
    #[serde(default)]
    deadline_ms: Option<u64>,
}

impl ShardedSearchQuery {
    /// Free-text query string.
    #[must_use]
    pub fn query_text(&self) -> &str {
        &self.query
    }
    /// Personalization user, if any.
    #[must_use]
    pub const fn user_id(&self) -> Option<u64> {
        self.user_id
    }
    /// Page size clamped to [`MAX_LIMIT`] as a `u32` (SEARCH limit is `u32`).
    #[must_use]
    pub const fn clamped_limit_u32(&self) -> u32 {
        if self.limit > MAX_LIMIT {
            MAX_LIMIT
        } else {
            self.limit
        }
    }
    /// Client-supplied per-shard deadline budget.
    #[must_use]
    pub const fn deadline_ms(&self) -> Option<u64> {
        self.deadline_ms
    }
}

/// `GET /sharded/search` response body.
#[derive(Serialize, ToSchema)]
pub struct ShardedSearchResponse {
    /// Merged ranked search results across all shards.
    pub(crate) items: Vec<SearchItem>,
    /// Total candidates considered across shards.
    pub(crate) total_candidates: usize,
    /// Scatter-gather execution metadata.
    pub(crate) scatter_gather: ScatterGatherInfo,
}

#[utoipa::path(
    get,
    path = "/sharded/search",
    tag = "sharded",
    params(ShardedSearchQuery),
    responses(
        (status = 200, description = "Scatter-gather ranked search", body = ShardedSearchResponse),
        (status = 400, description = "Invalid request"),
        (status = 401, description = "Missing or invalid API key"),
    ),
    security(("bearerAuth" = [])),
)]
pub async fn sharded_search(
    State(state): State<Arc<ClusterState>>,
    Query(query): Query<ShardedSearchQuery>,
) -> std::result::Result<Json<ShardedSearchResponse>, ClusterAppError> {
    // Clamp at the trust boundary before scatter-gather (inline against the
    // shared MAX_LIMIT — the local sharded query struct has no helper).
    let mut builder = Search::builder()
        .query(&query.query)
        .limit(query.limit.min(MAX_LIMIT));
    if let Some(user_id) = query.user_id {
        builder = builder.for_user(user_id);
    }
    let search_query = builder
        .build()
        .map_err(|e| ClusterAppError(ServerError::Tidal(e.into())))?;

    // Offload the blocking scatter-gather coordination off the reactor (see
    // [`sharded_feed`] / [`offload_cluster_read`]).
    let cluster = state.cluster_arc().map_err(ClusterAppError)?;
    let shards = state.shard_ids().map_err(ClusterAppError)?;
    let region_names = state.id_to_name_map().clone();
    let deadline_ms = query.deadline_ms;
    let (result, meta) = offload_cluster_read(move || {
        crate::scatter_gather::scatter_gather_search(
            &cluster,
            &search_query,
            &shards,
            &region_names,
            deadline_ms,
        )
    })
    .await?;

    Ok(Json(ShardedSearchResponse {
        items: search_items(&result.items),
        total_candidates: result.total_candidates,
        scatter_gather: meta.into(),
    }))
}

// ── Blocking-work offload ────────────────────────────────────────────────────

/// Run a blocking READ-only cluster query (RETRIEVE / SEARCH) on the tokio
/// blocking pool, off the async reactor, and await its result.
///
/// Thin adapter over the shared [`offload_read`] that maps its [`ServerError`]
/// into a [`ClusterAppError`] for the cluster handlers. Both the standalone and
/// cluster read paths route through the same helper so they cannot drift.
async fn offload_cluster_read<F, T>(f: F) -> std::result::Result<T, ClusterAppError>
where
    F: FnOnce() -> Result<T> + Send + 'static,
    T: Send + 'static,
{
    offload_read(f).await.map_err(ClusterAppError)
}

// ── Error handling ─────────────────────────────────────────────────────────

pub struct ClusterAppError(pub ServerError);

impl IntoResponse for ClusterAppError {
    fn into_response(self) -> Response {
        let status = crate::router::status_from_error(&self.0);
        // For the multi-process routing errors, name the responsible node in the
        // body so a client can re-target the write/read itself (task 03 turns
        // these into transparent forwarding). Other errors keep the flat shape.
        let body = match &self.0 {
            ServerError::NotLeader {
                leader,
                http_addr,
                term,
                shard,
            } => serde_json::json!({
                "error": self.0.to_string(),
                "leader": leader,
                "leader_http_addr": http_addr,
                "term": term,
                "shard": shard,
            }),
            ServerError::LeaderUnreachable {
                leader,
                http_addr,
                cause,
            } => serde_json::json!({
                "error": self.0.to_string(),
                "leader": leader,
                "leader_http_addr": http_addr,
                "cause": cause,
            }),
            ServerError::NotLocal { region } => serde_json::json!({
                "error": self.0.to_string(),
                "region": region,
            }),
            ServerError::RegionUnreachable { region, cause } => serde_json::json!({
                "error": self.0.to_string(),
                "region": region,
                "cause": cause,
            }),
            ServerError::QuorumTimeout {
                seq,
                needed,
                confirmed,
                committed,
                laggards,
            } => serde_json::json!({
                "error": self.0.to_string(),
                "retryable": true,
                "seq": seq,
                "needed": needed,
                "confirmed": confirmed,
                "commit_index": committed,
                "laggards": laggards,
            }),
            _ => serde_json::json!({ "error": self.0.to_string() }),
        };
        (status, Json(body)).into_response()
    }
}