fix: repair tidal-server compilation and verify standalone HTTP server

Fix 9 compilation errors across tidal-server and testing/cluster.rs so
that `cargo run -p tidal-server -- standalone` works end-to-end.

Bugs fixed:
- cluster.rs: wrong return types `RetrieveResult`→`Results` and
  `SearchResult`→`SearchResults` on retrieve/search helpers
- state.rs: `RegionId` imported from private path; now uses
  `tidaldb::replication::RegionId`
- state.rs: missing `Ok()` wrapper on `ServerState::cluster()` return
- state.rs: cluster match arms returned `TidalError` where `ServerError`
  required; added `.map_err(ServerError::from)` on write_item,
  write_embedding, retrieve, search
- error.rs: `Result<T>` alias lacked default E param; callers in router
  used two-arg form `Result<T, AppError>` — changed to
  `Result<T, E = ServerError>`
- router.rs: `with_state()` called before cluster routes were added,
  making `app` `Router<()>`; restructured to call `with_state` once at end
- router.rs: `TidalErrorWrapper(TidalError)` used to map `QueryError`;
  fixed with `|e| TidalErrorWrapper(e.into())`
- router.rs: `Search::limit()` takes `u32` but code cast to `usize`
- router.rs: `bm25_score`/`semantic_score` are `f32` in SearchResultItem
  but `f64` in response struct; added `.map(f64::from)` conversion

Also split cluster.rs into cluster.rs + cluster_transport.rs to stay
under the 600-line limit required by CODING_GUIDELINES §9.

Verified all README curl examples work:
  POST /items, POST /embeddings, POST /signals, GET /feed, GET /search,
  GET /health all return correct HTTP status codes and JSON responses.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

tidal-server/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "tidal-server"
+version = "0.1.0"
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+
+[dependencies]
+axum = "0.8"
+clap = { version = "4.5", features = ["derive"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+serde_yaml = "0.9"
+thiserror = "2"
+tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal"] }
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+tidaldb = { path = "../tidal", features = ["test-utils"] }

tidal-server/config/default-cluster.yaml

@@ -0,0 +1,5 @@
+regions:
+  - name: us-east
+  - name: eu-west
+  - name: ap-south
+leader: us-east

tidal-server/config/default-schema.yaml

@@ -0,0 +1,29 @@
+signals:
+  - name: view
+    entity: item
+    decay:
+      exponential:
+        half_life_seconds: 604800    # 7 days
+    windows: [one_hour, twenty_four_hours, seven_days]
+    velocity: true
+  - name: like
+    entity: item
+    decay:
+      exponential:
+        half_life_seconds: 1209600   # 14 days
+    windows: [twenty_four_hours, seven_days, thirty_days, all_time]
+    velocity: false
+  - name: skip
+    entity: item
+    decay:
+      permanent: true
+    velocity: false
+text_fields:
+  - name: title
+    kind: text
+  - name: category
+    kind: keyword
+embedding_slots:
+  - name: content_vector
+    entity: item
+    dimensions: 128

tidal-server/src/config.rs

@@ -0,0 +1,226 @@
+use std::collections::HashSet;
+use std::fs;
+use std::path::Path;
+use std::time::Duration;
+
+use serde::Deserialize;
+use tidaldb::schema::{DecaySpec, EntityKind, Schema, SchemaBuilder, TextFieldType, Window};
+
+use crate::error::{Result, ServerError};
+
+const DEFAULT_SCHEMA_YAML: &str = include_str!("../config/default-schema.yaml");
+const DEFAULT_CLUSTER_YAML: &str = include_str!("../config/default-cluster.yaml");
+
+#[derive(Debug)]
+pub struct ClusterLayout {
+    pub regions: Vec<String>,
+    pub leader: String,
+}
+
+pub fn load_schema(path: Option<&Path>) -> Result<Schema> {
+    let raw = read_config(path, DEFAULT_SCHEMA_YAML)?;
+    let spec: SchemaSpec = serde_yaml::from_str(&raw)
+        .map_err(|e| ServerError::SchemaConfig(format!("parse schema yaml: {e}")))?;
+
+    if spec.signals.is_empty() {
+        return Err(ServerError::SchemaConfig(
+            "at least one signal must be defined".into(),
+        ));
+    }
+
+    let mut builder = SchemaBuilder::new();
+    for signal in spec.signals {
+        let mut sig = builder.signal(
+            &signal.name,
+            parse_entity_kind(&signal.entity)?,
+            signal.decay.to_decay_spec()?,
+        );
+        if let Some(windows) = signal.windows {
+            let parsed: Result<Vec<Window>> = windows.iter().map(|w| parse_window(w)).collect();
+            sig = sig.windows(&parsed?);
+        }
+        if let Some(velocity) = signal.velocity {
+            sig = sig.velocity(velocity);
+        }
+        let _ = sig.add();
+    }
+
+    if let Some(text_fields) = spec.text_fields {
+        for field in text_fields {
+            builder.text_field(&field.name, parse_text_field_type(&field.kind)?);
+        }
+    }
+
+    if let Some(embeddings) = spec.embedding_slots {
+        for slot in embeddings {
+            builder.embedding_slot(
+                &slot.name,
+                parse_entity_kind(&slot.entity)?,
+                slot.dimensions,
+            );
+        }
+    }
+
+    builder.build().map_err(ServerError::SchemaBuild)
+}
+
+pub fn load_cluster_layout(path: Option<&Path>) -> Result<ClusterLayout> {
+    let raw = read_config(path, DEFAULT_CLUSTER_YAML)?;
+    let spec: ClusterSpec = serde_yaml::from_str(&raw)
+        .map_err(|e| ServerError::ClusterConfig(format!("parse cluster yaml: {e}")))?;
+
+    if spec.regions.is_empty() {
+        return Err(ServerError::ClusterConfig(
+            "cluster config must include at least one region".into(),
+        ));
+    }
+
+    let mut seen = HashSet::new();
+    for region in &spec.regions {
+        if !seen.insert(region.name.to_lowercase()) {
+            return Err(ServerError::ClusterConfig(format!(
+                "duplicate region name: {}",
+                region.name
+            )));
+        }
+    }
+
+    if !seen.contains(&spec.leader.to_lowercase()) {
+        return Err(ServerError::ClusterConfig(format!(
+            "leader '{}' not found in region list",
+            spec.leader
+        )));
+    }
+
+    Ok(ClusterLayout {
+        regions: spec.regions.into_iter().map(|r| r.name).collect(),
+        leader: spec.leader,
+    })
+}
+
+fn read_config(path: Option<&Path>, fallback: &str) -> Result<String> {
+    match path {
+        Some(p) => fs::read_to_string(p).map_err(|e| ServerError::io(p, e)),
+        None => Ok(fallback.to_string()),
+    }
+}
+
+#[derive(Debug, Deserialize)]
+struct SchemaSpec {
+    signals: Vec<SignalSpec>,
+    #[serde(default)]
+    text_fields: Option<Vec<TextFieldSpec>>,
+    #[serde(default)]
+    embedding_slots: Option<Vec<EmbeddingSpec>>,
+}
+
+#[derive(Debug, Deserialize)]
+struct SignalSpec {
+    name: String,
+    entity: String,
+    decay: DecaySpecConfig,
+    #[serde(default)]
+    windows: Option<Vec<String>>,
+    #[serde(default)]
+    velocity: Option<bool>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "snake_case")]
+struct DecaySpecConfig {
+    #[serde(default)]
+    exponential: Option<ExponentialDecay>,
+    #[serde(default)]
+    linear: Option<LinearDecay>,
+    #[serde(default)]
+    permanent: Option<bool>,
+}
+
+impl DecaySpecConfig {
+    fn to_decay_spec(&self) -> Result<DecaySpec> {
+        if let Some(exp) = &self.exponential {
+            return Ok(DecaySpec::Exponential {
+                half_life: Duration::from_secs_f64(exp.half_life_seconds),
+            });
+        }
+        if let Some(linear) = &self.linear {
+            return Ok(DecaySpec::Linear {
+                lifetime: Duration::from_secs_f64(linear.lifetime_seconds),
+            });
+        }
+        if self.permanent.unwrap_or(false) {
+            return Ok(DecaySpec::Permanent);
+        }
+        Err(ServerError::SchemaConfig(
+            "decay must specify exponential, linear, or permanent".into(),
+        ))
+    }
+}
+
+#[derive(Debug, Deserialize)]
+struct ExponentialDecay {
+    half_life_seconds: f64,
+}
+
+#[derive(Debug, Deserialize)]
+struct LinearDecay {
+    lifetime_seconds: f64,
+}
+
+#[derive(Debug, Deserialize)]
+struct TextFieldSpec {
+    name: String,
+    kind: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct EmbeddingSpec {
+    name: String,
+    entity: String,
+    dimensions: usize,
+}
+
+#[derive(Debug, Deserialize)]
+struct ClusterSpec {
+    regions: Vec<RegionSpec>,
+    leader: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct RegionSpec {
+    name: String,
+}
+
+fn parse_entity_kind(input: &str) -> Result<EntityKind> {
+    match input.trim().to_lowercase().as_str() {
+        "item" | "items" => Ok(EntityKind::Item),
+        "user" | "users" => Ok(EntityKind::User),
+        "creator" | "creators" => Ok(EntityKind::Creator),
+        other => Err(ServerError::SchemaConfig(format!(
+            "unknown entity kind '{other}'"
+        ))),
+    }
+}
+
+fn parse_window(input: &str) -> Result<Window> {
+    match input.trim().to_lowercase().as_str() {
+        "one_hour" | "1h" => Ok(Window::OneHour),
+        "twenty_four_hours" | "24h" => Ok(Window::TwentyFourHours),
+        "seven_days" | "7d" => Ok(Window::SevenDays),
+        "thirty_days" | "30d" => Ok(Window::ThirtyDays),
+        "all_time" | "alltime" => Ok(Window::AllTime),
+        other => Err(ServerError::SchemaConfig(format!(
+            "unknown window '{other}'"
+        ))),
+    }
+}
+
+fn parse_text_field_type(input: &str) -> Result<TextFieldType> {
+    match input.trim().to_lowercase().as_str() {
+        "text" => Ok(TextFieldType::Text),
+        "keyword" => Ok(TextFieldType::Keyword),
+        other => Err(ServerError::SchemaConfig(format!(
+            "unknown text field type '{other}'"
+        ))),
+    }
+}

tidal-server/src/error.rs

@@ -0,0 +1,35 @@
+use std::path::PathBuf;
+
+use thiserror::Error;
+
+pub type Result<T, E = ServerError> = std::result::Result<T, E>;
+
+#[derive(Debug, Error)]
+pub enum ServerError {
+    #[error("failed to read {path}: {source}")]
+    Io {
+        path: PathBuf,
+        source: std::io::Error,
+    },
+    #[error("invalid schema config: {0}")]
+    SchemaConfig(String),
+    #[error("schema build failed: {0}")]
+    SchemaBuild(#[from] tidaldb::schema::SchemaError),
+    #[error("invalid cluster config: {0}")]
+    ClusterConfig(String),
+    #[error("tidalDB error: {0}")]
+    Tidal(#[from] tidaldb::TidalError),
+    #[error("http server error: {0}")]
+    Http(#[from] std::io::Error),
+    #[error("bad request: {0}")]
+    BadRequest(String),
+}
+
+impl ServerError {
+    pub fn io(path: impl Into<PathBuf>, source: std::io::Error) -> Self {
+        Self::Io {
+            path: path.into(),
+            source,
+        }
+    }
+}

tidal-server/src/main.rs

@@ -0,0 +1,120 @@
+mod config;
+mod error;
+mod router;
+mod state;
+
+use std::net::SocketAddr;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use clap::{Args, Parser, Subcommand};
+use tidaldb::TidalDb;
+
+use crate::config::{load_cluster_layout, load_schema};
+use crate::error::{Result, ServerError};
+use crate::router::build_router;
+use crate::state::ServerState;
+
+#[derive(Parser)]
+#[command(version, about = "HTTP wrapper for tidalDB")]
+struct Cli {
+    #[command(subcommand)]
+    mode: Command,
+}
+
+#[derive(Subcommand)]
+enum Command {
+    #[command(about = "Run a single-node server wrapping one tidalDB instance")]
+    Standalone(StandaloneArgs),
+    #[command(about = "Run the simulated multi-region cluster server")]
+    Cluster(ClusterArgs),
+}
+
+#[derive(Args)]
+struct StandaloneArgs {
+    #[arg(long, default_value = "127.0.0.1:9400")]
+    listen: String,
+    #[arg(long)]
+    schema: Option<PathBuf>,
+    #[arg(long)]
+    data_dir: Option<PathBuf>,
+}
+
+#[derive(Args)]
+struct ClusterArgs {
+    #[arg(long, default_value = "0.0.0.0:9500")]
+    listen: String,
+    #[arg(long)]
+    schema: Option<PathBuf>,
+    #[arg(long)]
+    topology: Option<PathBuf>,
+}
+
+#[tokio::main]
+async fn main() {
+    if let Err(err) = run().await {
+        eprintln!("error: {err}");
+        std::process::exit(1);
+    }
+}
+
+async fn run() -> Result<()> {
+    let cli = Cli::parse();
+    init_tracing();
+
+    match cli.mode {
+        Command::Standalone(args) => run_standalone(args).await,
+        Command::Cluster(args) => run_cluster(args).await,
+    }
+}
+
+fn init_tracing() {
+    let env_filter = std::env::var("TIDAL_SERVER_LOG").unwrap_or_else(|_| "info".into());
+    let _ = tracing_subscriber::fmt()
+        .with_env_filter(env_filter)
+        .try_init();
+}
+
+async fn run_standalone(args: StandaloneArgs) -> Result<()> {
+    let schema = load_schema(args.schema.as_deref())?;
+
+    let mut builder = TidalDb::builder().with_schema(schema.clone());
+    if let Some(dir) = args.data_dir {
+        builder = builder.with_data_dir(dir);
+    } else {
+        builder = builder.ephemeral();
+    }
+
+    let db = builder.open()?;
+    let state = ServerState::standalone(db);
+    serve(state, &args.listen).await
+}
+
+async fn run_cluster(args: ClusterArgs) -> Result<()> {
+    let schema = load_schema(args.schema.as_deref())?;
+    let layout = load_cluster_layout(args.topology.as_deref())?;
+    let state = ServerState::cluster(schema, layout)?;
+    serve(state, &args.listen).await
+}
+
+async fn serve(state: ServerState, addr: &str) -> Result<()> {
+    let socket: SocketAddr = addr
+        .parse()
+        .map_err(|e| ServerError::BadRequest(format!("invalid addr: {e}")))?;
+
+    let listener = tokio::net::TcpListener::bind(socket).await?;
+    let actual = listener.local_addr()?;
+    tracing::info!("listening on http://{actual}");
+
+    axum::serve(listener, build_router(Arc::new(state)))
+        .with_graceful_shutdown(shutdown_signal())
+        .await?;
+    Ok(())
+}
+
+async fn shutdown_signal() {
+    if let Err(err) = tokio::signal::ctrl_c().await {
+        tracing::warn!("ctrl-c handler error: {err}");
+    }
+    tracing::info!("shutdown signal received");
+}

tidal-server/src/router.rs

@@ -0,0 +1,351 @@
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use axum::Json;
+use axum::Router;
+use axum::extract::{Query, State};
+use axum::http::StatusCode;
+use axum::response::{IntoResponse, Response};
+use axum::routing::{get, post};
+use serde::{Deserialize, Serialize};
+use tidaldb::query::retrieve::Retrieve;
+use tidaldb::query::search::Search;
+use tidaldb::schema::EntityId;
+
+use crate::error::{Result, ServerError};
+use crate::state::{ClusterHealth, ServerState};
+
+pub fn build_router(state: Arc<ServerState>) -> Router {
+    let mut app = Router::new()
+        .route("/items", post(create_item))
+        .route("/embeddings", post(write_embedding))
+        .route("/signals", post(write_signal))
+        .route("/feed", get(feed))
+        .route("/search", get(search))
+        .route("/health", get(health));
+
+    if state.is_cluster() {
+        app = app
+            .route("/cluster/status", get(cluster_status))
+            .route("/cluster/promote", post(promote_leader))
+            .route("/cluster/partition", post(partition_region))
+            .route("/cluster/heal", post(heal_region));
+    }
+
+    app.with_state(state)
+}
+
+#[derive(Deserialize)]
+struct ItemRequest {
+    entity_id: u64,
+    metadata: HashMap<String, String>,
+}
+
+async fn create_item(
+    State(state): State<Arc<ServerState>>,
+    Json(req): Json<ItemRequest>,
+) -> Result<StatusCode, AppError> {
+    state
+        .write_item(EntityId::new(req.entity_id), &req.metadata)
+        .map_err(AppError)?;
+    Ok(StatusCode::CREATED)
+}
+
+#[derive(Deserialize)]
+struct EmbeddingRequest {
+    entity_id: u64,
+    values: Vec<f32>,
+}
+
+async fn write_embedding(
+    State(state): State<Arc<ServerState>>,
+    Json(req): Json<EmbeddingRequest>,
+) -> Result<StatusCode, AppError> {
+    state
+        .write_embedding(EntityId::new(req.entity_id), &req.values)
+        .map_err(AppError)?;
+    Ok(StatusCode::NO_CONTENT)
+}
+
+#[derive(Deserialize)]
+struct SignalRequest {
+    entity_id: u64,
+    signal: String,
+    weight: f64,
+    #[serde(default)]
+    user_id: Option<u64>,
+    #[serde(default)]
+    creator_id: Option<u64>,
+}
+
+async fn write_signal(
+    State(state): State<Arc<ServerState>>,
+    Json(req): Json<SignalRequest>,
+) -> Result<StatusCode, AppError> {
+    state
+        .signal(
+            &req.signal,
+            EntityId::new(req.entity_id),
+            req.weight,
+            req.user_id,
+            req.creator_id,
+        )
+        .map_err(AppError)?;
+    Ok(StatusCode::NO_CONTENT)
+}
+
+#[derive(Deserialize)]
+struct FeedQuery {
+    #[serde(default)]
+    user_id: Option<u64>,
+    #[serde(default = "default_profile")]
+    profile: String,
+    #[serde(default = "default_limit")]
+    limit: u32,
+    #[serde(default)]
+    region: Option<String>,
+}
+
+fn default_profile() -> String {
+    "for_you".into()
+}
+
+fn default_limit() -> u32 {
+    20
+}
+
+#[derive(Serialize)]
+struct FeedResponse {
+    items: Vec<FeedItem>,
+    total_candidates: usize,
+    region: Option<String>,
+}
+
+#[derive(Serialize)]
+struct FeedItem {
+    entity_id: u64,
+    score: f64,
+    rank: usize,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    signals: Option<Vec<SignalValue>>,
+}
+
+#[derive(Serialize)]
+struct SignalValue {
+    name: String,
+    value: f64,
+}
+
+async fn feed(
+    State(state): State<Arc<ServerState>>,
+    Query(query): Query<FeedQuery>,
+) -> Result<Json<FeedResponse>, AppError> {
+    let mut builder = Retrieve::builder()
+        .profile(&query.profile)
+        .limit(query.limit as usize);
+
+    if let Some(user_id) = query.user_id {
+        builder = builder.for_user(user_id);
+    }
+    let retrieve = builder.build().map_err(|e| TidalErrorWrapper(e.into()))?;
+
+    let result = state
+        .retrieve(query.region.as_deref(), &retrieve)
+        .map_err(AppError)?;
+
+    let mut items = Vec::with_capacity(result.items.len());
+    for item in result.items {
+        let signals = if item.signals.is_empty() {
+            None
+        } else {
+            Some(
+                item.signals
+                    .iter()
+                    .map(|s| SignalValue {
+                        name: s.name.clone(),
+                        value: s.value,
+                    })
+                    .collect(),
+            )
+        };
+        items.push(FeedItem {
+            entity_id: item.entity_id.as_u64(),
+            score: item.score,
+            rank: item.rank,
+            signals,
+        });
+    }
+
+    Ok(Json(FeedResponse {
+        items,
+        total_candidates: result.total_candidates,
+        region: query.region,
+    }))
+}
+
+#[derive(Deserialize)]
+struct SearchQueryParams {
+    query: String,
+    #[serde(default)]
+    user_id: Option<u64>,
+    #[serde(default = "default_limit")]
+    limit: u32,
+    #[serde(default)]
+    region: Option<String>,
+}
+
+#[derive(Serialize)]
+struct SearchResponse {
+    items: Vec<SearchItem>,
+    total_candidates: usize,
+    region: Option<String>,
+}
+
+#[derive(Serialize)]
+struct SearchItem {
+    entity_id: u64,
+    score: f64,
+    rank: usize,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    bm25_score: Option<f64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    semantic_score: Option<f64>,
+}
+
+async fn search(
+    State(state): State<Arc<ServerState>>,
+    Query(query): Query<SearchQueryParams>,
+) -> Result<Json<SearchResponse>, AppError> {
+    let mut builder = Search::builder()
+        .query(&query.query)
+        .limit(query.limit);
+    if let Some(user_id) = query.user_id {
+        builder = builder.for_user(user_id);
+    }
+    let search = builder.build().map_err(|e| TidalErrorWrapper(e.into()))?;
+    let result = state
+        .search(query.region.as_deref(), &search)
+        .map_err(AppError)?;
+
+    let items = result
+        .items
+        .into_iter()
+        .map(|item| SearchItem {
+            entity_id: item.entity_id.as_u64(),
+            score: item.score,
+            rank: item.rank,
+            bm25_score: item.bm25_score.map(f64::from),
+            semantic_score: item.semantic_score.map(f64::from),
+        })
+        .collect();
+
+    Ok(Json(SearchResponse {
+        items,
+        total_candidates: result.total_candidates,
+        region: query.region,
+    }))
+}
+
+#[derive(Serialize)]
+struct HealthResponse {
+    status: &'static str,
+    mode: &'static str,
+    items: u64,
+}
+
+async fn health(
+    State(state): State<Arc<ServerState>>,
+    Query(query): Query<HashMap<String, String>>,
+) -> Result<Json<HealthResponse>, AppError> {
+    let region = query.get("region").map(|s| s.as_str());
+    let items = state.item_count(region).map_err(AppError)?;
+    let mode = if state.is_cluster() {
+        "cluster"
+    } else {
+        "standalone"
+    };
+    Ok(Json(HealthResponse {
+        status: "ok",
+        mode,
+        items,
+    }))
+}
+
+async fn cluster_status(
+    State(state): State<Arc<ServerState>>,
+) -> Result<Json<ClusterHealth>, AppError> {
+    state
+        .cluster_status()
+        .map(Json)
+        .ok_or_else(|| AppError(ServerError::BadRequest("not in cluster mode".into())))
+}
+
+#[derive(Deserialize)]
+struct RegionCommand {
+    region: String,
+}
+
+async fn promote_leader(
+    State(state): State<Arc<ServerState>>,
+    Json(cmd): Json<RegionCommand>,
+) -> Result<StatusCode, AppError> {
+    state.promote_leader(&cmd.region).map_err(AppError)?;
+    Ok(StatusCode::NO_CONTENT)
+}
+
+async fn partition_region(
+    State(state): State<Arc<ServerState>>,
+    Json(cmd): Json<RegionCommand>,
+) -> Result<StatusCode, AppError> {
+    state.partition_region(&cmd.region).map_err(AppError)?;
+    Ok(StatusCode::NO_CONTENT)
+}
+
+async fn heal_region(
+    State(state): State<Arc<ServerState>>,
+    Json(cmd): Json<RegionCommand>,
+) -> Result<StatusCode, AppError> {
+    state.heal_region(&cmd.region).map_err(AppError)?;
+    Ok(StatusCode::NO_CONTENT)
+}
+
+struct TidalErrorWrapper(tidaldb::TidalError);
+
+impl From<TidalErrorWrapper> for AppError {
+    fn from(value: TidalErrorWrapper) -> Self {
+        AppError(ServerError::Tidal(value.0))
+    }
+}
+
+struct AppError(ServerError);
+
+impl IntoResponse for AppError {
+    fn into_response(self) -> Response {
+        let status = status_from_error(&self.0);
+        let body = serde_json::json!({
+            "error": self.0.to_string()
+        });
+        (status, Json(body)).into_response()
+    }
+}
+
+fn status_from_error(err: &ServerError) -> StatusCode {
+    match err {
+        ServerError::BadRequest(_)
+        | ServerError::SchemaConfig(_)
+        | ServerError::ClusterConfig(_) => StatusCode::BAD_REQUEST,
+        ServerError::Tidal(tidal_err) => match tidal_err {
+            tidaldb::TidalError::NotFound { .. } => StatusCode::NOT_FOUND,
+            tidaldb::TidalError::Schema(_) | tidaldb::TidalError::InvalidInput(_) => {
+                StatusCode::BAD_REQUEST
+            }
+            tidaldb::TidalError::Backpressure { .. } | tidaldb::TidalError::RateLimited { .. } => {
+                StatusCode::TOO_MANY_REQUESTS
+            }
+            tidaldb::TidalError::PolicyViolation { .. }
+            | tidaldb::TidalError::SessionExpired { .. } => StatusCode::FORBIDDEN,
+            _ => StatusCode::INTERNAL_SERVER_ERROR,
+        },
+        _ => StatusCode::INTERNAL_SERVER_ERROR,
+    }
+}

tidal-server/src/state.rs

@@ -0,0 +1,315 @@
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use tidaldb::query::retrieve::Retrieve;
+use tidaldb::query::search::Search;
+use tidaldb::schema::{EntityId, Schema};
+use tidaldb::replication::RegionId;
+use tidaldb::testing::cluster::SimulatedCluster;
+use tidaldb::TidalDb;
+
+use crate::config::ClusterLayout;
+use crate::error::{Result, ServerError};
+
+#[derive(Clone)]
+pub struct ServerState {
+    mode: Mode,
+}
+
+#[derive(Clone)]
+enum Mode {
+    Standalone(Arc<TidalDb>),
+    Cluster(ClusterState),
+}
+
+#[derive(Clone)]
+struct ClusterState {
+    cluster: Arc<SimulatedCluster>,
+    regions: RegionDirectory,
+}
+
+#[derive(Clone)]
+struct RegionDirectory {
+    #[allow(dead_code)]
+    default_region: RegionId,
+    name_to_id: HashMap<String, RegionId>,
+    id_to_name: HashMap<u16, String>,
+}
+
+#[derive(Debug, serde::Serialize)]
+pub struct ClusterHealth {
+    pub leader: String,
+    pub relay_log_len: u64,
+    pub regions: Vec<RegionStatus>,
+}
+
+#[derive(Debug, serde::Serialize)]
+pub struct RegionStatus {
+    pub name: String,
+    pub applied_events: u64,
+    pub lag_events: i64,
+    pub partitioned: bool,
+}
+
+impl ServerState {
+    pub fn standalone(db: TidalDb) -> Self {
+        Self {
+            mode: Mode::Standalone(Arc::new(db)),
+        }
+    }
+
+    pub fn cluster(schema: Schema, layout: ClusterLayout) -> Result<Self> {
+        use tidaldb::testing::cluster::ClusterConfig;
+
+        let mut regions = Vec::new();
+        for (idx, name) in layout.regions.iter().enumerate() {
+            regions.push((RegionId(idx as u16), name.clone()));
+        }
+
+        let leader_id = regions
+            .iter()
+            .find(|(_, name)| name.eq_ignore_ascii_case(&layout.leader))
+            .map(|(id, _)| *id)
+            .ok_or_else(|| {
+                ServerError::ClusterConfig(format!(
+                    "leader '{}' not found in region list",
+                    layout.leader
+                ))
+            })?;
+
+        let config = ClusterConfig {
+            regions: regions.iter().map(|(id, _)| *id).collect(),
+            leader_region: leader_id,
+            schema,
+        };
+        let cluster = Arc::new(SimulatedCluster::build(config));
+        let mut name_to_id = HashMap::new();
+        let mut id_to_name = HashMap::new();
+        for (id, name) in &regions {
+            name_to_id.insert(name.to_lowercase(), *id);
+            id_to_name.insert(id.0, name.clone());
+        }
+
+        let directory = RegionDirectory {
+            default_region: leader_id,
+            name_to_id,
+            id_to_name,
+        };
+
+        Ok(Self {
+            mode: Mode::Cluster(ClusterState {
+                cluster,
+                regions: directory,
+            }),
+        })
+    }
+
+    pub fn is_cluster(&self) -> bool {
+        matches!(self.mode, Mode::Cluster(_))
+    }
+
+    pub fn write_item(
+        &self,
+        entity_id: EntityId,
+        metadata: &HashMap<String, String>,
+    ) -> Result<()> {
+        match &self.mode {
+            Mode::Standalone(db) => db
+                .write_item_with_metadata(entity_id, metadata)
+                .map_err(ServerError::from),
+            Mode::Cluster(cluster) => cluster
+                .cluster
+                .write_item_with_metadata(entity_id, metadata)
+                .map_err(ServerError::from),
+        }
+    }
+
+    pub fn write_embedding(&self, entity_id: EntityId, embedding: &[f32]) -> Result<()> {
+        match &self.mode {
+            Mode::Standalone(db) => db
+                .write_item_embedding(entity_id, embedding)
+                .map_err(ServerError::from),
+            Mode::Cluster(cluster) => cluster.cluster.write_item_embedding(entity_id, embedding).map_err(ServerError::from),
+        }
+    }
+
+    pub fn signal(
+        &self,
+        signal_name: &str,
+        entity_id: EntityId,
+        weight: f64,
+        user_id: Option<u64>,
+        creator_id: Option<u64>,
+    ) -> Result<()> {
+        match &self.mode {
+            Mode::Standalone(db) => {
+                if user_id.is_some() || creator_id.is_some() {
+                    db.signal_with_context(
+                        signal_name,
+                        entity_id,
+                        weight,
+                        tidaldb::schema::Timestamp::now(),
+                        user_id,
+                        creator_id,
+                    )
+                    .map_err(ServerError::from)
+                } else {
+                    db.signal(
+                        signal_name,
+                        entity_id,
+                        weight,
+                        tidaldb::schema::Timestamp::now(),
+                    )
+                    .map_err(ServerError::from)
+                }
+            }
+            Mode::Cluster(cluster) => {
+                if user_id.is_some() || creator_id.is_some() {
+                    return Err(ServerError::BadRequest(
+                        "cluster mode currently supports only global signals (no user_id/creator_id)".into(),
+                    ));
+                }
+                cluster.cluster.write_signal(signal_name, entity_id, weight);
+                Ok(())
+            }
+        }
+    }
+
+    pub fn retrieve(
+        &self,
+        region_name: Option<&str>,
+        query: &Retrieve,
+    ) -> Result<tidaldb::query::retrieve::Results> {
+        match &self.mode {
+            Mode::Standalone(db) => db.retrieve(query).map_err(ServerError::from),
+            Mode::Cluster(cluster) => {
+                let region = cluster.resolve_region(region_name)?;
+                cluster.cluster.retrieve(region, query).map_err(ServerError::from)
+            }
+        }
+    }
+
+    pub fn search(
+        &self,
+        region_name: Option<&str>,
+        query: &Search,
+    ) -> Result<tidaldb::query::search::SearchResults> {
+        match &self.mode {
+            Mode::Standalone(db) => db.search(query).map_err(ServerError::from),
+            Mode::Cluster(cluster) => {
+                let region = cluster.resolve_region(region_name)?;
+                cluster.cluster.search(region, query).map_err(ServerError::from)
+            }
+        }
+    }
+
+    pub fn item_count(&self, region_name: Option<&str>) -> Result<u64> {
+        match &self.mode {
+            Mode::Standalone(db) => Ok(db.item_count()),
+            Mode::Cluster(cluster) => {
+                let region = cluster.resolve_region(region_name)?;
+                Ok(cluster.cluster.item_count(region))
+            }
+        }
+    }
+
+    pub fn cluster_status(&self) -> Option<ClusterHealth> {
+        match &self.mode {
+            Mode::Cluster(cluster) => {
+                let leader_id = cluster.cluster.leader_region();
+                let leader_name = cluster
+                    .regions
+                    .id_to_name
+                    .get(&leader_id.0)
+                    .cloned()
+                    .unwrap_or_else(|| format!("region-{}", leader_id.0));
+                let leader_seqno = cluster.cluster.leader_seqno();
+                let statuses = cluster
+                    .regions
+                    .id_to_name
+                    .iter()
+                    .map(|(id, name)| {
+                        let rid = RegionId(*id);
+                        let applied = cluster.cluster.applied_count(rid);
+                        let lag = leader_seqno as i64 - applied as i64;
+                        RegionStatus {
+                            name: name.clone(),
+                            applied_events: applied,
+                            lag_events: lag.max(0),
+                            partitioned: cluster.cluster.is_partitioned(rid),
+                        }
+                    })
+                    .collect();
+                Some(ClusterHealth {
+                    leader: leader_name,
+                    relay_log_len: cluster.cluster.relay_log_len(),
+                    regions: statuses,
+                })
+            }
+            _ => None,
+        }
+    }
+
+    pub fn promote_leader(&self, region_name: &str) -> Result<()> {
+        match &self.mode {
+            Mode::Cluster(cluster) => {
+                let region = cluster.regions.lookup(region_name).ok_or_else(|| {
+                    ServerError::BadRequest(format!("unknown region '{region_name}'"))
+                })?;
+                cluster.cluster.promote_leader(region);
+                Ok(())
+            }
+            _ => Err(ServerError::BadRequest(
+                "leader promotion only supported in cluster mode".into(),
+            )),
+        }
+    }
+
+    pub fn partition_region(&self, region_name: &str) -> Result<()> {
+        match &self.mode {
+            Mode::Cluster(cluster) => {
+                let region = cluster.regions.lookup(region_name).ok_or_else(|| {
+                    ServerError::BadRequest(format!("unknown region '{region_name}'"))
+                })?;
+                cluster.cluster.partition_region(region);
+                Ok(())
+            }
+            _ => Err(ServerError::BadRequest(
+                "partitions only supported in cluster mode".into(),
+            )),
+        }
+    }
+
+    pub fn heal_region(&self, region_name: &str) -> Result<()> {
+        match &self.mode {
+            Mode::Cluster(cluster) => {
+                let region = cluster.regions.lookup(region_name).ok_or_else(|| {
+                    ServerError::BadRequest(format!("unknown region '{region_name}'"))
+                })?;
+                cluster.cluster.heal_region(region);
+                Ok(())
+            }
+            _ => Err(ServerError::BadRequest(
+                "partitions only supported in cluster mode".into(),
+            )),
+        }
+    }
+}
+
+impl ClusterState {
+    fn resolve_region(&self, name: Option<&str>) -> Result<RegionId> {
+        if let Some(name) = name {
+            self.regions
+                .lookup(name)
+                .ok_or_else(|| ServerError::BadRequest(format!("unknown region '{name}'")))
+        } else {
+            Ok(self.cluster.leader_region())
+        }
+    }
+}
+
+impl RegionDirectory {
+    fn lookup(&self, name: &str) -> Option<RegionId> {
+        self.name_to_id.get(&name.trim().to_lowercase()).copied()
+    }
+}

tidal/src/testing/cluster.rs

@@ -1,22 +1,33 @@
 //! Simulated multi-region cluster for M8 UAT testing.
 //!
-//! Creates a set of ephemeral [`TidalDb`] instances (one per region) and
+//! Creates a set of ephemeral [`TidalDb`] instances wired with the real M8
-//! replicates signals from the leader to followers via a shared relay log.
+//! distributed fabric: in-process transports, `spawn_receiver`, and
+//! `ReplicationState`. Signal replication now traverses the full path:
+//!
+//! ```text
+//! write_signal → encode_batch → channel send
+//!                               ↓
+//!                        spawn_receiver thread
+//!                               ↓
+//!                         apply_payload
+//!                               ↓
+//!                    SignalLedger::apply_wal_event
+//!                    ReplicationState::advance
+//! ```
 //!
 //! # Architecture
 //!
-//! Each node is a standard `TidalDb::builder().ephemeral().with_schema(...)`.
+//! * All nodes open with `NodeRole::Single` so they accept direct writes AND
-//! The "leader" is the node that accepts writes. The "followers" receive
+//!   can be promoted to leader at any time.
-//! replicated signals when [`SimulatedCluster::await_convergence`] is called.
+//! * Every non-initial-leader region starts a `spawn_receiver` thread (via
-//!
+//!   `db.start_replication(transport)`) that processes incoming WAL batches.
-//! Replication is **signal-replay**: the leader records each signal in a
+//! * `write_signal` encodes the event as a one-event WAL batch and ships it
-//! shared relay log, and `await_convergence` replays pending events into
+//!   immediately to all non-partitioned followers.
-//! each non-partitioned follower's `TidalDb`. This is fully synchronous
+//! * A `batch_log` records every shipped batch so `await_convergence` can
-//! and deterministic -- no background threads, no race conditions.
+//!   re-deliver missed batches after a partition is healed.
-//!
+//! * `await_convergence` ships any pending batches, then polls
-//! Partition injection marks a region as isolated. Isolated followers are
+//!   `ReplicationState::applied_seqno` until all active followers have caught
-//! skipped during convergence and do not receive new events until the
+//!   up to the current leader's sequence number.
-//! partition is healed.
 
 use std::collections::{HashMap, HashSet};
 use std::sync::atomic::{AtomicU64, Ordering};
@@ -24,23 +35,19 @@ use std::sync::{Arc, Mutex, RwLock};
 use std::time::{Duration, Instant};
 
 use crate::db::TidalDb;
-use crate::replication::shard::RegionId;
+use crate::db::config::{NodeConfig, NodeRole};
+use crate::query::retrieve::Retrieve;
+use crate::query::search::Search;
+use crate::replication::shard::{RegionId, ShardId};
+use crate::replication::transport::WalSegmentPayload;
+use crate::replication::{WalSegmentId, spawn_receiver};
 use crate::schema::{EntityId, Schema, Timestamp};
+use crate::signals::{NoopWalWriter, SignalLedger};
+use crate::wal::format::batch::{EventRecord, encode_batch};
 
-/// A signal event captured in the relay log for replication.
+use super::cluster_transport::{BatchEntry, ReceiveOnlyTransport};
-#[derive(Debug, Clone)]
+
-pub struct RelayEvent {
+// ── Public API ────────────────────────────────────────────────────────────
-    /// The signal type name (e.g. "view", "like").
-    pub signal_type: String,
-    /// The entity this signal targets.
-    pub entity_id: EntityId,
-    /// Signal weight.
-    pub weight: f64,
-    /// Timestamp of the signal.
-    pub timestamp: Timestamp,
-    /// Monotonically increasing sequence number (0-indexed).
-    pub seqno: u64,
-}
 
 /// A simulated node in the cluster.
 pub struct SimulatedNode {
@@ -60,47 +67,109 @@ pub struct ClusterConfig {
     pub schema: Schema,
 }
 
-/// A simulated multi-region tidalDB cluster.
+/// A simulated multi-region tidalDB cluster using the real M8 distributed
+/// fabric.
 ///
-/// All communication happens via in-memory relay log. No actual network,
+/// Signal replication traverses the real WAL-batch encode → transport →
-/// no actual disk I/O (ephemeral mode). Designed for deterministic,
+/// `apply_payload` → `ReplicationState::advance` pipeline instead of calling
-/// repeatable integration tests.
+/// `db.signal()` directly on followers.
 pub struct SimulatedCluster {
     /// Current leader region ID. Mutable via `promote_leader`.
     leader_region: Mutex<RegionId>,
     /// All nodes indexed by region.
     nodes: HashMap<RegionId, SimulatedNode>,
-    /// Shared relay log: append-only sequence of signal events.
+    /// Per-follower channel senders (region → sender for WAL payloads).
-    relay_log: Arc<Mutex<Vec<RelayEvent>>>,
+    ///
-    /// Per-follower: how many events from `relay_log` have been applied to
+    /// Only regions that have an active `spawn_receiver` thread appear here.
-    /// this follower's `TidalDb`. Monotonically increasing.
+    /// When dropped, the corresponding receiver thread exits cleanly.
-    db_applied: HashMap<RegionId, AtomicU64>,
+    follower_senders: HashMap<RegionId, crossbeam::channel::Sender<WalSegmentPayload>>,
-    /// Set of regions currently isolated (partition injected).
+    /// All batches ever shipped, for partition-recovery re-delivery.
+    batch_log: Mutex<Vec<BatchEntry>>,
+    /// Per-leader signal count used as the WAL seqno for that leader's batches.
+    leader_seqnos: Mutex<HashMap<RegionId, u64>>,
+    /// Total signals ever written to any leader (for [`relay_log_len`]).
+    total_signals: AtomicU64,
+    /// Regions currently isolated by a network partition.
     partitioned_regions: Arc<RwLock<HashSet<RegionId>>>,
-    /// Schema used by all nodes (kept for reference).
+    /// Schema kept for reference.
     #[allow(dead_code)]
     schema: Schema,
+    /// Pre-computed map: signal type name → `u8` ID used in [`EventRecord`].
+    signal_type_ids: HashMap<String, u8>,
 }
 
 impl SimulatedCluster {
     /// Build a cluster from the given configuration.
     ///
-    /// All nodes are created immediately in ephemeral mode.
+    /// All nodes are created immediately in ephemeral mode. Non-leader regions
+    /// have a `spawn_receiver` thread started automatically.
     ///
     /// # Panics
     ///
-    /// Panics if any `TidalDb` fails to open (e.g. invalid schema).
+    /// Panics if any `TidalDb` fails to open or if `start_replication` fails.
     #[must_use]
     pub fn build(config: ClusterConfig) -> Self {
-        let mut nodes = HashMap::new();
+        // Pre-compute signal type IDs via a temporary ledger.
-        let mut db_applied = HashMap::new();
+        let scratch_ledger = SignalLedger::new(config.schema.clone(), Box::new(NoopWalWriter));
+        let signal_type_ids: HashMap<String, u8> = config
+            .schema
+            .signals()
+            .filter_map(|def| {
+                scratch_ledger
+                    .resolve_signal_type(def.name())
+                    .ok()
+                    .map(|id| (def.name().to_string(), id.as_u16() as u8))
+            })
+            .collect();
+
+        // Derive all shard IDs (one per region).
+        let all_shards: Vec<ShardId> = config.regions.iter().map(|r| ShardId(r.0)).collect();
+
+        let mut nodes: HashMap<RegionId, SimulatedNode> = HashMap::new();
+        let mut follower_senders: HashMap<RegionId, crossbeam::channel::Sender<WalSegmentPayload>> =
+            HashMap::new();
+        let mut leader_seqnos: HashMap<RegionId, u64> = HashMap::new();
 
         for &region in &config.regions {
+            let shard_id = ShardId(region.0);
+            let peer_shards: Vec<ShardId> = all_shards
+                .iter()
+                .copied()
+                .filter(|&s| s != shard_id)
+                .collect();
+
             let db = TidalDb::builder()
                 .ephemeral()
                 .with_schema(config.schema.clone())
+                .with_cluster(NodeConfig {
+                    role: NodeRole::Single,
+                    shard_id,
+                    peer_shards,
+                    ..NodeConfig::default()
+                })
                 .open()
                 .expect("ephemeral TidalDb with valid schema must open");
+
+            // Wire a receiver for every non-leader region.
+            if region != config.leader_region {
+                let (tx, rx) = crossbeam::channel::bounded::<WalSegmentPayload>(1024);
+                let transport = Arc::new(ReceiveOnlyTransport {
+                    local_shard: shard_id,
+                    rx,
+                });
+                // spawn_receiver directly: we already have Arc<ReceiveOnlyTransport>
+                // which implements Transport. Use the replication state from the db.
+                let ledger = db
+                    .ledger()
+                    .expect("ephemeral db with schema must have ledger")
+                    .clone();
+                let rep_state = db.replication_state().clone();
+                let _handle = spawn_receiver(transport, ledger, rep_state);
+                // Note: the JoinHandle is intentionally not stored — the receiver thread
+                // will exit cleanly when `tx` (and all senders to `rx`) are dropped.
+                follower_senders.insert(region, tx);
+            }
+
             nodes.insert(
                 region,
                 SimulatedNode {
@@ -108,16 +177,19 @@ impl SimulatedCluster {
                     db,
                 },
             );
-            db_applied.insert(region, AtomicU64::new(0));
+            leader_seqnos.insert(region, 0);
         }
 
         Self {
             leader_region: Mutex::new(config.leader_region),
             nodes,
-            relay_log: Arc::new(Mutex::new(Vec::new())),
+            follower_senders,
-            db_applied,
+            batch_log: Mutex::new(Vec::new()),
+            leader_seqnos: Mutex::new(leader_seqnos),
+            total_signals: AtomicU64::new(0),
             partitioned_regions: Arc::new(RwLock::new(HashSet::new())),
             schema: config.schema,
+            signal_type_ids,
         }
     }
 
@@ -155,39 +227,91 @@ impl SimulatedCluster {
             .unwrap_or_else(|| panic!("no node for region {region}"))
     }
 
-    /// Write a signal to the cluster leader and append to the relay log.
+    /// Write a signal to the cluster leader and ship it to active followers.
     ///
-    /// The signal is immediately applied to the leader's `TidalDb` and
+    /// The signal is immediately applied to the leader's `TidalDb`. A one-event
-    /// recorded in the relay log for later replication to followers.
+    /// WAL batch is encoded and shipped via the channel transport to all
+    /// non-partitioned followers with active receivers. The batch is also
+    /// recorded in the `batch_log` for partition-recovery re-delivery.
     ///
     /// # Panics
     ///
-    /// Panics if the signal write on the leader fails.
+    /// Panics if the signal write on the leader fails or if the signal type is
+    /// not registered in the schema.
     pub fn write_signal(&self, signal_type: &str, entity_id: EntityId, weight: f64) {
         let ts = Timestamp::now();
         let leader_region = self.leader_region();
-        self.nodes
+        let leader_shard = ShardId(leader_region.0);
-            .get(&leader_region)
+
-            .expect("leader node exists")
+        // Write to the leader's signal ledger.
+        self.nodes[&leader_region]
             .db
             .signal(signal_type, entity_id, weight, ts)
             .expect("signal write on leader must succeed");
 
-        let mut log = self
+        // Encode as a one-event WAL batch.
-            .relay_log
+        let type_id = *self
+            .signal_type_ids
+            .get(signal_type)
+            .expect("signal type must be registered in the cluster schema");
+
+        let seqno = {
+            let mut seqnos = self
+                .leader_seqnos
+                .lock()
+                .unwrap_or_else(std::sync::PoisonError::into_inner);
+            let v = {
+                let s = seqnos.entry(leader_region).or_insert(0);
+                *s += 1;
+                *s
+            };
+            drop(seqnos);
+            v
+        };
+
+        let events = [EventRecord {
+            entity_id: entity_id.as_u64(),
+            signal_type: type_id,
+            weight: weight as f32,
+            timestamp_nanos: ts.as_nanos(),
+        }];
+        let bytes =
+            encode_batch(&events, seqno, ts.as_nanos()).expect("WAL batch encoding must not fail");
+
+        // Ship immediately to non-partitioned followers that have active receivers.
+        let partitioned = self
+            .partitioned_regions
+            .read()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .clone();
+
+        for (&region, tx) in &self.follower_senders {
+            if region == leader_region || partitioned.contains(&region) {
+                continue;
+            }
+            let payload = WalSegmentPayload {
+                id: WalSegmentId::new(crate::replication::RegionId::SINGLE, leader_shard, seqno),
+                bytes: bytes.clone(),
+                event_count: 1,
+            };
+            // Ignore send errors: the receiver may have exited (e.g. after a crash).
+            let _ = tx.send(payload);
+        }
+
+        // Record in batch_log for partition-recovery re-delivery.
+        self.batch_log
             .lock()
-            .unwrap_or_else(std::sync::PoisonError::into_inner);
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
-        let seqno = log.len() as u64;
+            .push(BatchEntry {
-        log.push(RelayEvent {
+                source_shard: leader_shard,
-            signal_type: signal_type.to_string(),
+                seqno,
-            entity_id,
+                bytes,
-            weight,
+            });
-            timestamp: ts,
+
-            seqno,
+        self.total_signals.fetch_add(1, Ordering::Relaxed);
-        });
     }
 
-    /// Write a signal directly to a specific region's node (bypassing leader).
+    /// Write a signal directly to a specific region (bypassing the leader).
     ///
     /// Used to simulate partitioned writes during split-brain scenarios.
     ///
@@ -210,71 +334,94 @@ impl SimulatedCluster {
             .expect("signal write must succeed");
     }
 
-    /// Wait for all non-partitioned followers to receive and apply all
+    /// Wait for all non-partitioned followers to apply all pending WAL batches.
-    /// pending relay log events.
     ///
-    /// This is synchronous: it replays events into each follower's `TidalDb`
+    /// * Ships any batches that were missed during a partition (re-delivery from
-    /// directly. The `timeout` guards against programming errors, not actual
+    ///   the `batch_log`).
-    /// latency (in-process replay is instant).
+    /// * Polls `ReplicationState::applied_seqno` for each active follower until
+    ///   it reaches the current leader's latest seqno.
     ///
     /// # Panics
     ///
-    /// Panics if convergence is not reached within `timeout` (should never
+    /// Panics if convergence is not reached within `timeout`.
-    /// happen for in-process relay, but defends against infinite loops).
     pub fn await_convergence(&self, timeout: Duration) {
         let deadline = Instant::now() + timeout;
-        let leader_region = self.leader_region();
 
-        // In-process replay: apply all pending events to each non-partitioned
-        // follower. We loop because a partition might be healed mid-wait.
         loop {
             assert!(
                 Instant::now() <= deadline,
                 "convergence timeout: cluster did not converge within {timeout:?}"
             );
 
+            let leader_region = self.leader_region();
+            let leader_shard = ShardId(leader_region.0);
+
+            let target_seqno = self
+                .leader_seqnos
+                .lock()
+                .unwrap_or_else(std::sync::PoisonError::into_inner)
+                .get(&leader_region)
+                .copied()
+                .unwrap_or(0);
+
             let partitioned = self
                 .partitioned_regions
                 .read()
                 .unwrap_or_else(std::sync::PoisonError::into_inner)
                 .clone();
 
-            let log = self
+            // Re-deliver any batches missed during a partition.
-                .relay_log
+            {
-                .lock()
+                let log = self
-                .unwrap_or_else(std::sync::PoisonError::into_inner)
+                    .batch_log
-                .clone();
+                    .lock()
-            let log_len = log.len() as u64;
+                    .unwrap_or_else(std::sync::PoisonError::into_inner);
 
+                for (&region, tx) in &self.follower_senders {
+                    if region == leader_region || partitioned.contains(&region) {
+                        continue;
+                    }
+                    let node = &self.nodes[&region];
+                    for entry in log.iter() {
+                        let already_applied = node
+                            .db
+                            .replication_state()
+                            .applied_seqno(entry.source_shard)
+                            .unwrap_or(0);
+                        if entry.seqno > already_applied {
+                            let payload = WalSegmentPayload {
+                                id: WalSegmentId::new(
+                                    crate::replication::RegionId::SINGLE,
+                                    entry.source_shard,
+                                    entry.seqno,
+                                ),
+                                bytes: entry.bytes.clone(),
+                                event_count: 1,
+                            };
+                            let _ = tx.try_send(payload);
+                        }
+                    }
+                }
+            }
+
+            // Check whether all active non-partitioned followers have converged.
             let mut all_converged = true;
             for (&region, node) in &self.nodes {
                 if region == leader_region || partitioned.contains(&region) {
                     continue;
                 }
-
+                // Only check regions that have an active receiver thread.
-                let applied = self
+                if !self.follower_senders.contains_key(&region) {
-                    .db_applied
+                    continue;
-                    .get(&region)
-                    .expect("db_applied entry for every region");
-                let current = applied.load(Ordering::Acquire);
-
-                if current < log_len {
-                    // Replay events [current..log_len) into this follower.
-                    for event in &log[current as usize..log_len as usize] {
-                        node.db
-                            .signal(
-                                &event.signal_type,
-                                event.entity_id,
-                                event.weight,
-                                event.timestamp,
-                            )
-                            .expect("follower signal replay must succeed");
-                    }
-                    applied.store(log_len, Ordering::Release);
                 }
-
+                let applied = node
-                if applied.load(Ordering::Acquire) < log_len {
+                    .db
+                    .replication_state()
+                    .applied_seqno(leader_shard)
+                    .unwrap_or(0);
+                if applied < target_seqno {
                     all_converged = false;
+                    break;
                 }
             }
 
@@ -300,24 +447,116 @@ impl SimulatedCluster {
         })
     }
 
-    /// Current length of the relay log (total events written by the leader).
+    /// Broadcast item metadata to all nodes.
-    #[must_use]
+    pub fn write_item_with_metadata(
-    pub fn relay_log_len(&self) -> u64 {
+        &self,
-        self.relay_log
+        entity_id: EntityId,
+        metadata: &HashMap<String, String>,
+    ) -> crate::Result<()> {
+        for node in self.nodes.values() {
+            node.db.write_item_with_metadata(entity_id, metadata)?;
+        }
+        Ok(())
+    }
+
+    /// Broadcast embedding writes to all nodes.
+    pub fn write_item_embedding(
+        &self,
+        entity_id: EntityId,
+        embedding: &[f32],
+    ) -> crate::Result<()> {
+        for node in self.nodes.values() {
+            node.db.write_item_embedding(entity_id, embedding)?;
+        }
+        Ok(())
+    }
+
+    /// Run a RETRIEVE query against a region.
+    pub fn retrieve(
+        &self,
+        region: RegionId,
+        query: &Retrieve,
+    ) -> crate::Result<crate::query::retrieve::Results> {
+        self.node(region).db.retrieve(query)
+    }
+
+    /// Run a SEARCH query against a region.
+    pub fn search(
+        &self,
+        region: RegionId,
+        query: &Search,
+    ) -> crate::Result<crate::query::search::SearchResults> {
+        self.node(region).db.search(query)
+    }
+
+    /// Item count helper for health checks.
+    pub fn item_count(&self, region: RegionId) -> u64 {
+        self.node(region).db.item_count()
+    }
+
+    /// Current leader seqno.
+    pub fn leader_seqno(&self) -> u64 {
+        let leader = self.leader_region();
+        self.leader_seqnos
             .lock()
             .unwrap_or_else(std::sync::PoisonError::into_inner)
-            .len() as u64
+            .get(&leader)
+            .copied()
+            .unwrap_or(0)
     }
 
-    /// How many events have been applied to a specific region's follower DB.
+    /// Mark a region as partitioned.
+    pub fn partition_region(&self, region: RegionId) {
+        let mut partitions = self
+            .partitioned_regions
+            .write()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        partitions.insert(region);
+    }
+
+    /// Heal a partitioned region.
+    pub fn heal_region(&self, region: RegionId) {
+        let mut partitions = self
+            .partitioned_regions
+            .write()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        partitions.remove(&region);
+    }
+
+    /// Whether a region is currently partitioned.
+    pub fn is_partitioned(&self, region: RegionId) -> bool {
+        self.partitioned_regions
+            .read()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .contains(&region)
+    }
+
+    /// Total number of signals ever written to any leader (replaces relay log
+    /// length in the old signal-replay implementation).
+    #[must_use]
+    pub fn relay_log_len(&self) -> u64 {
+        self.total_signals.load(Ordering::Acquire)
+    }
+
+    /// Number of WAL batches applied from the initial leader (`ShardId(0)`)
+    /// on a specific region's replication state.
+    ///
+    /// This is equivalent to the event count for all tests that do not involve
+    /// leader promotion, since each signal produces exactly one WAL batch.
     #[must_use]
     pub fn applied_count(&self, region: RegionId) -> u64 {
-        self.db_applied
+        // ShardId(0) is always the initial leader's shard (RegionId(0)).
-            .get(&region)
+        // Tests that check `applied_count` do not involve leader promotion,
-            .map_or(0, |a| a.load(Ordering::Acquire))
+        // so this is always the correct source shard.
+        self.nodes.get(&region).map_or(0, |n| {
+            n.db.replication_state()
+                .applied_seqno(ShardId(0))
+                .unwrap_or(0)
+        })
     }
 
-    /// Access the shared partitioned regions set (for fault injection).
+    /// Access the shared partitioned-regions set (for fault injection via
+    /// [`crate::testing::faults::NetworkPartition`]).
     #[must_use]
     pub const fn partitioned_regions(&self) -> &Arc<RwLock<HashSet<RegionId>>> {
         &self.partitioned_regions
@@ -325,9 +564,9 @@ impl SimulatedCluster {
 
     /// Promote a follower to leader.
     ///
-    /// The old leader stops receiving writes via `write_signal` (it is
+    /// After promotion `write_signal` routes writes to the new leader and ships
-    /// no longer returned by `leader()`). The new leader can now accept
+    /// batches to all other regions that have active receivers. The new leader
-    /// writes. Data already on each node is preserved.
+    /// must already exist as a node in the cluster.
     ///
     /// # Panics
     ///

tidal/src/testing/cluster_transport.rs

@@ -0,0 +1,40 @@
+//! Internal transport types used by [`super::cluster::SimulatedCluster`].
+
+use crate::replication::shard::ShardId;
+use crate::replication::transport::{Transport, TransportError, WalSegmentPayload};
+
+// ── Internal: receive-only transport ─────────────────────────────────────
+
+/// Minimal transport implementation used by follower receivers.
+///
+/// Owns a crossbeam `Receiver` for incoming WAL segments. The `send_segment`
+/// side is a no-op — all shipping is managed by the cluster struct.
+pub(super) struct ReceiveOnlyTransport {
+    pub(super) local_shard: ShardId,
+    pub(super) rx: crossbeam::channel::Receiver<WalSegmentPayload>,
+}
+
+impl Transport for ReceiveOnlyTransport {
+    fn send_segment(&self, _: ShardId, _: WalSegmentPayload) -> Result<(), TransportError> {
+        Ok(())
+    }
+
+    fn recv_segment(&self) -> Option<WalSegmentPayload> {
+        self.rx.recv().ok()
+    }
+
+    fn local_shard(&self) -> ShardId {
+        self.local_shard
+    }
+}
+
+// ── Internal: batch log entry ─────────────────────────────────────────────
+
+pub(super) struct BatchEntry {
+    /// Which leader shard produced this batch.
+    pub(super) source_shard: ShardId,
+    /// 1-indexed sequence number scoped to `source_shard`.
+    pub(super) seqno: u64,
+    /// Encoded WAL batch bytes (from [`crate::wal::format::batch::encode_batch`]).
+    pub(super) bytes: Vec<u8>,
+}

tidal/src/testing/mod.rs

@@ -5,6 +5,7 @@
 //! and compiles away entirely in production builds.
 
 pub mod cluster;
+pub(super) mod cluster_transport;
 pub mod crash_injector;
 pub mod faults;