tidaldb/tidal/src/db/text_syncer.rs

//! Background text-index syncer infrastructure.
//!
//! Opening a text index and running its syncer is split into two phases so the
//! crash-recovery rebuild-from-store can run while NO thread holds the Tantivy
//! writer lock:
//!
//! 1. [`open_text_syncer`] opens the `TextIndex` and creates the channels, but
//!    does NOT spawn the syncer thread. The caller now holds the only handle to
//!    the index and the writer lock is free.
//! 2. The caller runs `rebuild_from_store` (see `state_rebuild`) on the open
//!    index — it acquires and releases the writer lock for the rebuild.
//! 3. [`TextSyncerPending::spawn`] starts the syncer thread, which acquires the
//!    writer lock for its entire lifetime (Tantivy single-writer design).
//!
//! Spawning the thread *before* the rebuild deadlocks: the syncer holds the
//! writer lock forever, so `rebuild_from`'s `writer_guard()` blocks forever and
//! the open hangs. [`shutdown_text_syncer`] joins the thread on shutdown.

use std::sync::Arc;

use super::config::Config;
use crate::text::{PendingWrite, TextIndex, TextIndexConfig, TextIndexSyncer};

/// Bound on the pending-write outbox between the entity write path and the
/// background syncer thread.
///
/// The producer (`write_item_with_metadata` / `write_creator`) ships one
/// `PendingWrite` per indexed entity. An *unbounded* channel grows without limit
/// whenever the syncer cannot drain it — e.g. a sustained Tantivy commit outage
/// (disk full, corrupt segment) where `run()` retries forever while writers keep
/// enqueuing. That turns a degraded-text-search incident into an OOM of the whole
/// process. Bounding the channel converts that failure mode into *backpressure*:
/// once `WRITE_OUTBOX_CAPACITY` writes are buffered, the producer's blocking
/// `send` stalls until the syncer makes room, so memory is capped at the buffer
/// plus Tantivy's own un-committed batch. The cap is generous enough to absorb a
/// normal commit-interval burst without throttling steady-state writes, but small
/// enough that a stalled syncer cannot consume unbounded heap. The text index is
/// derived state rebuilt from the durable store at open, so back-pressuring (or,
/// at the call site, dropping) a write here never loses durable data.
const WRITE_OUTBOX_CAPACITY: usize = 16_384;

/// Result from spawning a background text syncer thread.
pub struct TextSyncerBundle {
    pub index: Option<Arc<TextIndex>>,
    pub write_tx: Option<crossbeam::channel::Sender<PendingWrite>>,
    pub flush_tx: Option<crossbeam::channel::Sender<crossbeam::channel::Sender<()>>>,
    pub thread: std::sync::Mutex<Option<std::thread::JoinHandle<crate::Result<()>>>>,
}

/// Phase 1 of text-syncer setup: an opened [`TextIndex`] plus its channels, with
/// NO syncer thread yet running.
///
/// While this value is held, the Tantivy writer lock is free, so the caller can
/// safely run the crash-recovery rebuild-from-store on [`index`](Self::index)
/// before [`spawn`](Self::spawn)ing the thread that takes the lock for life.
///
/// An empty `text_fields` set (no text index) yields a [`TextSyncerPending`] with
/// `index: None`; calling [`spawn`](Self::spawn) on it produces an empty
/// [`TextSyncerBundle`] (no thread, no channels).
pub struct TextSyncerPending {
    index: Option<Arc<TextIndex>>,
    /// `(write_rx, flush_tx, flush_rx, commit_n, commit_secs, thread_name)`, all
    /// captured at open time and consumed by [`spawn`](Self::spawn). `None` when
    /// there is no index.
    spawn_inputs: Option<SpawnInputs>,
    write_tx: Option<crossbeam::channel::Sender<PendingWrite>>,
    flush_tx: Option<crossbeam::channel::Sender<crossbeam::channel::Sender<()>>>,
}

/// Inputs captured at open time and consumed when the syncer thread is spawned.
struct SpawnInputs {
    write_rx: crossbeam::channel::Receiver<PendingWrite>,
    flush_rx: crossbeam::channel::Receiver<crossbeam::channel::Sender<()>>,
    commit_n: usize,
    commit_secs: u64,
    thread_name: String,
}

impl TextSyncerPending {
    /// Borrow the opened index so the caller can run the rebuild-from-store while
    /// no syncer thread holds the writer lock. `None` when no text fields were
    /// declared (no index exists).
    #[must_use]
    pub const fn index(&self) -> Option<&Arc<TextIndex>> {
        self.index.as_ref()
    }

    /// Phase 2: spawn the syncer thread for the opened index and return the
    /// running [`TextSyncerBundle`].
    ///
    /// MUST be called only after any rebuild-from-store on [`index`](Self::index)
    /// has completed: the spawned syncer acquires the Tantivy writer lock for its
    /// entire lifetime, so a rebuild attempted afterward would deadlock.
    #[must_use]
    pub fn spawn(self) -> TextSyncerBundle {
        let (Some(index), Some(inputs)) = (self.index, self.spawn_inputs) else {
            // No text index — nothing to spawn.
            return TextSyncerBundle {
                index: None,
                write_tx: None,
                flush_tx: None,
                thread: std::sync::Mutex::new(None),
            };
        };

        let idx_clone = Arc::clone(&index);
        let SpawnInputs {
            write_rx,
            flush_rx,
            commit_n,
            commit_secs,
            thread_name,
        } = inputs;

        let handle = match std::thread::Builder::new()
            .name(thread_name)
            .spawn(move || {
                TextIndexSyncer::new(idx_clone, write_rx, commit_n, commit_secs)
                    .with_flush_rx(flush_rx)
                    .run()
            }) {
            Ok(h) => Some(h),
            Err(e) => {
                // Thread spawn failed (e.g. EAGAIN / resource limits). The bundle
                // still carries the open index and live channels, but with no
                // syncer thread draining them, queued writes never reach Tantivy.
                // `.ok()` discarded this silently; log it so the disabled syncer
                // is observable rather than a mystery stale index.
                tracing::error!(
                    error = %e,
                    "text syncer thread spawn failed; index writes will not be committed"
                );
                None
            }
        };

        TextSyncerBundle {
            index: Some(index),
            write_tx: self.write_tx,
            flush_tx: self.flush_tx,
            thread: std::sync::Mutex::new(handle),
        }
    }
}

/// Phase 1 of background text-syncer setup: open the index and create channels,
/// WITHOUT spawning the syncer thread.
///
/// Creates a `TextIndex` (on-disk if `config.data_dir` is set, ephemeral
/// otherwise), a crossbeam channel for pending writes, and an optional flush
/// channel for synchronous commit requests. The syncer thread is NOT started —
/// call [`TextSyncerPending::spawn`] after running the crash-recovery rebuild on
/// [`TextSyncerPending::index`].
///
/// On empty `text_fields` (no text index) or an index-open failure, returns a
/// pending handle with `index: None` whose `spawn()` is an empty no-op bundle.
pub fn open_text_syncer(
    text_fields: &[crate::schema::TextFieldDef],
    config: &Config,
    index_name: &str,
    thread_name: &str,
) -> TextSyncerPending {
    let empty = || TextSyncerPending {
        index: None,
        spawn_inputs: None,
        write_tx: None,
        flush_tx: None,
    };

    if text_fields.is_empty() {
        return empty();
    }

    let text_config = config
        .data_dir
        .as_ref()
        .map_or_else(TextIndexConfig::default, |dir| TextIndexConfig {
            index_dir: dir.join(index_name),
            ..TextIndexConfig::default()
        });

    let index = match &config.data_dir {
        Some(_) => TextIndex::open(text_config.clone(), text_fields),
        None => TextIndex::ephemeral(text_fields),
    };

    let index = match index {
        Ok(idx) => Arc::new(idx),
        Err(e) => {
            // A corrupt Tantivy directory, a permission error, or a full disk all
            // surface here. Swallowing the error returned an empty bundle
            // (`index: None`) that is INDISTINGUISHABLE from "no text fields
            // declared", so a hard text-index failure silently disabled BM25
            // search with no operator signal. Log it loudly before degrading.
            tracing::error!(
                index = index_name,
                error = %e,
                "text index open failed; text search disabled for this index"
            );
            return empty();
        }
    };

    // Bounded (not unbounded): a stalled syncer applies backpressure to the
    // write path instead of growing the outbox without limit. See
    // [`WRITE_OUTBOX_CAPACITY`] for the rationale and the bound.
    let (tx, rx) = crossbeam::channel::bounded(WRITE_OUTBOX_CAPACITY);
    let (flush_tx, flush_rx) = crossbeam::channel::bounded::<crossbeam::channel::Sender<()>>(4);

    TextSyncerPending {
        index: Some(index),
        spawn_inputs: Some(SpawnInputs {
            write_rx: rx,
            flush_rx,
            commit_n: text_config.commit_every_n_docs,
            commit_secs: text_config.commit_every_secs,
            thread_name: thread_name.to_owned(),
        }),
        write_tx: Some(tx),
        flush_tx: Some(flush_tx),
    }
}

/// Drop the write-channel sender then join the syncer thread.
///
/// Used by `shutdown_inner` for both item and creator text syncers.
pub fn shutdown_text_syncer(
    tx_mutex: &std::sync::Mutex<Option<crossbeam::channel::Sender<PendingWrite>>>,
    thread_mutex: &std::sync::Mutex<Option<std::thread::JoinHandle<crate::Result<()>>>>,
    label: &str,
) {
    // Drop the sender to disconnect the channel -- the syncer flushes and exits.
    let sender = match tx_mutex.lock() {
        Ok(mut g) => g.take(),
        Err(poisoned) => poisoned.into_inner().take(),
    };
    drop(sender);

    // Join the syncer thread.
    let thread = match thread_mutex.lock() {
        Ok(mut g) => g.take(),
        Err(poisoned) => poisoned.into_inner().take(),
    };
    if let Some(handle) = thread {
        match handle.join() {
            Ok(Ok(())) => {}
            Ok(Err(e)) => tracing::error!(error = %e, "{label} syncer thread returned an error"),
            Err(_) => tracing::error!("{label} syncer thread panicked"),
        }
    }
}

#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod tests {
    use std::collections::HashMap;

    use super::*;
    use crate::schema::{EntityId, TextFieldDef, TextFieldType};

    fn one_text_field() -> Vec<TextFieldDef> {
        vec![TextFieldDef {
            key: "title".to_owned(),
            field_type: TextFieldType::Text,
        }]
    }

    fn pending_write(id: u64) -> PendingWrite {
        let mut metadata = HashMap::new();
        metadata.insert("title".to_owned(), "x".to_owned());
        PendingWrite {
            entity_id: EntityId::new(id),
            metadata,
            deleted: false,
        }
    }

    /// W41 regression: the write outbox must be *bounded* so a stalled syncer
    /// applies backpressure instead of growing memory without limit. We open the
    /// pending syncer but deliberately do NOT `spawn()` it, so nothing drains the
    /// channel; a bounded channel then refuses `try_send` once it is full, while
    /// the previous unbounded channel would accept every send forever.
    #[test]
    fn write_outbox_is_bounded() {
        // Ephemeral config (no data_dir) so the index is in-memory and the test
        // is fast and self-contained.
        let config = Config::default();
        let pending = open_text_syncer(&one_text_field(), &config, "items", "test-syncer");

        let tx = pending
            .write_tx
            .as_ref()
            .expect("text field declared, so a write sender exists")
            .clone();

        // Fill exactly to capacity: every send up to the cap must succeed.
        for id in 0..WRITE_OUTBOX_CAPACITY as u64 {
            tx.try_send(pending_write(id))
                .expect("bounded channel must accept up to its capacity");
        }

        // One past capacity must be rejected as full -- the proof that the
        // channel is bounded and cannot grow without limit while no syncer drains
        // it. An unbounded channel would accept this send.
        match tx.try_send(pending_write(WRITE_OUTBOX_CAPACITY as u64)) {
            Err(crossbeam::channel::TrySendError::Full(_)) => {}
            Ok(()) => panic!("write outbox accepted a send past its capacity -- not bounded"),
            Err(other) => panic!("unexpected send error: {other:?}"),
        }
    }

    /// CRITICAL regression (pass2): the item/creator write path must enqueue
    /// via the producer pattern "clone the Sender out of the guard, DROP the
    /// guard, then `try_send`" — never a blocking `send` under a held mutex.
    /// This reproduces that exact sequence against a FULL bounded outbox (no
    /// syncer spawned, so nothing drains it) and asserts the enqueue returns
    /// immediately with `Full` instead of blocking. A blocking `send` here would
    /// hang forever — and under the old code, with the mutex still held, it would
    /// stall every concurrent write. The test running to completion is the proof
    /// that the producer never blocks.
    #[test]
    fn producer_does_not_block_when_outbox_full() {
        let config = Config::default();
        let pending = open_text_syncer(&one_text_field(), &config, "items", "test-syncer");

        // Model the field exactly as `write_item_with_metadata` holds it.
        let tx_mutex = std::sync::Mutex::new(pending.write_tx.clone());

        // Saturate the outbox to capacity — nothing drains it (no spawn()).
        let fill_tx = pending
            .write_tx
            .as_ref()
            .expect("text field declared")
            .clone();
        for id in 0..WRITE_OUTBOX_CAPACITY as u64 {
            fill_tx
                .try_send(pending_write(id))
                .expect("bounded channel accepts up to capacity");
        }

        // The producer pattern: clone out of the guard, DROP the guard, try_send.
        // If this used a blocking `send`, the test would hang here forever.
        let tx = tx_mutex.lock().ok().and_then(|g| g.as_ref().cloned());
        let tx = tx.expect("sender present");
        match tx.try_send(pending_write(WRITE_OUTBOX_CAPACITY as u64)) {
            Err(crossbeam::channel::TrySendError::Full(_)) => {} // non-blocking drop
            Ok(()) => panic!("outbox accepted a send past capacity — not bounded"),
            Err(other) => panic!("unexpected send error: {other:?}"),
        }

        // The mutex is immediately re-lockable: the producer never held it across
        // the send, so a concurrent writer is not stalled.
        assert!(
            tx_mutex.try_lock().is_ok(),
            "mutex must be free — the producer must not hold it across the send"
        );
    }

    /// An empty text-field set yields a pending handle with no index and no
    /// channels, so there is nothing to bound.
    #[test]
    fn no_text_fields_has_no_write_channel() {
        let config = Config::default();
        let pending = open_text_syncer(&[], &config, "items", "test-syncer");
        assert!(pending.write_tx.is_none());
        assert!(pending.index().is_none());
    }
}