From e5fd19eb73a43fda08f52382e8621d47e767fbc2 Mon Sep 17 00:00:00 2001 From: jordan Date: Mon, 17 Aug 2026 17:47:55 -0600 Subject: [PATCH] fix(vector): load a persisted slot graph by its own backend, not an assumed one Found by running the standalone server end to end: after a clean shutdown that logged "persisted HNSW graphs to disk (next boot loads instead of rebuilding)", the next boot logged WARN persisted HNSW graph failed to load; falling back to rebuild slot="content_vector" error=USearch load failed: Failed to read vectors and rebuilt the index. Every boot. Correct results, no data loss, a WARN nobody reads - and the optimization was dead. Cause: `checkpoint_graphs` saves whichever index the slot holds, and `build_slot_index` holds a BRUTE-FORCE index below the dimension-aware crossover. Both write to the same `__.usearch` path, so a small slot persisted a `BFVI` file that `load_persisted_slot` then handed to the USearch reader. Observed on a 6-item, 128-D store; at production scale the slot is USearch-backed, which is why the cluster never surfaced it - but every dev, staging, and small-tenant instance pays a full index rebuild on every start, and that rebuild is what the 20-minute startup probe budget exists for. Fix: sniff the file's magic (brute-force `MAGIC` is now `pub(crate)` so the reader and writer cannot drift) and dispatch to the matching loader. A brute-force graph is still rejected when `expected_count` has grown past the crossover, so the rebuild upgrades the backend to HNSW rather than pinning a linear scan forever. Log lines now say "vector graph" and name the backend instead of claiming HNSW for both. Tests: `brute_force_slot_graph_round_trips_through_registry_persistence` covers the case that was broken (the existing round-trip test forces USearch, which is why it passed throughout), and `grown_corpus_rejects_a_brute_force_graph_so_the_rebuild_upgrades_it` pins the upgrade path. Verified live: the same data directory that produced the WARN now logs `loaded persisted vector graph from disk (skipped full rebuild) backend="brute-force" count=6`, with feed, text search, and vector search all returning the pre-restart results. --- tidal/src/storage/vector/brute/mod.rs | 6 +- tidal/src/storage/vector/registry.rs | 169 ++++++++++++++++++++++++-- 2 files changed, 167 insertions(+), 8 deletions(-) diff --git a/tidal/src/storage/vector/brute/mod.rs b/tidal/src/storage/vector/brute/mod.rs index 1cecfb7..45d056f 100644 --- a/tidal/src/storage/vector/brute/mod.rs +++ b/tidal/src/storage/vector/brute/mod.rs @@ -17,7 +17,11 @@ use super::{ // --------------------------------------------------------------------------- /// Magic bytes identifying a brute-force vector index file. -const MAGIC: &[u8; 4] = b"BFVI"; +/// +/// `pub(crate)` so the slot-graph loader can sniff a persisted file's backend +/// instead of assuming one; a second copy of these bytes there would be free to +/// drift from this writer. +pub(crate) const MAGIC: &[u8; 4] = b"BFVI"; /// Current binary format version. const FORMAT_VERSION: u8 = 0x01; diff --git a/tidal/src/storage/vector/registry.rs b/tidal/src/storage/vector/registry.rs index da8dc72..72e4e02 100644 --- a/tidal/src/storage/vector/registry.rs +++ b/tidal/src/storage/vector/registry.rs @@ -186,6 +186,17 @@ pub(crate) fn slot_graph_path( /// The load uses [`slot_index_config`] so the loaded graph's metric / /// quantization / connectivity match what [`build_slot_index`] built — a /// mismatch there would silently wreck recall. +/// +/// # Why the backend is sniffed, not assumed +/// +/// [`EmbeddingSlotRegistry::checkpoint_graphs`] persists whichever index the +/// slot actually holds, and [`build_slot_index`] holds a brute-force index below +/// the dimension-aware crossover. Both land at the same `.usearch` path, so +/// assuming the USearch reader made every small slot fail to load with +/// "Failed to read vectors" and rebuild on EVERY boot — a permanently broken +/// round trip that only showed up as a WARN. Dispatch on the file's own magic +/// instead, then reject a brute-force graph for a corpus that has since grown +/// past the crossover so the rebuild can upgrade it to HNSW. fn load_persisted_slot( vector_dir: &Path, entity_kind: EntityKind, @@ -198,18 +209,63 @@ fn load_persisted_slot( return None; } let config = slot_index_config(dimensions); - let index = match super::UsearchIndex::load(&path, &config) { - Ok(idx) => idx, + + let mut magic = [0u8; 4]; + match std::fs::File::open(&path).and_then(|mut f| std::io::Read::read_exact(&mut f, &mut magic)) + { + Ok(()) => {} Err(e) => { tracing::warn!( entity_kind = %entity_kind, slot = slot_name, path = %path.display(), error = %e, - "persisted HNSW graph failed to load; falling back to rebuild" + "persisted vector graph is unreadable; falling back to rebuild" ); return None; } + } + + let is_brute_force = &magic == super::brute::MAGIC; + if is_brute_force && expected_count >= usearch_min_vectors(dimensions) { + tracing::info!( + entity_kind = %entity_kind, + slot = slot_name, + expected = expected_count, + "persisted graph is brute-force but the corpus now warrants HNSW; \ + rebuilding to upgrade the backend" + ); + return None; + } + + let index: Box = if is_brute_force { + match super::BruteForceIndex::load(&path, &config) { + Ok(idx) => Box::new(idx), + Err(e) => { + tracing::warn!( + entity_kind = %entity_kind, + slot = slot_name, + path = %path.display(), + error = %e, + "persisted brute-force graph failed to load; falling back to rebuild" + ); + return None; + } + } + } else { + match super::UsearchIndex::load(&path, &config) { + Ok(idx) => Box::new(idx), + Err(e) => { + tracing::warn!( + entity_kind = %entity_kind, + slot = slot_name, + path = %path.display(), + error = %e, + "persisted HNSW graph failed to load; falling back to rebuild" + ); + return None; + } + } }; let loaded = index.len_live(); @@ -219,7 +275,7 @@ fn load_persisted_slot( slot = slot_name, loaded, expected = expected_count, - "persisted HNSW graph count does not match the durable corpus \ + "persisted vector graph count does not match the durable corpus \ (stale/partial graph); falling back to rebuild" ); return None; @@ -229,9 +285,10 @@ fn load_persisted_slot( entity_kind = %entity_kind, slot = slot_name, count = loaded, - "loaded persisted HNSW graph from disk (skipped full rebuild)" + backend = if is_brute_force { "brute-force" } else { "usearch" }, + "loaded persisted vector graph from disk (skipped full rebuild)" ); - Some(Box::new(index)) + Some(index) } /// Number of bytes per vector component at a given quantization level. @@ -856,7 +913,7 @@ impl EmbeddingSlotRegistry { tracing::info!( slots = saved, dir = %vector_dir.display(), - "persisted HNSW graphs to disk (next boot loads instead of rebuilding)" + "persisted vector graphs to disk (next boot loads instead of rebuilding)" ); } Ok(saved) @@ -1727,4 +1784,102 @@ mod tests { "loaded USearch graph must return the correct nearest neighbor" ); } + + /// A slot small enough to be brute-force backed must ALSO survive the + /// persist -> load round trip. + /// + /// Regression: `checkpoint_graphs` saves whatever index the slot holds, and + /// `build_slot_index` holds brute force below the crossover, so a small slot + /// wrote a `BFVI` file to the `.usearch` path. The loader assumed USearch, + /// failed with "Failed to read vectors", and rebuilt the index on EVERY + /// boot - visible only as a WARN, with correct results afterwards, so the + /// dead optimization went unnoticed. Observed live on a 6-item, 128-D store. + #[test] + fn brute_force_slot_graph_round_trips_through_registry_persistence() { + let dir = tempfile::tempdir().unwrap(); + let vector_dir = dir.path().join("vector"); + let dim = 128; + let count = 6usize; + assert!( + count < usearch_min_vectors(dim), + "this test must exercise the brute-force branch of build_slot_index" + ); + + let mut reg = EmbeddingSlotRegistry::new(); + let index = build_slot_index(dim, count); + for id in 0..count as u64 { + let mut v = vec![0.0f32; dim]; + v[id as usize] = 1.0; + index.insert(id, &v).unwrap(); + } + reg.register( + EntityKind::Item, + "content_vector".to_string(), + EmbeddingSlotState { + index, + dimensions: dim, + quantization: QuantizationLevel::F32, + source: EmbeddingSource::External, + params: HnswParams::default(), + }, + ) + .unwrap(); + + assert_eq!(reg.checkpoint_graphs(&vector_dir).unwrap(), 1); + let loaded = + load_persisted_slot(&vector_dir, EntityKind::Item, "content_vector", dim, count) + .expect("a brute-force graph with matching count must load, not rebuild"); + assert_eq!(loaded.len_live(), count); + let mut q = vec![0.0f32; dim]; + q[4] = 1.0; + let results = loaded.search(&q, 1, 0).unwrap(); + assert_eq!( + results[0].id, 4, + "loaded brute-force graph must return the correct nearest neighbor" + ); + } + + /// A persisted brute-force graph is REJECTED once the corpus has grown past + /// the HNSW crossover, so the rebuild upgrades the backend instead of + /// pinning the linear scan forever. + #[test] + fn grown_corpus_rejects_a_brute_force_graph_so_the_rebuild_upgrades_it() { + let dir = tempfile::tempdir().unwrap(); + let vector_dir = dir.path().join("vector"); + let dim = 128; + + let mut reg = EmbeddingSlotRegistry::new(); + let index = build_slot_index(dim, 2); + for id in 0..2u64 { + let mut v = vec![0.0f32; dim]; + v[id as usize] = 1.0; + index.insert(id, &v).unwrap(); + } + reg.register( + EntityKind::Item, + "content_vector".to_string(), + EmbeddingSlotState { + index, + dimensions: dim, + quantization: QuantizationLevel::F32, + source: EmbeddingSource::External, + params: HnswParams::default(), + }, + ) + .unwrap(); + assert_eq!(reg.checkpoint_graphs(&vector_dir).unwrap(), 1); + + let expected = usearch_min_vectors(dim); + assert!( + load_persisted_slot( + &vector_dir, + EntityKind::Item, + "content_vector", + dim, + expected + ) + .is_none(), + "a brute-force graph must not be loaded for a corpus that now warrants HNSW" + ); + } }