jordan/tidaldb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(vector): load a persisted slot graph by its own backend, not an assumed one

Browse Source 
Found by running the standalone server end to end: after a clean shutdown that
logged "persisted HNSW graphs to disk (next boot loads instead of rebuilding)",
the next boot logged

  WARN persisted HNSW graph failed to load; falling back to rebuild
       slot="content_vector" error=USearch load failed: Failed to read vectors

and rebuilt the index. Every boot. Correct results, no data loss, a WARN nobody
reads - and the optimization was dead.

Cause: `checkpoint_graphs` saves whichever index the slot holds, and
`build_slot_index` holds a BRUTE-FORCE index below the dimension-aware
crossover. Both write to the same `<kind>__<slot>.usearch` path, so a small slot
persisted a `BFVI` file that `load_persisted_slot` then handed to the USearch
reader. Observed on a 6-item, 128-D store; at production scale the slot is
USearch-backed, which is why the cluster never surfaced it - but every dev,
staging, and small-tenant instance pays a full index rebuild on every start, and
that rebuild is what the 20-minute startup probe budget exists for.

Fix: sniff the file's magic (brute-force `MAGIC` is now `pub(crate)` so the
reader and writer cannot drift) and dispatch to the matching loader. A
brute-force graph is still rejected when `expected_count` has grown past the
crossover, so the rebuild upgrades the backend to HNSW rather than pinning a
linear scan forever. Log lines now say "vector graph" and name the backend
instead of claiming HNSW for both.

Tests: `brute_force_slot_graph_round_trips_through_registry_persistence` covers
the case that was broken (the existing round-trip test forces USearch, which is
why it passed throughout), and
`grown_corpus_rejects_a_brute_force_graph_so_the_rebuild_upgrades_it` pins the
upgrade path. Verified live: the same data directory that produced the WARN now
logs `loaded persisted vector graph from disk (skipped full rebuild)
backend="brute-force" count=6`, with feed, text search, and vector search all
returning the pre-restart results.
This commit is contained in:
jordan 2026-08-17 17:47:55 -06:00
parent ce68496fdd
commit e5fd19eb73
2 changed files with 167 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
tidal/src/storage/vector/brute/mod.rs
View File

@ -17,7 +17,11 @@ use super::{
// ---------------------------------------------------------------------------
/// Magic bytes identifying a brute-force vector index file.
const MAGIC: &[u8; 4] = b"BFVI";
///
/// `pub(crate)` so the slot-graph loader can sniff a persisted file's backend
/// instead of assuming one; a second copy of these bytes there would be free to
/// drift from this writer.
pub(crate) const MAGIC: &[u8; 4] = b"BFVI";
/// Current binary format version.
const FORMAT_VERSION: u8 = 0x01;

167
tidal/src/storage/vector/registry.rs
View File

@ -186,6 +186,17 @@ pub(crate) fn slot_graph_path(
/// The load uses [`slot_index_config`] so the loaded graph's metric /
/// quantization / connectivity match what [`build_slot_index`] built — a
/// mismatch there would silently wreck recall.
///
/// # Why the backend is sniffed, not assumed
///
/// [`EmbeddingSlotRegistry::checkpoint_graphs`] persists whichever index the
/// slot actually holds, and [`build_slot_index`] holds a brute-force index below
/// the dimension-aware crossover. Both land at the same `.usearch` path, so
/// assuming the USearch reader made every small slot fail to load with
/// "Failed to read vectors" and rebuild on EVERY boot — a permanently broken
/// round trip that only showed up as a WARN. Dispatch on the file's own magic
/// instead, then reject a brute-force graph for a corpus that has since grown
/// past the crossover so the rebuild can upgrade it to HNSW.
fn load_persisted_slot(
vector_dir: &Path,
entity_kind: EntityKind,
@ -198,8 +209,52 @@ fn load_persisted_slot(
return None;
}
let config = slot_index_config(dimensions);
let index = match super::UsearchIndex::load(&path, &config) {
Ok(idx) => idx,
let mut magic = [0u8; 4];
match std::fs::File::open(&path).and_then(|mut f| std::io::Read::read_exact(&mut f, &mut magic))
{
Ok(()) => {}
Err(e) => {
tracing::warn!(
entity_kind = %entity_kind,
slot = slot_name,
path = %path.display(),
error = %e,
"persisted vector graph is unreadable; falling back to rebuild"
);
return None;
}
}
let is_brute_force = &magic == super::brute::MAGIC;
if is_brute_force && expected_count >= usearch_min_vectors(dimensions) {
tracing::info!(
entity_kind = %entity_kind,
slot = slot_name,
expected = expected_count,
"persisted graph is brute-force but the corpus now warrants HNSW; \
rebuilding to upgrade the backend"
);
return None;
}
let index: Box<dyn VectorIndex> = if is_brute_force {
match super::BruteForceIndex::load(&path, &config) {
Ok(idx) => Box::new(idx),
Err(e) => {
tracing::warn!(
entity_kind = %entity_kind,
slot = slot_name,
path = %path.display(),
error = %e,
"persisted brute-force graph failed to load; falling back to rebuild"
);
return None;
}
}
} else {
match super::UsearchIndex::load(&path, &config) {
Ok(idx) => Box::new(idx),
Err(e) => {
tracing::warn!(
entity_kind = %entity_kind,
@ -210,6 +265,7 @@ fn load_persisted_slot(
);
return None;
}
}
};
let loaded = index.len_live();
@ -219,7 +275,7 @@ fn load_persisted_slot(
slot = slot_name,
loaded,
expected = expected_count,
"persisted HNSW graph count does not match the durable corpus \
"persisted vector graph count does not match the durable corpus \
(stale/partial graph); falling back to rebuild"
);
return None;
@ -229,9 +285,10 @@ fn load_persisted_slot(
entity_kind = %entity_kind,
slot = slot_name,
count = loaded,
"loaded persisted HNSW graph from disk (skipped full rebuild)"
backend = if is_brute_force { "brute-force" } else { "usearch" },
"loaded persisted vector graph from disk (skipped full rebuild)"
);
Some(Box::new(index))
Some(index)
}
/// Number of bytes per vector component at a given quantization level.
@ -856,7 +913,7 @@ impl EmbeddingSlotRegistry {
tracing::info!(
slots = saved,
dir = %vector_dir.display(),
"persisted HNSW graphs to disk (next boot loads instead of rebuilding)"
"persisted vector graphs to disk (next boot loads instead of rebuilding)"
);
}
Ok(saved)
@ -1727,4 +1784,102 @@ mod tests {
"loaded USearch graph must return the correct nearest neighbor"
);
}
/// A slot small enough to be brute-force backed must ALSO survive the
/// persist -> load round trip.
///
/// Regression: `checkpoint_graphs` saves whatever index the slot holds, and
/// `build_slot_index` holds brute force below the crossover, so a small slot
/// wrote a `BFVI` file to the `.usearch` path. The loader assumed USearch,
/// failed with "Failed to read vectors", and rebuilt the index on EVERY
/// boot - visible only as a WARN, with correct results afterwards, so the
/// dead optimization went unnoticed. Observed live on a 6-item, 128-D store.
#[test]
fn brute_force_slot_graph_round_trips_through_registry_persistence() {
let dir = tempfile::tempdir().unwrap();
let vector_dir = dir.path().join("vector");
let dim = 128;
let count = 6usize;
assert!(
count < usearch_min_vectors(dim),
"this test must exercise the brute-force branch of build_slot_index"
);
let mut reg = EmbeddingSlotRegistry::new();
let index = build_slot_index(dim, count);
for id in 0..count as u64 {
let mut v = vec![0.0f32; dim];
v[id as usize] = 1.0;
index.insert(id, &v).unwrap();
}
reg.register(
EntityKind::Item,
"content_vector".to_string(),
EmbeddingSlotState {
index,
dimensions: dim,
quantization: QuantizationLevel::F32,
source: EmbeddingSource::External,
params: HnswParams::default(),
},
)
.unwrap();
assert_eq!(reg.checkpoint_graphs(&vector_dir).unwrap(), 1);
let loaded =
load_persisted_slot(&vector_dir, EntityKind::Item, "content_vector", dim, count)
.expect("a brute-force graph with matching count must load, not rebuild");
assert_eq!(loaded.len_live(), count);
let mut q = vec![0.0f32; dim];
q[4] = 1.0;
let results = loaded.search(&q, 1, 0).unwrap();
assert_eq!(
results[0].id, 4,
"loaded brute-force graph must return the correct nearest neighbor"
);
}
/// A persisted brute-force graph is REJECTED once the corpus has grown past
/// the HNSW crossover, so the rebuild upgrades the backend instead of
/// pinning the linear scan forever.
#[test]
fn grown_corpus_rejects_a_brute_force_graph_so_the_rebuild_upgrades_it() {
let dir = tempfile::tempdir().unwrap();
let vector_dir = dir.path().join("vector");
let dim = 128;
let mut reg = EmbeddingSlotRegistry::new();
let index = build_slot_index(dim, 2);
for id in 0..2u64 {
let mut v = vec![0.0f32; dim];
v[id as usize] = 1.0;
index.insert(id, &v).unwrap();
}
reg.register(
EntityKind::Item,
"content_vector".to_string(),
EmbeddingSlotState {
index,
dimensions: dim,
quantization: QuantizationLevel::F32,
source: EmbeddingSource::External,
params: HnswParams::default(),
},
)
.unwrap();
assert_eq!(reg.checkpoint_graphs(&vector_dir).unwrap(), 1);
let expected = usearch_min_vectors(dim);
assert!(
load_persisted_slot(
&vector_dir,
EntityKind::Item,
"content_vector",
dim,
expected
)
.is_none(),
"a brute-force graph must not be loaded for a corpus that now warrants HNSW"
);
}
}