From 7c1c80dd90841224623e670bc31f533359891d96 Mon Sep 17 00:00:00 2001 From: jordan Date: Sun, 30 Aug 2026 11:33:03 -0600 Subject: [PATCH] fix(vector): grow the HNSW graph on insert; it could never accept a new vector once full Measured on the live RF3 cluster 2026-08-30 while verifying it end to end. Every NEW embedding returned HTTP 500: POST /sharded/embeddings -> 500 {"error":"... [op=write_item_embedding] backend error: USearch insert failed: Reserve capacity ahead of insertions!"} while POST /sharded/items returned 201, POST /sharded/signals 204, text search and feed 200, and re-embedding an ALREADY-INDEXED entity returned 204. So the store had silently become read/update-only for vectors: a consumer could write items and signals all day and only its embeddings would fail, with nothing alerting on it. Cause: USearch's add() cannot grow the graph. build_slot_index() reserves the rebuild's expected count once and its comment claimed 'the write path grows it further as needed' - the write path never reserved anything. insert() went straight to add() for a new key, so the moment size() reached the reservation every new key failed permanently. The upsert path kept working because remove() tombstones and size() excludes tombstones, so the re-add lands in the slot just freed - which is exactly why this looked healthy from the outside. insert() now reserves before adding a new key at capacity, growing by max(size/8, 1024) so reserve's reallocation is amortised rather than per-insert. The regression test drives off the REPORTED capacity, not the requested one: USearch rounds a reservation up (reserve(4) reported 64), so a hardcoded insert count sits inside the reservation and exercises nothing. Verified against the pre-fix source it fails with the production error at insert 64 of capacity 64, and it asserts the grown graph still answers searches and still upserts. storage::vector 101 passed, db::items 7 passed. --- tidal/src/storage/vector/usearch_index.rs | 86 +++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/tidal/src/storage/vector/usearch_index.rs b/tidal/src/storage/vector/usearch_index.rs index 9ababe7..227f3d3 100644 --- a/tidal/src/storage/vector/usearch_index.rs +++ b/tidal/src/storage/vector/usearch_index.rs @@ -116,6 +116,15 @@ pub struct UsearchIndex { } impl UsearchIndex { + /// Minimum number of extra vector slots to reserve when the HNSW graph hits + /// its capacity during an insert. + /// + /// `reserve` reallocates, so growth is amortized: `insert` grows by + /// `max(size / 8, MIN_CAPACITY_GROWTH)`. This floor is what covers a small + /// or freshly built index, where an eighth of the current size rounds down + /// to nothing and the index would otherwise reallocate on every insert. + const MIN_CAPACITY_GROWTH: usize = 1024; + /// Create a new, empty `USearch` HNSW index with the given configuration. /// /// # Errors @@ -276,9 +285,32 @@ impl VectorIndex for UsearchIndex { if !is_new { // `remove` lazily tombstones the old slot; the subsequent `add` // re-inserts under the same key (the freed key is no longer a dup). + // This path needs no capacity: `size()` excludes the tombstone, so + // the re-`add` lands in the slot just freed. That is why a re-embed + // of an existing entity kept working even with the graph full. self.inner .remove(id) .map_err(|e| VectorError::Backend(format!("USearch upsert remove failed: {e}")))?; + } else if self.inner.size() >= self.inner.capacity() { + // USearch's `add` CANNOT grow the graph. At capacity it fails with + // "Reserve capacity ahead of insertions!", and nothing else on the + // write path reserves — build_slot_index reserves the rebuild's + // expected count once and its comment claimed "the write path grows + // it further as needed", which was not true. The consequence in + // production (measured 2026-08-30 on the RF3 cluster): every NEW + // embedding returned HTTP 500 while items, signals, searches and + // re-embeds of existing entities all succeeded, so the store had + // silently become read/update-only for vectors with nothing alerting. + // + // Grow in amortized chunks rather than per-insert: `reserve` + // reallocates the graph, so growing by a fraction of the current + // size keeps insert O(1) amortized. The floor covers small indexes + // where size/8 rounds to nothing. + let size = self.inner.size(); + let growth = core::cmp::max(size / 8, Self::MIN_CAPACITY_GROWTH); + self.inner.reserve(size + growth).map_err(|e| { + VectorError::Backend(format!("USearch grow to {} failed: {e}", size + growth)) + })?; } self.inner @@ -481,6 +513,60 @@ mod tests { } } + /// A full HNSW graph must keep accepting NEW keys. + /// + /// Regression for the production defect measured 2026-08-30 on the RF3 + /// cluster: `add` cannot grow the graph, and nothing on the write path + /// reserved, so once `size()` reached the reserved capacity every new + /// embedding failed with "Reserve capacity ahead of insertions!" and + /// returned HTTP 500. Items, signals, searches and re-embeds of EXISTING + /// entities all still worked, so the store had silently become + /// read/update-only for vectors. This asserts inserts continue past the + /// reservation, and that the vectors are actually retrievable afterwards. + #[test] + fn usearch_insert_grows_past_reserved_capacity() { + let index = UsearchIndex::new(default_config(4)).unwrap(); + index.reserve(4).unwrap(); + // Drive off the REPORTED capacity, not the requested one: USearch rounds a + // reservation up (reserve(4) reported 64 here), so a hardcoded insert count + // can sit entirely inside the reservation and never exercise growth at all. + let reserved = index.inner.capacity(); + let target = reserved + 100; + + for i in 0..target as u64 { + let f = i as f32; + index + .insert(i, &[f, f + 1.0, f + 2.0, f + 3.0]) + .unwrap_or_else(|e| { + panic!("insert {i} failed past reserved capacity {reserved}: {e}") + }); + } + assert_eq!(index.len_live(), target, "every new key must be indexed"); + assert!( + index.inner.capacity() > reserved, + "capacity should have grown beyond the original {reserved}" + ); + + // The grown graph must still answer, not merely accept writes. + let hits = index.search(&[10.0, 11.0, 12.0, 13.0], 3, 0).unwrap(); + assert!( + !hits.is_empty(), + "search over the grown index returned nothing" + ); + assert_eq!( + hits[0].id, 10, + "nearest neighbour should be the exact match" + ); + + // Re-embedding an existing key must still work after growth. + index.insert(10, &[0.0, 0.0, 0.0, 0.0]).unwrap(); + assert_eq!( + index.len_live(), + target, + "upsert must not change the live count" + ); + } + #[test] fn usearch_new_is_empty() { let index = UsearchIndex::new(default_config(128)).unwrap();