a0399550d6
Boot now LOADS the per-slot HNSW graph instead of rebuilding it. Clean
shutdown writes {data_dir}/vector/<kind>__<slot>.usearch; the next open loads
it when it matches the durable corpus (seconds), falling back to a full rebuild
only when the graph is missing/stale/corrupt. Eliminates the multi-minute boot
rebuild (~50-70 min at 1M/1536-D) that let the WAL compact past a restarting
node and triggered the reseed cascade.
Graceful SIGTERM now actually runs the close: bounded_drain caps the post-signal
HTTP drain (TIDAL_SHUTDOWN_DRAIN_MS, default 15s) then runs the deterministic
close regardless — sibling keep-alive connections no longer block the drain past
the k8s 60s grace into a SIGKILL (which cannot run Drop). ClusterNode and
ShardReplica::shutdown are now &self (db handle is an ArcSwapOption) so the close
fires even when a stuck connection task holds an Arc.
Fix USearch insert to be a true upsert (remove+add): it was unconditional add,
which a multi:false index rejects on a reseeding follower's post-snapshot WAL
replay -> applied_events stalls -> catch-up deadlock -> unrecoverable cluster.
Also: circuit-breaker peer last-contact tracking; real k3s 1536-dim deploy +
recall findings (recall@10 0.9869, read p99 8.71ms @ 200rps @ 100k) in
docs/profiling/m12-cluster-deploy-findings.md; new tidal-stress k8s jobs and
m12p6 graph-persistence + SIGTERM tier-3 regression tests.
275 lines
6.1 KiB
TOML
275 lines
6.1 KiB
TOML
[package]
|
|
name = "tidaldb"
|
|
version = "0.1.0"
|
|
edition = "2024"
|
|
rust-version = "1.91"
|
|
description = "Embeddable database for personalized content ranking"
|
|
license = "MIT"
|
|
|
|
[features]
|
|
default = ["metrics"]
|
|
test-utils = ["dep:tempfile"]
|
|
metrics = [] # hand-rolled HTTP, no new crate deps
|
|
# m11p9 chaos testing: WAL slow-fsync + disk-full injection (src/fault.rs).
|
|
# NOT in `default` and never passed by the production image build, so the
|
|
# shipped binary compiles it out entirely. Inert until an env var arms it even
|
|
# when compiled in. See src/fault.rs.
|
|
fault-injection = []
|
|
|
|
[dependencies]
|
|
base64 = "0.22"
|
|
blake3 = "1"
|
|
crossbeam = "0.8"
|
|
dashmap = "6"
|
|
fjall = "3"
|
|
lru = "0.12"
|
|
fs4 = "0.8"
|
|
rand = "0.9"
|
|
roaring = "0.10"
|
|
# Inline storage for the per-candidate ranking signal-snapshot (ranking/executor):
|
|
# the common formula-sort breadth fits inline, so the hot scoring loop builds the
|
|
# explain-ability breakdown with zero heap allocation. Already in the lock
|
|
# transitively (tantivy/fjall), so no new dependency surface — promoted to a
|
|
# direct dep here, matching the arc-swap/blake3/base64 pattern.
|
|
smallvec = "1"
|
|
serde = { version = "1", features = ["derive"] }
|
|
serde_json = "1"
|
|
thiserror = "2"
|
|
tantivy = "0.22"
|
|
tempfile = { version = "3", optional = true }
|
|
tracing = "0.1"
|
|
usearch = "2.24.0"
|
|
|
|
# macOS-only: a plain fsync(2) on Apple platforms does NOT flush the storage
|
|
# device's volatile write cache (Apple documents this explicitly); true
|
|
# crash durability requires fcntl(fd, F_FULLFSYNC). We use rustix's *safe*
|
|
# `fcntl_fullfsync` wrapper rather than a raw `libc::fcntl` call so the WAL's
|
|
# macOS durability path stays compatible with this crate's `unsafe_code =
|
|
# "forbid"` posture. rustix is already in the dependency tree (via fjall), so
|
|
# this adds no new compile-time cost on Linux (where it is not pulled in).
|
|
[target.'cfg(target_os = "macos")'.dependencies]
|
|
rustix = { version = "1", features = ["fs"] }
|
|
|
|
[dev-dependencies]
|
|
actix-web = "4"
|
|
axum = "0.8"
|
|
criterion = { version = "0.5", features = ["html_reports"] }
|
|
proptest = "1"
|
|
tempfile = "3"
|
|
tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal"] }
|
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
|
|
|
# ── tidal-crate lint posture (single source of truth) ──────────────────────
|
|
# The four tidal crates (tidaldb / tidal-net / tidal-server / tidalctl) carry an
|
|
# IDENTICAL, stricter-than-workspace [lints] posture and deliberately DO NOT
|
|
# inherit `[workspace.lints]` (`lints.workspace = false` is the Cargo default).
|
|
# Rationale: this is the embedded recommendation DB plus its transport, server,
|
|
# and CLI — a correctness-critical, mostly self-contained subsystem held to a
|
|
# higher bar (`unsafe_code = forbid`, `clippy::all = deny`, `unwrap_used = deny`).
|
|
# `unwrap_used = "deny"` is kept per-crate rather than promoted into
|
|
# `[workspace.lints]` because the workspace also hosts the example/consumer
|
|
# crates under `applications/`, which are not held to the engine's bar.
|
|
# Keeping the block per-crate turns an accidental divergence into an
|
|
# intentional, documented one.
|
|
# Keep these four blocks BYTE-IDENTICAL when changing one.
|
|
[lints.rust]
|
|
unsafe_code = "forbid"
|
|
|
|
[lints.clippy]
|
|
all = { level = "deny", priority = -1 }
|
|
pedantic = { level = "warn", priority = -1 }
|
|
nursery = { level = "warn", priority = -1 }
|
|
# Justified allows (lossy numeric casts are pervasive + intentional in the
|
|
# ranking/scoring math; module_name_repetitions is idiomatic for the flat
|
|
# module layout documented in CLAUDE.md):
|
|
cast_possible_truncation = "allow"
|
|
module_name_repetitions = "allow"
|
|
unwrap_used = "deny"
|
|
|
|
[[example]]
|
|
name = "quickstart"
|
|
|
|
[[example]]
|
|
name = "foryou_feed"
|
|
|
|
[[example]]
|
|
name = "axum_embedding"
|
|
|
|
[[example]]
|
|
name = "actix_embedding"
|
|
|
|
[[example]]
|
|
name = "cli_embedding"
|
|
|
|
[[example]]
|
|
name = "ann_grid_search"
|
|
|
|
[[test]]
|
|
name = "sandboxed_storage"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m9p1_scopes"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m9p2_membership"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m9p3_purge"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m10p1_governance"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m10p2_capabilities"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m10p3_provenance"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "metrics_integration"
|
|
required-features = ["metrics"]
|
|
|
|
[[test]]
|
|
name = "m6_crash_surfaces"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m7_crash_property"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m7p2_load"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m7_uat"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "tantivy_merge"
|
|
|
|
[[test]]
|
|
name = "backup"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m7p3_social_scale"
|
|
|
|
[[test]]
|
|
name = "m8p2_replication"
|
|
|
|
[[test]]
|
|
name = "m8p2_replication_durability"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m8p3_crdt"
|
|
|
|
[[test]]
|
|
name = "m8p3_reconcile_production"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m8p4_session"
|
|
|
|
[[test]]
|
|
name = "m8p5_multitenancy"
|
|
|
|
[[test]]
|
|
name = "m8_uat"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "vector_usearch"
|
|
|
|
[[test]]
|
|
name = "m0m10_recovery"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m0m10_durability"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "review_pass2_zone_a_sessions"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m11p5_membership_record"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m12p6_graph_persistence"
|
|
required-features = ["test-utils"]
|
|
|
|
[[bench]]
|
|
name = "signals"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "storage"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "vector"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "filters"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "ranking"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "diversity"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "query"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "session"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "text_index"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "fusion"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "search"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "social"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "recovery"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "scale"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "sort"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "wal"
|
|
harness = false
|