bb21e69ae6
m12p1 (measurement truth): TidalDb::vector_search_items pure k-NN probe + POST /vector_search (standalone + region node, merge-by-distance) + tidal-stress --verify-recall (deterministic id-keyed corpus, in-RAM brute-force cosine oracle, open-loop ramp → recall@k + true p99 + read-knee + JSON/gate exit). Repaired fabricated p99 columns (mean-as-p99) in social-scale.md / scale.rs. Verified real: recall@10=0.9997 at 20k/1536-D vs brute-force. m12p2 (G1 unblock): ANN candidate-gen wired into RETRIEVE — for_you=preference vector, related=seed embedding (similar_to), graceful scan-fallback. Cached per-signal-type top-K (signals/ledger/hot_top_k.rs, decay-order-invariant) so trending serves O(K). related over HTTP (FeedQuery.similar_to). Harness gains --feed-profile / --seed-preferences. Verified: trending retrieve p99 3.5-7.7ms. m12p3 (G2): per-query ef_search now honored (RwLock epoch-guard with_expansion, shared guard for same-ef concurrency) + dimension-aware brute→HNSW crossover usearch_min_vectors(dim) + memory_usage() + examples/ann_grid_search.rs. Measured 1536-D/100k clustered: default M=16/ef_c=400/F16/ef_s=200 clears G1+G2 (recall 0.997, p99 1.4ms); F16 -0.25% vs F32; Int8 rejected (-28%). Recall corpus is now clustered (Gaussian mixture) in grid + harness.
271 lines
6.1 KiB
TOML
271 lines
6.1 KiB
TOML
[package]
|
|
name = "tidaldb"
|
|
version = "0.1.0"
|
|
edition = "2024"
|
|
rust-version = "1.91"
|
|
description = "Embeddable database for personalized content ranking"
|
|
license = "MIT"
|
|
|
|
[features]
|
|
default = ["metrics"]
|
|
test-utils = ["dep:tempfile"]
|
|
metrics = [] # hand-rolled HTTP, no new crate deps
|
|
# m11p9 chaos testing: WAL slow-fsync + disk-full injection (src/fault.rs).
|
|
# NOT in `default` and never passed by the production image build, so the
|
|
# shipped binary compiles it out entirely. Inert until an env var arms it even
|
|
# when compiled in. See src/fault.rs.
|
|
fault-injection = []
|
|
|
|
[dependencies]
|
|
base64 = "0.22"
|
|
blake3 = "1"
|
|
crossbeam = "0.8"
|
|
dashmap = "6"
|
|
fjall = "3"
|
|
lru = "0.12"
|
|
fs4 = "0.8"
|
|
rand = "0.9"
|
|
roaring = "0.10"
|
|
# Inline storage for the per-candidate ranking signal-snapshot (ranking/executor):
|
|
# the common formula-sort breadth fits inline, so the hot scoring loop builds the
|
|
# explain-ability breakdown with zero heap allocation. Already in the lock
|
|
# transitively (tantivy/fjall), so no new dependency surface — promoted to a
|
|
# direct dep here, matching the arc-swap/blake3/base64 pattern.
|
|
smallvec = "1"
|
|
serde = { version = "1", features = ["derive"] }
|
|
serde_json = "1"
|
|
thiserror = "2"
|
|
tantivy = "0.22"
|
|
tempfile = { version = "3", optional = true }
|
|
tracing = "0.1"
|
|
usearch = "2.24.0"
|
|
|
|
# macOS-only: a plain fsync(2) on Apple platforms does NOT flush the storage
|
|
# device's volatile write cache (Apple documents this explicitly); true
|
|
# crash durability requires fcntl(fd, F_FULLFSYNC). We use rustix's *safe*
|
|
# `fcntl_fullfsync` wrapper rather than a raw `libc::fcntl` call so the WAL's
|
|
# macOS durability path stays compatible with this crate's `unsafe_code =
|
|
# "forbid"` posture. rustix is already in the dependency tree (via fjall), so
|
|
# this adds no new compile-time cost on Linux (where it is not pulled in).
|
|
[target.'cfg(target_os = "macos")'.dependencies]
|
|
rustix = { version = "1", features = ["fs"] }
|
|
|
|
[dev-dependencies]
|
|
actix-web = "4"
|
|
axum = "0.8"
|
|
criterion = { version = "0.5", features = ["html_reports"] }
|
|
proptest = "1"
|
|
tempfile = "3"
|
|
tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal"] }
|
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
|
|
|
# ── tidal-crate lint posture (single source of truth) ──────────────────────
|
|
# The four tidal crates (tidaldb / tidal-net / tidal-server / tidalctl) carry an
|
|
# IDENTICAL, stricter-than-workspace [lints] posture and deliberately DO NOT
|
|
# inherit `[workspace.lints]` (`lints.workspace = false` is the Cargo default).
|
|
# Rationale: this is the embedded recommendation DB plus its transport, server,
|
|
# and CLI — a correctness-critical, mostly self-contained subsystem held to a
|
|
# higher bar (`unsafe_code = forbid`, `clippy::all = deny`, `unwrap_used = deny`).
|
|
# `unwrap_used = "deny"` is kept per-crate rather than promoted into
|
|
# `[workspace.lints]` because the workspace also hosts the example/consumer
|
|
# crates under `applications/`, which are not held to the engine's bar.
|
|
# Keeping the block per-crate turns an accidental divergence into an
|
|
# intentional, documented one.
|
|
# Keep these four blocks BYTE-IDENTICAL when changing one.
|
|
[lints.rust]
|
|
unsafe_code = "forbid"
|
|
|
|
[lints.clippy]
|
|
all = { level = "deny", priority = -1 }
|
|
pedantic = { level = "warn", priority = -1 }
|
|
nursery = { level = "warn", priority = -1 }
|
|
# Justified allows (lossy numeric casts are pervasive + intentional in the
|
|
# ranking/scoring math; module_name_repetitions is idiomatic for the flat
|
|
# module layout documented in CLAUDE.md):
|
|
cast_possible_truncation = "allow"
|
|
module_name_repetitions = "allow"
|
|
unwrap_used = "deny"
|
|
|
|
[[example]]
|
|
name = "quickstart"
|
|
|
|
[[example]]
|
|
name = "foryou_feed"
|
|
|
|
[[example]]
|
|
name = "axum_embedding"
|
|
|
|
[[example]]
|
|
name = "actix_embedding"
|
|
|
|
[[example]]
|
|
name = "cli_embedding"
|
|
|
|
[[example]]
|
|
name = "ann_grid_search"
|
|
|
|
[[test]]
|
|
name = "sandboxed_storage"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m9p1_scopes"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m9p2_membership"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m9p3_purge"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m10p1_governance"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m10p2_capabilities"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m10p3_provenance"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "metrics_integration"
|
|
required-features = ["metrics"]
|
|
|
|
[[test]]
|
|
name = "m6_crash_surfaces"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m7_crash_property"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m7p2_load"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m7_uat"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "tantivy_merge"
|
|
|
|
[[test]]
|
|
name = "backup"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m7p3_social_scale"
|
|
|
|
[[test]]
|
|
name = "m8p2_replication"
|
|
|
|
[[test]]
|
|
name = "m8p2_replication_durability"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m8p3_crdt"
|
|
|
|
[[test]]
|
|
name = "m8p3_reconcile_production"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m8p4_session"
|
|
|
|
[[test]]
|
|
name = "m8p5_multitenancy"
|
|
|
|
[[test]]
|
|
name = "m8_uat"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "vector_usearch"
|
|
|
|
[[test]]
|
|
name = "m0m10_recovery"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m0m10_durability"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "review_pass2_zone_a_sessions"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m11p5_membership_record"
|
|
required-features = ["test-utils"]
|
|
|
|
[[bench]]
|
|
name = "signals"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "storage"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "vector"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "filters"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "ranking"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "diversity"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "query"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "session"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "text_index"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "fusion"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "search"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "social"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "recovery"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "scale"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "sort"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "wal"
|
|
harness = false
|