c8ea05b032
Root cause: after a checkpoint-based snapshot install the engine WAL is empty, so wal_term_mark() reports tail_term=0. decide_join compares (tail_term, frontier) lexicographically — tail_term FIRST — so 0 < leader_term classifies the reseeded shard ReseedRequired on EVERY boot regardless of the correctly-seeded frontier, re-latching the marker and self-restarting forever. Observed live on tidaldb-2: 30 CrashLoopBackOff restarts, leader tidaldb-1 term 5, baseline=536647, the frontier seeded correctly (from_seqno=536647) yet the loop persists because the (tail_term, frontier) compare never reaches the frontier. Fix 1 (already in tree): seed the post-open frontier from sentinel.snapshot_seq, not last_wal_seq() (which a checkpoint restore leaves at 0). Fix 2 (loop-breaker): durably synthesize the artifact's kind-3 TERM_MARKER WAL record in the post-open reseed seed, at the artifact's captured term + the reseed-leader region (threaded through an extended 18-byte install sentinel, back-compat with 10/8-byte). Makes wal_term_mark() truthful on this boot AND every reboot (blob records are NOT checkpoint-filtered on recovery), so decide_join returns Clean. Truthful, not a bypass: the artifact IS the leader's authoritative state at (term, seq); a genuinely-divergent node (no install sentinel) still surfaces tail_term > term -> Quarantine. Crash-idempotent via a monotonic-by-term guard. Fix 3: node-level reseed-restart coordinator — the single process-wide exit fires once, only after every hosted shard requests a restart or a bounded grace elapses, so one shard's self-restart never aborts a co-hosted sibling's in-flight install (S>1). No-op on the S=1 production topology. Fix 4: is_ready() returns 503 while any reseed marker (SnapshotRequired or Quarantine) is latched, closing the plain-restart serve-while-behind gap; readiness is bounded staleness, not "ready the instant the process is up". Tests: decide_join loop/fix/bounded-reseed unit; install-sentinel 18-byte round-trip + back-compat; engine durability (term marker survives a checkpoint advanced past it + crash-reopen); reseed-restart gate (5 cases); reseed_install carries the term. Verified: cluster_reseed 4/4 (zero-loss rolling restart x2, quarantine reseed, failover oracle), reseed_install 3/3, m12_reseed_term_marker 4/4, cluster_membership mp_idle/mp_dns/mp_remove x2, tidal-server lib 154/154. mp_scale_3_5_3 and mp_seed_join_snapshot_catchup OOM on this host (22GB colima VM); their /health/startup failure is process-down, not the is_ready path Fix 4 touches.
279 lines
6.2 KiB
TOML
279 lines
6.2 KiB
TOML
[package]
|
|
name = "tidaldb"
|
|
version = "0.1.0"
|
|
edition = "2024"
|
|
rust-version = "1.91"
|
|
description = "Embeddable database for personalized content ranking"
|
|
license = "MIT"
|
|
|
|
[features]
|
|
default = ["metrics"]
|
|
test-utils = ["dep:tempfile"]
|
|
metrics = [] # hand-rolled HTTP, no new crate deps
|
|
# m11p9 chaos testing: WAL slow-fsync + disk-full injection (src/fault.rs).
|
|
# NOT in `default` and never passed by the production image build, so the
|
|
# shipped binary compiles it out entirely. Inert until an env var arms it even
|
|
# when compiled in. See src/fault.rs.
|
|
fault-injection = []
|
|
|
|
[dependencies]
|
|
base64 = "0.22"
|
|
blake3 = "1"
|
|
crossbeam = "0.8"
|
|
dashmap = "6"
|
|
fjall = "3"
|
|
lru = "0.12"
|
|
fs4 = "0.8"
|
|
rand = "0.9"
|
|
roaring = "0.10"
|
|
# Inline storage for the per-candidate ranking signal-snapshot (ranking/executor):
|
|
# the common formula-sort breadth fits inline, so the hot scoring loop builds the
|
|
# explain-ability breakdown with zero heap allocation. Already in the lock
|
|
# transitively (tantivy/fjall), so no new dependency surface — promoted to a
|
|
# direct dep here, matching the arc-swap/blake3/base64 pattern.
|
|
smallvec = "1"
|
|
serde = { version = "1", features = ["derive"] }
|
|
serde_json = "1"
|
|
thiserror = "2"
|
|
tantivy = "0.22"
|
|
tempfile = { version = "3", optional = true }
|
|
tracing = "0.1"
|
|
usearch = "2.24.0"
|
|
|
|
# macOS-only: a plain fsync(2) on Apple platforms does NOT flush the storage
|
|
# device's volatile write cache (Apple documents this explicitly); true
|
|
# crash durability requires fcntl(fd, F_FULLFSYNC). We use rustix's *safe*
|
|
# `fcntl_fullfsync` wrapper rather than a raw `libc::fcntl` call so the WAL's
|
|
# macOS durability path stays compatible with this crate's `unsafe_code =
|
|
# "forbid"` posture. rustix is already in the dependency tree (via fjall), so
|
|
# this adds no new compile-time cost on Linux (where it is not pulled in).
|
|
[target.'cfg(target_os = "macos")'.dependencies]
|
|
rustix = { version = "1", features = ["fs"] }
|
|
|
|
[dev-dependencies]
|
|
actix-web = "4"
|
|
axum = "0.8"
|
|
criterion = { version = "0.5", features = ["html_reports"] }
|
|
proptest = "1"
|
|
tempfile = "3"
|
|
tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal"] }
|
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
|
|
|
# ── tidal-crate lint posture (single source of truth) ──────────────────────
|
|
# The four tidal crates (tidaldb / tidal-net / tidal-server / tidalctl) carry an
|
|
# IDENTICAL, stricter-than-workspace [lints] posture and deliberately DO NOT
|
|
# inherit `[workspace.lints]` (`lints.workspace = false` is the Cargo default).
|
|
# Rationale: this is the embedded recommendation DB plus its transport, server,
|
|
# and CLI — a correctness-critical, mostly self-contained subsystem held to a
|
|
# higher bar (`unsafe_code = forbid`, `clippy::all = deny`, `unwrap_used = deny`).
|
|
# `unwrap_used = "deny"` is kept per-crate rather than promoted into
|
|
# `[workspace.lints]` because the workspace also hosts the example/consumer
|
|
# crates under `applications/`, which are not held to the engine's bar.
|
|
# Keeping the block per-crate turns an accidental divergence into an
|
|
# intentional, documented one.
|
|
# Keep these four blocks BYTE-IDENTICAL when changing one.
|
|
[lints.rust]
|
|
unsafe_code = "forbid"
|
|
|
|
[lints.clippy]
|
|
all = { level = "deny", priority = -1 }
|
|
pedantic = { level = "warn", priority = -1 }
|
|
nursery = { level = "warn", priority = -1 }
|
|
# Justified allows (lossy numeric casts are pervasive + intentional in the
|
|
# ranking/scoring math; module_name_repetitions is idiomatic for the flat
|
|
# module layout documented in CLAUDE.md):
|
|
cast_possible_truncation = "allow"
|
|
module_name_repetitions = "allow"
|
|
unwrap_used = "deny"
|
|
|
|
[[example]]
|
|
name = "quickstart"
|
|
|
|
[[example]]
|
|
name = "foryou_feed"
|
|
|
|
[[example]]
|
|
name = "axum_embedding"
|
|
|
|
[[example]]
|
|
name = "actix_embedding"
|
|
|
|
[[example]]
|
|
name = "cli_embedding"
|
|
|
|
[[example]]
|
|
name = "ann_grid_search"
|
|
|
|
[[test]]
|
|
name = "sandboxed_storage"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m9p1_scopes"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m9p2_membership"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m9p3_purge"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m10p1_governance"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m10p2_capabilities"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m10p3_provenance"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "metrics_integration"
|
|
required-features = ["metrics"]
|
|
|
|
[[test]]
|
|
name = "m6_crash_surfaces"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m7_crash_property"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m7p2_load"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m7_uat"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "tantivy_merge"
|
|
|
|
[[test]]
|
|
name = "backup"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m7p3_social_scale"
|
|
|
|
[[test]]
|
|
name = "m8p2_replication"
|
|
|
|
[[test]]
|
|
name = "m8p2_replication_durability"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m8p3_crdt"
|
|
|
|
[[test]]
|
|
name = "m8p3_reconcile_production"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m8p4_session"
|
|
|
|
[[test]]
|
|
name = "m8p5_multitenancy"
|
|
|
|
[[test]]
|
|
name = "m8_uat"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "vector_usearch"
|
|
|
|
[[test]]
|
|
name = "m0m10_recovery"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m0m10_durability"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "review_pass2_zone_a_sessions"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m11p5_membership_record"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m12_reseed_term_marker"
|
|
required-features = ["test-utils"]
|
|
|
|
[[test]]
|
|
name = "m12p6_graph_persistence"
|
|
required-features = ["test-utils"]
|
|
|
|
[[bench]]
|
|
name = "signals"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "storage"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "vector"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "filters"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "ranking"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "diversity"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "query"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "session"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "text_index"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "fusion"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "search"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "social"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "recovery"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "scale"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "sort"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "wal"
|
|
harness = false
|