From 1092d34c395b9b6e40f02eb0ca2168d2a4c887ab Mon Sep 17 00:00:00 2001 From: jx12n Date: Tue, 9 Jun 2026 17:06:34 -0600 Subject: [PATCH] feat: kubernetes deployment, OpenAPI spec, guides, and docker consolidation - Add k8s/ manifests (StatefulSet, kustomize, PDB, ServiceMonitor) + docs/runbooks/kubernetes.md - Add tidal-server/src/openapi.rs (utoipa OpenAPI spec) and wire into router - Add docs/guides/ (build-a-feed-app, embeddings, server-deployment) + foryou_feed example - Consolidate tidal/docker/ into root docker/ (single canonical home) - Update API.md, QUICKSTART.md, README.md, CLAUDE.md, check-docs.sh accordingly --- .dockerignore | 2 +- API.md | 17 + CLAUDE.md | 4 + Cargo.lock | 25 + QUICKSTART.md | 17 +- README.md | 4 + ai-lookup/index.md | 2 +- ai-lookup/services/ranking-profiles.md | 127 ++++- docker/cluster/Dockerfile | 65 ++- docker/deploy/Dockerfile | 74 ++- docker/docker-compose.yml | 5 +- docker/standalone/Dockerfile | 66 ++- docs/README.md | 11 +- docs/guides/build-a-feed-app.md | 620 ++++++++++++++++++++++++ docs/guides/embeddings.md | 436 +++++++++++++++++ docs/guides/server-deployment.md | 364 ++++++++++++++ docs/runbooks/cluster.md | 410 +++++++++++++--- docs/runbooks/kubernetes.md | 206 ++++++++ k8s/kustomization.yaml | 26 + k8s/namespace.yaml | 6 + k8s/poddisruptionbudget.yaml | 20 + k8s/schema-configmap.yaml | 50 ++ k8s/secret.example.yaml | 25 + k8s/service.yaml | 24 + k8s/servicemonitor.yaml | 21 + k8s/statefulset.yaml | 142 ++++++ scripts/check-docs.sh | 11 + tidal-server/Cargo.toml | 6 + tidal-server/config/default-schema.yaml | 2 + tidal-server/src/cluster.rs | 275 +++++++++-- tidal-server/src/config.rs | 34 ++ tidal-server/src/dto.rs | 67 ++- tidal-server/src/lib.rs | 1 + tidal-server/src/openapi.rs | 225 +++++++++ tidal-server/src/router.rs | 87 +++- tidal-server/tests/standalone.rs | 35 ++ tidal/CLAUDE.md | 8 +- tidal/Cargo.toml | 3 + tidal/docker/cluster/Dockerfile | 73 --- tidal/docker/deploy/Dockerfile | 64 --- tidal/docker/docker-compose.yml | 35 -- tidal/docker/prometheus.yml | 8 - tidal/docker/standalone/Dockerfile | 66 --- tidal/examples/foryou_feed.rs | 472 ++++++++++++++++++ tidal/examples/quickstart.rs | 4 + 45 files changed, 3782 insertions(+), 463 deletions(-) create mode 100644 docs/guides/build-a-feed-app.md create mode 100644 docs/guides/embeddings.md create mode 100644 docs/guides/server-deployment.md create mode 100644 docs/runbooks/kubernetes.md create mode 100644 k8s/kustomization.yaml create mode 100644 k8s/namespace.yaml create mode 100644 k8s/poddisruptionbudget.yaml create mode 100644 k8s/schema-configmap.yaml create mode 100644 k8s/secret.example.yaml create mode 100644 k8s/service.yaml create mode 100644 k8s/servicemonitor.yaml create mode 100644 k8s/statefulset.yaml create mode 100644 tidal-server/src/openapi.rs delete mode 100644 tidal/docker/cluster/Dockerfile delete mode 100644 tidal/docker/deploy/Dockerfile delete mode 100644 tidal/docker/docker-compose.yml delete mode 100644 tidal/docker/prometheus.yml delete mode 100644 tidal/docker/standalone/Dockerfile create mode 100644 tidal/examples/foryou_feed.rs diff --git a/.dockerignore b/.dockerignore index 99b81a7..ce8dfe4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,6 @@ # Allowlist .dockerignore — ignore everything, then re-include only the # workspace sources cargo needs to build `-p tidal-server` from the repo root. -# The `tidal/docker/*` Dockerfiles `COPY . .` against this repository root. +# The `docker/*` Dockerfiles `COPY . .` against this repository root. * # Workspace manifests + lockfile (cargo needs every member manifest present to diff --git a/API.md b/API.md index ad83979..0b25728 100644 --- a/API.md +++ b/API.md @@ -1037,6 +1037,23 @@ GET /search?query=jazz+piano&user_id=123&limit=10 | `GET /health` | No | Readiness probe — 200 when ready, 503 when shutting down | | `GET /health/startup` | No | Startup probe — always 200 | | `GET /health/live` | No | Liveness probe — always 200 | +| `GET /openapi.json` | No | Machine-readable OpenAPI 3.1 spec for this server (data + cluster routes) | + +These map directly to Kubernetes startup/liveness/readiness probes — see +[docs/runbooks/kubernetes.md](docs/runbooks/kubernetes.md). + +**`GET /openapi.json`** returns the canonical, served HTTP API contract. It is +generated from the handlers (so it never drifts from the code) and is exempt +from auth — clients need it to learn how to authenticate. Load it into any +OpenAPI viewer or generate a client: + +```bash +curl -s http://localhost:9400/openapi.json | jq '.info, (.paths | keys)' +``` + +The cluster server serves a superset document that also includes the +`/cluster/*` and `/sharded/*` routes. See +[docs/guides/server-deployment.md](docs/guides/server-deployment.md). **`GET /health` response:** diff --git a/CLAUDE.md b/CLAUDE.md index fc3aea3..fc55f70 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,6 +12,10 @@ A single-node-first, embeddable Rust database for the **personalized content ran | If you need to... | Read this | |-------------------|-----------| | **Get started quickly** | [README.md](README.md) → [QUICKSTART.md](QUICKSTART.md) | +| **Build a real app (feed)** | [docs/guides/build-a-feed-app.md](docs/guides/build-a-feed-app.md) | +| **Wire an embedding model** | [docs/guides/embeddings.md](docs/guides/embeddings.md) | +| **Run / deploy the HTTP server** | [docs/guides/server-deployment.md](docs/guides/server-deployment.md) | +| **Deploy on Kubernetes** | [docs/runbooks/kubernetes.md](docs/runbooks/kubernetes.md) + [k8s/](k8s/) | | **Understand the vision** | [VISION.md](VISION.md) | | **See use cases and surfaces** | [USE_CASES.md](USE_CASES.md) | | **See sequence diagrams** | [SEQUENCE.md](SEQUENCE.md) | diff --git a/Cargo.lock b/Cargo.lock index 98a90ee..4907b3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3601,6 +3601,7 @@ dependencies = [ "tower-http 0.6.8", "tracing", "tracing-subscriber", + "utoipa", ] [[package]] @@ -4084,6 +4085,30 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "utoipa" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bde15df68e80b16c7d16b9616e80770ad158988daa56a27dccd1e55558b0160" +dependencies = [ + "indexmap 2.13.0", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba0b99ee52df3028635d93840c797102da61f8a7bb3cf751032455895b52ef8" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "syn", +] + [[package]] name = "uuid" version = "1.21.0" diff --git a/QUICKSTART.md b/QUICKSTART.md index 877917b..00bd6a2 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -42,14 +42,18 @@ let mut schema = SchemaBuilder::new(); // View signal: 7-day half-life, three windows, velocity enabled. // You declare the decay. tidalDB applies it at query time — no formula to maintain. +// `positive_engagement(true)` is what makes this signal fold into a user's taste +// vector when you record it with context (Step 7) — without it, personalization +// falls back to name-based heuristics. Declare it explicitly on signals that +// should shape preferences. let _ = schema.signal("view", EntityKind::Item, DecaySpec::Exponential { half_life: Duration::from_secs(7 * 24 * 3600), -}).windows(&[Window::OneHour, Window::TwentyFourHours, Window::AllTime]).velocity(true).add(); +}).windows(&[Window::OneHour, Window::TwentyFourHours, Window::AllTime]).velocity(true).positive_engagement(true).add(); // Like signal: 30-day half-life. Durable engagement decays slowly. let _ = schema.signal("like", EntityKind::Item, DecaySpec::Exponential { half_life: Duration::from_secs(30 * 24 * 3600), -}).windows(&[Window::AllTime]).velocity(false).add(); +}).windows(&[Window::AllTime]).velocity(false).positive_engagement(true).add(); // Share signal: 3-day half-life. Short-lived but strongly trending. let _ = schema.signal("share", EntityKind::Item, DecaySpec::Exponential { @@ -288,12 +292,19 @@ This flushes the WAL, checkpoints signal state, and persists indexes. In persist | Topic | Where to look | |-------|--------------| +| **Build a real app (TikTok-style feed)** | [docs/guides/build-a-feed-app.md](docs/guides/build-a-feed-app.md) | +| **Wire a real embedding model** | [docs/guides/embeddings.md](docs/guides/embeddings.md) | +| **Run the HTTP server / deploy** | [docs/guides/server-deployment.md](docs/guides/server-deployment.md) | +| **Deploy on Kubernetes** | [docs/runbooks/kubernetes.md](docs/runbooks/kubernetes.md) | | Full API reference | [API.md](API.md) | +| HTTP API contract (served, machine-readable) | `GET /openapi.json` (see server-deployment guide) | | Filters — format, duration, location, engagement thresholds | [API.md — Filters](API.md#filters) | | Diversity constraints | [API.md — Diversity Constraints](API.md#diversity-constraints) | -| All 25 ranking profiles | [API.md — Sort Modes](API.md#sort-modes) | +| All 25 ranking profiles | [ai-lookup/services/ranking-profiles.md](ai-lookup/services/ranking-profiles.md) | | Cohort-scoped trending | [API.md — Cohorts](API.md#cohort-definitions) | | Collections and saved searches | [API.md — Collections](API.md#collections) | +| Short-video feed example | `tidal/examples/foryou_feed.rs` | | Axum embedding example | `tidal/examples/axum_embedding.rs` | | 14 content discovery surfaces | [USE_CASES.md](USE_CASES.md) | | Architecture and design decisions | [ARCHITECTURE.md](ARCHITECTURE.md) | +| Cluster mode (experimental) | [docs/runbooks/cluster.md](docs/runbooks/cluster.md) | diff --git a/README.md b/README.md index 9dcf7ec..7094702 100644 --- a/README.md +++ b/README.md @@ -275,6 +275,10 @@ migration scenarios inside your own test suites. |----------|----------| | [QUICKSTART.md](QUICKSTART.md) | Step-by-step guide: schema, ingest, signals, ranking, search | | [API.md](API.md) | Full API reference with code examples | +| [Build a feed app](docs/guides/build-a-feed-app.md) | End-to-end TikTok/Reels-style "For You" feed tutorial | +| [Embedding integration](docs/guides/embeddings.md) | Wiring a real embedding model into the write + query paths | +| [Server deployment](docs/guides/server-deployment.md) | Running `tidal-server`: config, auth, OpenAPI, Docker | +| [Kubernetes runbook](docs/runbooks/kubernetes.md) | Deploying on k8s (manifests in [`k8s/`](k8s/)) | | [VISION.md](VISION.md) | Problem statement and design thesis | | [ARCHITECTURE.md](ARCHITECTURE.md) | Storage, signal system, vector index, query pipeline | | [USE_CASES.md](USE_CASES.md) | 14 content discovery surfaces, filter and sort references | diff --git a/ai-lookup/index.md b/ai-lookup/index.md index b15a7ee..a272a8f 100644 --- a/ai-lookup/index.md +++ b/ai-lookup/index.md @@ -4,7 +4,7 @@ |-------|------|------------|---------|---------| | Entities | `services/entities.md` | High | 2026-02-19 | Items, Users, Creators — core data model | | Signals | `services/signals.md` | High | 2026-02-19 | Typed event streams with decay, velocity, windowed aggregation | -| Ranking Profiles | `services/ranking-profiles.md` | High | 2026-02-19 | Named scoring functions declared in schema | +| Ranking Profiles | `services/ranking-profiles.md` | High | 2026-06-09 | All 25 built-in profiles: strategy + sort + diversity + UC map | | Query Language | `features/query-language.md` | High | 2026-02-19 | RETRIEVE/SEARCH/SIGNAL query surface | | Sort Modes | `features/sort-modes.md` | High | 2026-02-19 | 25+ native sort modes (hot, trending, rising, etc.) | | Filters | `features/filters.md` | High | 2026-02-19 | Composable filter dimensions across all queries | diff --git a/ai-lookup/services/ranking-profiles.md b/ai-lookup/services/ranking-profiles.md index 17e5825..a16ac4c 100644 --- a/ai-lookup/services/ranking-profiles.md +++ b/ai-lookup/services/ranking-profiles.md @@ -1,38 +1,115 @@ # Ranking Profiles -**Last Updated:** 2026-02-19 +**Last Updated:** 2026-06-09 **Confidence:** High ## Summary -Ranking profiles are named, versioned scoring functions declared in schema. They reference signals, relationship weights, recency curves, and diversity rules. Profiles live in the database, are versioned alongside data, and can be swapped at query time by name. +A ranking profile is a named, versioned bundle that fully specifies how a query turns candidates into an ordered result page: it names the **retrieval strategy** (how candidates are sourced — scan, ANN, signal-ranked, relationship, hybrid, cohort), the **primary sort** (the ordering formula — trending, hot, top-window, etc.), any **boosts / gates / penalties / excludes** (signal-driven score adjustments and filters), the **diversity** rules (per-creator caps and format-mix caps), and an **exploration** fraction (random injection to break filter bubbles). Profiles are selected by name at query time (`.profile("for_you")`), declared in schema, and versioned alongside data — old versions remain queryable. tidalDB ships **25 built-in profiles** registered at `version = 1` with `is_builtin = true`; applications add their own with the schema builder or the server config YAML. **Key Facts:** -- Profiles are schema-level declarations, not application code -- Each profile defines: primary signals, secondary signals, boosts, gates, and diversity rules -- The same profile can operate on different candidate sets (global vs category vs social graph) -- Profiles are versioned — old versions remain queryable +- Profiles are schema-level declarations, not application code. The app names a profile; the database executes the whole pipeline. +- The same profile operates over different candidate sets (global / category / social graph / cohort) depending on its candidate strategy and the query's filters. +- Personalization (`for_you`, `following`, `related`, `notification`, `date_saved`) requires `FOR USER` context — pass `.for_user(uid)` on the query. Without it the executor cannot read the user's preference vector / saved-state and either degrades to non-personalized scoring or, for `date_saved`, returns an error. +- **Preference-vector personalization only updates from signals declared `.positive_engagement(true)`.** `signal_with_context(..)` folds the item embedding into the user's taste vector only for positive-engagement signal types; negative signals (skip/hide/block) update seen-state and hard-negatives but do not pull the taste vector toward the item. Declare engagement signals accordingly or `for_you` will not learn. +- Built-in defaults below are read straight from `tidal/src/ranking/builtins.rs`. Where a value is intentionally application-tuning territory, it is described qualitatively rather than invented. -**File Pointer:** `VISION.md:43-55` +**File Pointers:** `tidal/src/ranking/builtins.rs` (the 25 built-ins), `tidal/src/ranking/profile.rs` (`RankingProfile`, `DiversitySpec`, `Sort`, `Boost`, `CandidateStrategy`), `VISION.md:43-55`. -## Built-in Profiles +## The 25 Built-in Profiles -| Profile | Primary Signal | Use Case | -|---------|---------------|----------| -| `for_you` | preference_match + engagement_velocity | Personalized feed | -| `search` | text_relevance + semantic_similarity | Search results | -| `trending` | share_velocity + view_velocity | Trending surfaces | -| `rising` | velocity relative to baseline, age-boosted | Breakout content | -| `following` | created_at DESC | Subscription feed | -| `related` | semantic_similarity + collaborative_filtering | Up next / related | -| `browse` | quality_score (completion + like_ratio + reach) | Category pages | -| `hot` | score / (age + 2)^gravity | Community frontpages | -| `controversial` | max(positive * negative signals) | Debate surfaces | -| `hidden_gems` | high quality, inverse reach | Discovery | -| `notification` | relationship_strength + item quality | Push prioritization | -| `live` | relationship_weight + viewer_count | Live surfaces | +`DiversitySpec` defaults to **no diversity** (`max_per_creator = None`, `format_mix_max_fraction = None`) and `exploration` defaults to `0.0`. Profiles below list only the values they override. "Needs FOR USER" marks profiles whose ordering or scoring depends on per-user context. + +| Profile | Optimizes for | Primary sort | Diversity / exploration (defaults) | Needs FOR USER | Use case | +|---------|---------------|--------------|------------------------------------|----------------|----------| +| `trending` | Pure short-window engagement velocity (`view_vel + 2·share_vel`, 24h) | `Trending` | `max_per_creator=1`; no exploration | No | UC-03 | +| `hot` | Cumulative score decayed by age (Reddit/HN style) | `Hot { gravity = 1.8 }` | `max_per_creator=2`; no exploration | No | UC-14 | +| `new` | Freshest content first | `New` (`created_at DESC`) | none | No | UC-03, UC-04 | +| `for_you` | Personalized home feed (decayed view/like + 24h share velocity, on top of preference match) | `Hot { gravity = 1.5 }` | `max_per_creator=2`, `format_mix_max_fraction=0.4`, `exploration=0.1` | Yes | UC-01 | +| `following` | Recent content from creators the user follows | `New` | `max_per_creator=3`; no exploration | Yes | UC-04 | +| `related` | "More like this" for a seed item (semantic similarity + completion-weighted quality) | `Hot { gravity = 1.2 }` | `max_per_creator=2`; no exploration | No (use `.similar_to(item_id)`) | UC-05 | +| `notification` | High-velocity content from followed creators, worth interrupting for | `Trending` | `max_per_creator=1`; no exploration | Yes | UC-07 | +| `search` | Text + vector relevance (RRF fusion) with a light view/like quality overlay | `None` (fused RRF score from search executor is the primary order) | none (caller sets diversity); no exploration (deterministic) | No (personalizable via `.for_user`) | UC-02, UC-10, UC-11 | +| `top_week` | Best-of over a 7-day window | `TopWindow { SevenDays }` | none | No | UC-06 | +| `top_month` | Best-of over a 30-day window | `TopWindow { ThirtyDays }` | none | No | UC-06 | +| `top_all_time` | Cumulative quality, no decay (classic / best-of) | `TopWindow { AllTime }` | none | No | UC-06 | +| `hidden_gems` | High quality, low reach (inverse-reach boost) | `HiddenGems` | none | No | UC-13 | +| `controversial` | Maximum product of positive × negative signals (debate surfaces) | `Controversial` | none | No | UC-14 | +| `most_viewed` | Raw view count, 7-day window | `MostViewed { SevenDays }` | none | No | UC-06 | +| `most_liked` | Raw like count, 7-day window | `MostLiked { SevenDays }` | none | No | UC-06 | +| `shuffle` | Quality-weighted random ("surprise me") | `Shuffle` | `exploration=0.5` (half random, half signal-ranked); no creator cap | No | UC-06 | +| `cohort_trending` | Trending scoped to a named audience cohort ("trending for people like you") | `Trending` | `max_per_creator=1`; no exploration | No (use `.cohort("name")`) | UC-15 | +| `live` | Currently-live content by viewer count, lifting creators the user's circle watches | `LiveViewerCount` | `max_per_creator=1`; no exploration | Personalizable (relationship boost uses `.for_user`) | UC-12 | +| `alphabetical_asc` | Title A→Z (case-insensitive) | `AlphabeticalAsc` | none | No | UC-06 | +| `alphabetical_desc` | Title Z→A (case-insensitive) | `AlphabeticalDesc` | none | No | UC-06 | +| `shortest` | Quick content (`duration ASC`) | `Shortest` | none | No | UC-06 | +| `longest` | Deep dives (`duration DESC`) | `Longest` | none | No | UC-06 | +| `most_commented` | Discussion volume (comment count, all-time) | `MostCommented { AllTime }` | none | No | UC-06, UC-14 | +| `most_shared` | Virality (share count, all-time) | `MostShared { AllTime }` | none | No | UC-06 | +| `date_saved` | The user's personal library, latest save first | `DateSaved` | none | **Yes (required)** — missing `FOR USER` returns `InvalidFilter` | UC-09 | + +### Notes on specific built-ins + +- **`trending` / `cohort_trending` and the `view`/`share` boosts.** The `Sort::Trending` formula (`view_vel + 2·share_vel` over 24h) is the ordering authority on the global path and *replaces* the boost loop, so the executor skips boosts there (no double-count). The same `view`/`share` velocity boosts are not dead weight: the cohort rescore path (`rescore_with_cohort`) consumes `profile.boosts` directly to re-score against a cohort's ledger. Keep the boost weights mirroring the formula so cohort ordering matches global ordering. +- **`for_you` boosts vs. preference match.** Beyond the `Hot { gravity = 1.5 }` recency curve, `for_you` boosts decayed `view` (weight 1.0), decayed `like` (weight 2.0), and 24h `share` velocity (weight 1.5), layered on top of the user-preference-vector match. The 10% exploration injection is what keeps it from collapsing into a filter bubble. +- **`search` has `sort = None` on purpose.** The fused RRF (BM25 + ANN) score from the search executor is the primary ordering signal; the profile only adds a small `view`/`like` quality overlay (weights 0.5 / 0.8) so relevance stays in charge. Exploration is `0.0` so results are deterministic for a given query. Set diversity yourself via `SearchBuilder::diversity(..)`. +- **Three sort modes have no built-in profile shortcut.** `MostFollowed` and `CreatorEngagementRate` rank **creators** (not items) and need a creator-scoped candidate strategy plus a `follow` signal the generic schema does not guarantee; `Rising` (1h/24h view-velocity acceleration ratio) overlaps `trending`/`hot` and is sensitive to the exact window pair. All three are reachable via a custom schema profile (`SchemaBuilder::ranking_profile(..).sort(..)`); only the named-convenience built-in is omitted. + +## Candidate (retrieval) strategies + +The candidate strategy decides *which* items the profile even considers before sorting: + +| Strategy | What it does | Built-ins that use it | +|----------|--------------|-----------------------| +| `Scan` (`sort_field`) | Population scan over the item keyspace; default for built-ins (`sort_field = "created_at"`) | most population + sort-coverage profiles | +| `Relationship` | Sources candidates from the user's relationship graph (followed creators) | `following`, `notification` | +| `Ann { slot, limit }` | k-NN over an embedding slot (set via the query's `.similar_to` / `Search`'s `.vector`) | custom; `related` uses the seed embedding via the query | +| `SignalRanked { signal, window }` | Pre-orders candidates by a signal's windowed aggregate | custom | +| `Hybrid` | Fuses lexical + vector retrieval | the `search` query path (RRF fusion in the search executor) | +| `CohortTrending` | Cohort-scoped signal aggregation | reached via `.cohort("name")` with `cohort_trending` | + +> Embeddings are app-supplied. tidalDB retrieves and ranks over vectors but does **not** generate them; `RETRIEVE`/`SEARCH` route ANN through the **first declared embedding slot only** (fuse multi-modal vectors offline or use separate entity kinds). See [API.md](../../API.md) and [QUICKSTART.md](../../QUICKSTART.md). + +## Defining a custom profile + +Profiles are declared in schema (Rust `SchemaBuilder`) or, for the HTTP server, in the config YAML loaded at startup. The YAML shape mirrors the `RankingProfile` type exactly: + +```yaml +profiles: + - name: my_feed + version: 1 + # candidate_strategy: scan | { ann: { slot, limit } } + # | { signal_ranked: { signal, window } } + # | relationship | hybrid | cohort_trending + candidate_strategy: scan # default: scan over created_at + # sort: trending | rising | controversial | hidden_gems | shuffle | new + # | { hot: { gravity } } | { top_window: { window } } + # | { most_viewed: { window } } | ... + sort: { hot: { gravity: 1.8 } } + boosts: + # agg: value | velocity | decay_score | ratio | relative_velocity + - { signal: like, agg: decay_score, window: all_time, weight: 2.0 } + - { signal: share, agg: velocity, window: twenty_four_hours, weight: 1.5 } + gates: [] # drop candidates failing a signal threshold + penalties: [] # subtractive score adjustments + excludes: [] # hard removals + diversity: + max_per_creator: 2 # at most 2 items per creator in the page + format_mix_max_fraction: 0.4 # no single format > 40% of the page + exploration: 0.1 # 10% random injection (0.0 = none) +``` + +Windows are `one_hour`, `twenty_four_hours`, `seven_days`, `thirty_days`, `all_time`. Only signals declared in the same schema may be referenced by boosts/gates — the server validates references at load time and rejects a profile naming an unknown signal. Gates, penalties, and excludes are deliberately omitted from the built-ins because they would assume signals not guaranteed to exist in a generic schema; add them in your own profile once you know your signal set. + +In Rust, the equivalent lives behind `SchemaBuilder::ranking_profile(..)`; see [API.md](../../API.md) for the builder surface and the runnable `tidal/examples/foryou_feed.rs` / `tidal/examples/quickstart.rs` for end-to-end usage. ## Related Topics -- [Signals](./signals.md) -- [Query Language](../features/query-language.md) -- [Sort Modes](../features/sort-modes.md) + +- [Signals](./signals.md) — the decay/velocity/windowed-aggregation primitives boosts read +- [Entities](./entities.md) — items/users/creators the profiles rank over +- [Query Language](../features/query-language.md) — `RETRIEVE` / `SEARCH` / `SIGNAL` surface +- [Sort Modes](../features/sort-modes.md) — the 25+ native sort formulas profiles select from +- [Filters](../features/filters.md) — composable filter dimensions layered onto any profile +- [USE_CASES.md](../../USE_CASES.md) — UC-01…UC-15 and **Appendix B · Sort Mode Reference** +- [API.md](../../API.md) — builder API + HTTP routes +- [QUICKSTART.md](../../QUICKSTART.md) — first feed in minutes diff --git a/docker/cluster/Dockerfile b/docker/cluster/Dockerfile index d1565ce..cf7010f 100644 --- a/docker/cluster/Dockerfile +++ b/docker/cluster/Dockerfile @@ -1,34 +1,41 @@ # Multi-region tidalDB cluster image. # -# Runs a simulated 3-region cluster (us-east, eu-west, ap-south) behind a -# single HTTP surface on port 9500. See docs/runbooks/cluster.md for the -# operational API. -# Pin the builder to bookworm so its glibc matches the bookworm-slim runtime; -# the default tag tracks trixie, whose `libmvec.so.1` is absent on bookworm. +# Runs a 3-region cluster (us-east, eu-west, ap-south) behind a single HTTP +# surface on port 9500. Regions replicate over the real tidal-net gRPC transport +# (loopback), but all run in this single container process — there is no host or +# process isolation, so it is NOT production HA (true multi-process deployment is +# m8p10). See docs/runbooks/cluster.md for the operational API. +# +# Build context is this repository's root (the workspace `Cargo.toml` / +# `Cargo.lock` and every member crate live there). Build from the repo root: +# +# docker build -f docker/cluster/Dockerfile -t tidaldb:cluster . +# +# The repo-root allowlist `.dockerignore` strips target/, node_modules/, .git, +# and .env* from the context, so `COPY . .` brings in exactly the workspace +# sources cargo needs to resolve the member graph and build `-p tidal-server`. +# Pin the builder to bookworm so its glibc matches the bookworm-slim runtime +# below. The default `rust:1.91` tracks Debian trixie (glibc 2.41), whose +# auto-vectorized math pulls in `libmvec.so.1` — a library bookworm's glibc does +# not ship, so a trixie-built binary aborts at startup on bookworm with +# "libmvec.so.1: cannot open shared object file". FROM rust:1.91-bookworm AS builder ARG DEBIAN_FRONTEND=noninteractive WORKDIR /app -# Copy workspace manifests first for caching. -COPY Cargo.toml Cargo.lock ./ -COPY tidal/Cargo.toml tidal/Cargo.toml -COPY tidal-net/Cargo.toml tidal-net/Cargo.toml -COPY tidalctl/Cargo.toml tidalctl/Cargo.toml -COPY tidal-server/Cargo.toml tidal-server/Cargo.toml -COPY applications/forage/engine/Cargo.toml applications/forage/engine/Cargo.toml -COPY applications/forage/server/Cargo.toml applications/forage/server/Cargo.toml -COPY applications/forage/embedder/Cargo.toml applications/forage/embedder/Cargo.toml -COPY applications/iknowyou/engine/Cargo.toml applications/iknowyou/engine/Cargo.toml - -# Copy full workspace. +# Copy the full workspace and build only the server binary. The unified +# workspace requires every member manifest present to resolve metadata, so we +# copy the whole (already-pruned) context rather than a fragile manifest-only +# subset. COPY . . -# g++ builds the usearch C++ HNSW core; protobuf-compiler (protoc) is required by -# tidal-net's build script (tonic-build compiles the WAL-shipping .proto). -RUN apt-get update && apt-get install -y g++ protobuf-compiler && rm -rf /var/lib/apt/lists/* -RUN cargo build -p tidal-server --release +# g++ builds the usearch C++ HNSW core; protobuf-compiler (protoc) is needed by +# tidal-net's build script (tonic-build compiles the WAL-shipping .proto), which +# tidal-server now depends on for real gRPC cluster replication. +RUN apt-get update && apt-get install -y --no-install-recommends g++ protobuf-compiler && rm -rf /var/lib/apt/lists/* +RUN cargo build -p tidal-server --release --locked FROM debian:bookworm-slim @@ -36,19 +43,31 @@ ARG DEBIAN_FRONTEND=noninteractive WORKDIR /srv RUN useradd --system --home /srv tidal && \ - apt-get update && apt-get install -y ca-certificates curl && \ + apt-get update && apt-get install -y --no-install-recommends ca-certificates curl && \ rm -rf /var/lib/apt/lists/* COPY --from=builder /app/target/release/tidal-server /usr/local/bin/tidal-server -COPY --chown=tidal tidal-server/config /etc/tidal-server +COPY --chown=tidal:tidal tidal-server/config /etc/tidal-server USER tidal EXPOSE 9500 +# Path the server reads its schema/topology from. The binary-side clap wiring +# for this lives in tidal-server (see sibling task T5); keep the name and +# default in sync with it here. ENV TIDAL_CONFIG=/etc/tidal-server +# Cluster mode is gated as experimental: replication is real gRPC but all +# regions share one process (no host/process isolation), so it refuses to start +# without an explicit opt-in. The image opts in here so `docker run` works; the +# server still logs a loud WARN that this is not production HA. +ENV TIDAL_ALLOW_EXPERIMENTAL_CLUSTER=1 + HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \ CMD curl -f http://localhost:9500/health || exit 1 +# ENTRYPOINT is just the binary so `docker run tidaldb:cluster ` +# overrides work. CMD carries the default cluster invocation against the baked +# schema + topology. ENTRYPOINT ["tidal-server"] CMD ["cluster", "--listen", "0.0.0.0:9500", "--schema", "/etc/tidal-server/default-schema.yaml", "--topology", "/etc/tidal-server/default-cluster.yaml"] diff --git a/docker/deploy/Dockerfile b/docker/deploy/Dockerfile index ad1edf8..e0113d6 100644 --- a/docker/deploy/Dockerfile +++ b/docker/deploy/Dockerfile @@ -1,38 +1,64 @@ +# Production single-node tidalDB deploy image (slim toolchain, durable /data). +# +# Build context is this repository's root (the workspace `Cargo.toml` / +# `Cargo.lock` and every member crate live there). Build from the repo root: +# +# docker build -f docker/deploy/Dockerfile -t tidaldb:deploy . +# +# The repo-root allowlist `.dockerignore` strips target/, node_modules/, .git, +# and .env* from the context, so `COPY . .` brings in exactly the workspace +# sources cargo needs to resolve the member graph and build `-p tidal-server`. # Pin the builder to bookworm so its glibc matches the bookworm-slim runtime. -# The default slim tag tracks Debian trixie, whose vectorized-math `libmvec.so.1` -# is absent on bookworm and aborts the binary at startup. +# The default slim tag tracks Debian trixie, whose vectorized-math +# `libmvec.so.1` is absent on bookworm and aborts the binary at startup. FROM rust:1.91-slim-bookworm AS builder +ARG DEBIAN_FRONTEND=noninteractive WORKDIR /build # protobuf-compiler (protoc) is required by tidal-net's build script (tonic-build # compiles the WAL-shipping .proto); tidal-server depends on tidal-net for gRPC -# cluster replication. g++ builds the usearch C++ HNSW core. -RUN apt-get update && apt-get install -y pkg-config libssl-dev g++ protobuf-compiler && rm -rf /var/lib/apt/lists/* +# cluster replication. +RUN apt-get update && apt-get install -y --no-install-recommends pkg-config libssl-dev g++ protobuf-compiler && rm -rf /var/lib/apt/lists/* -# Copy workspace manifests first for layer caching. -COPY Cargo.toml Cargo.lock ./ -COPY tidal/Cargo.toml tidal/Cargo.toml -COPY tidalctl/Cargo.toml tidalctl/Cargo.toml -COPY tidal-server/Cargo.toml tidal-server/Cargo.toml -COPY applications/forage/engine/Cargo.toml applications/forage/engine/Cargo.toml -COPY applications/forage/server/Cargo.toml applications/forage/server/Cargo.toml -COPY applications/forage/embedder/Cargo.toml applications/forage/embedder/Cargo.toml -COPY applications/iknowyou/engine/Cargo.toml applications/iknowyou/engine/Cargo.toml - -# Copy full workspace and build. +# Copy the full workspace and build only the server binary. The unified +# workspace requires every member manifest present to resolve metadata, so we +# copy the whole (already-pruned) context rather than a fragile manifest-only +# subset. COPY . . -RUN cargo build -p tidal-server --release +RUN cargo build -p tidal-server --release --locked FROM debian:bookworm-slim +ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl && rm -rf /var/lib/apt/lists/* -RUN useradd -m -u 10001 tidal +RUN useradd --system -u 10001 tidal + COPY --from=builder --chown=tidal:tidal /build/target/release/tidal-server /usr/local/bin/tidal-server -RUN mkdir -p /config && chown tidal:tidal /config +COPY --chown=tidal:tidal tidal-server/config /config + +# Persistent data lives under /data, owned by the runtime user. Without this +# directory + the `--data-dir /data` flag below the server boots EPHEMERAL and +# loses every write on restart. +RUN mkdir -p /data && chown tidal:tidal /data +VOLUME ["/data"] + USER tidal:tidal WORKDIR /data -EXPOSE 9500 -HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ - CMD curl -sf http://localhost:${PORT:-9500}/health || exit 1 +EXPOSE 9400 9091 + ENV TIDAL_SERVER_LOG=info -ENV PORT=9500 -ENTRYPOINT ["/usr/local/bin/tidal-server"] -CMD ["standalone", "--schema", "/config/schema.yaml", "--data-dir", "/data"] +# Path the server reads its schema/topology from. The binary-side clap wiring +# for this lives in tidal-server (see sibling task T5); keep the name and +# default in sync with it here. +ENV TIDAL_CONFIG=/config + +HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ + CMD curl -sf http://localhost:9400/health || exit 1 + +# ENTRYPOINT is just the binary so `docker run tidaldb:deploy ` +# overrides work. CMD carries the durable standalone invocation against the +# baked schema, persisting to the /data volume. +ENTRYPOINT ["tidal-server"] +CMD ["standalone", \ + "--listen", "0.0.0.0:9400", \ + "--schema", "/config/default-schema.yaml", \ + "--data-dir", "/data", \ + "--metrics", "0.0.0.0:9091"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c3d4f38..d460d9f 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -10,7 +10,10 @@ services: - TIDAL_API_KEY=${TIDAL_API_KEY} - TIDAL_SERVER_LOG=info volumes: - - tidaldb-data:/var/lib/tidaldb + # Matches the standalone image's `VOLUME ["/data"]` + `--data-dir /data`. + # Mounting anywhere else leaves the data dir on the ephemeral container + # layer and loses every write on restart. + - tidaldb-data:/data restart: unless-stopped healthcheck: test: ["CMD", "curl", "-f", "-H", "Authorization: Bearer ${TIDAL_API_KEY}", "http://localhost:9400/health"] diff --git a/docker/standalone/Dockerfile b/docker/standalone/Dockerfile index 571e400..78d498e 100644 --- a/docker/standalone/Dockerfile +++ b/docker/standalone/Dockerfile @@ -1,35 +1,54 @@ -# Pin the builder to bookworm so its glibc matches the bookworm-slim runtime; -# the default tag tracks trixie, whose `libmvec.so.1` is absent on bookworm. +# Single-node tidalDB server image. +# +# Build context is this repository's root (the workspace `Cargo.toml` / +# `Cargo.lock` and every member crate live there). Build from the repo root: +# +# docker build -f docker/standalone/Dockerfile -t tidaldb:standalone . +# +# tidal-server is a workspace member at `tidal-server/` and depends on the +# `tidal/` engine crate plus `tidal-net/` (gRPC cluster transport). The repo-root +# allowlist `.dockerignore` strips target/, node_modules/, .git, and .env* from +# the context, so `COPY . .` brings in exactly the workspace sources cargo needs +# to resolve the member graph and build `-p tidal-server`. +# Pin the builder to bookworm so its glibc matches the bookworm-slim runtime. +# The default `rust:1.91` tracks Debian trixie, whose vectorized-math +# `libmvec.so.1` is absent on bookworm and aborts the binary at startup. FROM rust:1.91-bookworm AS builder +ARG DEBIAN_FRONTEND=noninteractive WORKDIR /app -# protobuf-compiler (protoc) is required by tidal-net's build script (tonic-build -# compiles the WAL-shipping .proto); tidal-server depends on tidal-net for gRPC -# cluster replication. The full rust image already ships g++/pkg-config/libssl. -RUN apt-get update && apt-get install -y --no-install-recommends protobuf-compiler && rm -rf /var/lib/apt/lists/* +# g++ builds the usearch C++ HNSW core; protobuf-compiler (protoc) is needed by +# tidal-net's build script (tonic-build compiles the WAL-shipping .proto). +RUN apt-get update && apt-get install -y --no-install-recommends g++ protobuf-compiler && rm -rf /var/lib/apt/lists/* -# Copy workspace manifests first for layer caching. -COPY Cargo.toml Cargo.lock ./ -COPY tidal/Cargo.toml tidal/Cargo.toml -COPY tidalctl/Cargo.toml tidalctl/Cargo.toml -COPY tidal-server/Cargo.toml tidal-server/Cargo.toml -COPY applications/forage/engine/Cargo.toml applications/forage/engine/Cargo.toml -COPY applications/forage/server/Cargo.toml applications/forage/server/Cargo.toml -COPY applications/forage/embedder/Cargo.toml applications/forage/embedder/Cargo.toml -COPY applications/iknowyou/engine/Cargo.toml applications/iknowyou/engine/Cargo.toml - -# Copy full workspace and build. +# Copy the full workspace and build only the server binary. The unified +# workspace requires every member manifest present to resolve metadata, so we +# copy the whole (already-pruned) context rather than a fragile manifest-only +# subset. COPY . . -RUN cargo build -p tidal-server --release +RUN cargo build -p tidal-server --release --locked FROM debian:bookworm-slim +ARG DEBIAN_FRONTEND=noninteractive WORKDIR /srv + RUN useradd --system --home /srv tidal && \ - apt-get update && apt-get install -y ca-certificates curl && \ + apt-get update && apt-get install -y --no-install-recommends ca-certificates curl && \ rm -rf /var/lib/apt/lists/* COPY --from=builder /app/target/release/tidal-server /usr/local/bin/tidal-server -COPY tidal-server/config /etc/tidal-server +COPY --chown=tidal:tidal tidal-server/config /etc/tidal-server + +# Persistent data lives under /data, owned by the runtime user. Without this +# directory + the `--data-dir /data` flag below the server boots EPHEMERAL and +# loses every write on restart. +RUN mkdir -p /data && chown tidal:tidal /data +VOLUME ["/data"] + +# Path the server reads its schema/topology from. The binary-side clap wiring +# for this lives in tidal-server (see sibling task T5); keep the name and +# default in sync with it here. +ENV TIDAL_CONFIG=/etc/tidal-server USER tidal EXPOSE 9400 9091 @@ -37,6 +56,11 @@ EXPOSE 9400 9091 HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \ CMD curl -f -H "Authorization: Bearer ${TIDAL_API_KEY:-}" http://localhost:9400/health || exit 1 -ENTRYPOINT ["tidal-server", "standalone", \ +# ENTRYPOINT is just the binary so `docker run tidaldb:standalone ` +# overrides work (e.g. `... cluster ...` or an ad-hoc inspect). CMD carries the +# default standalone invocation, including `--data-dir /data` for durability. +ENTRYPOINT ["tidal-server"] +CMD ["standalone", \ "--listen", "0.0.0.0:9400", \ + "--data-dir", "/data", \ "--metrics", "0.0.0.0:9091"] diff --git a/docs/README.md b/docs/README.md index 082218a..921214e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -34,10 +34,19 @@ The authoritative component specifications (status: Implemented, M0–M8). - [M0–M10 code review — 2026-06-08](reviews/M0-M10-code-review-2026-06-08.md) — seven-dimension review, 142 findings (latest pass) - [M0–M10 seven-dimension review](reviews/M0-M10-seven-dimension-review.md) — additional pass (2 BLOCKERs: signal-checkpoint trim, 30-day window) +## Guides — `guides/` + +Task-oriented, build-an-app docs (complements the root [QUICKSTART.md](../QUICKSTART.md) and [API.md](../API.md)): + +- [**Build a feed app**](guides/build-a-feed-app.md) — end-to-end TikTok/Reels-style "For You" feed, embedded and over HTTP +- [**Embedding integration**](guides/embeddings.md) — wiring a real embedding model (OpenAI / Cohere / local) into the write + query paths +- [**Server deployment**](guides/server-deployment.md) — running the `tidal-server` HTTP service: config, auth, the served OpenAPI spec, Docker +- Ranking-profile reference: [ai-lookup/services/ranking-profiles.md](../ai-lookup/services/ranking-profiles.md) — all 25 built-in profiles + ## Operations — `ops/` and `runbooks/` - [Monitoring](ops/monitoring.md) · [Prometheus alerts](ops/prometheus-alerts.yaml) · [Grafana dashboard](ops/grafana-dashboard.json) · [Capacity planning](ops/capacity-planning.md) · [Recovery](ops/recovery.md) -- Runbooks: [Cluster](runbooks/cluster.md) +- Runbooks: [Kubernetes](runbooks/kubernetes.md) · [Cluster (experimental)](runbooks/cluster.md) ## Research — `research/` diff --git a/docs/guides/build-a-feed-app.md b/docs/guides/build-a-feed-app.md new file mode 100644 index 0000000..778bca3 --- /dev/null +++ b/docs/guides/build-a-feed-app.md @@ -0,0 +1,620 @@ +# Build a Short-Video "For You" Feed on tidalDB + +This is an end-to-end tutorial for building a TikTok/Reels-style short-video feed — +a swipeable home feed ("For You"), a Following feed, and a Discover/Trending tab — +on top of tidalDB. You will model the signals a short-video app actually emits, +ingest videos, wire the swipe feedback loop so a user's taste updates *immediately*, +and serve all three feed surfaces. You will do it twice: once embedded in a Rust +process, and once over HTTP with `curl`. + +There is a verified, runnable version of the embedded path in this repo: + +```bash +cargo run -p tidaldb --example foryou_feed +``` + +That example ([`tidal/examples/foryou_feed.rs`](../../tidal/examples/foryou_feed.rs)) +is the source of truth for every Rust call shown below — this guide reproduces and +explains its key calls. Skim it after reading section 6. + +**New to tidalDB?** Read [QUICKSTART.md](../../QUICKSTART.md) first for the 5-minute +schema → ingest → rank loop. This guide assumes you have run the quickstart and +understand schemas, signals, and the `RETRIEVE` query. The product framing for this +surface is [USE_CASES.md → UC-01 Personalized Feed (For You)](../../USE_CASES.md#uc-01--personalized-feed--for-you). + +--- + +## 1. What tidalDB owns vs. what you build + +tidalDB answers exactly one question: **given a user and a context, what content +should they see, in what order?** It owns *retrieval and ranking* — and the +feedback loop that learns taste from engagement. It owns nothing else. + +Everything that produces, stores, or serves the actual video bytes is yours. + +| Concern | Who owns it | +|---|---| +| Video upload, blob storage, object store | **You** | +| CDN, transcoding, HLS/DASH packaging, thumbnails | **You** | +| Embedding generation (run the video through a multimodal model) | **You** — see [docs/guides/embeddings.md](./embeddings.md) | +| Authentication, identity, sessions | **You** | +| Moderation, trust & safety, takedowns | **You** | +| Payments, creator payouts | **You** | +| Player UI, swipe gestures, autoplay | **You** | +| **Candidate retrieval (ANN + filters)** | **tidalDB** | +| **Signal ledgers: decay, velocity, windowed counts** | **tidalDB** | +| **Per-user preference (taste) vectors** | **tidalDB** | +| **Ranking profiles (`for_you`, `following`, `trending`, …)** | **tidalDB** | +| **Diversity enforcement + exploration** | **tidalDB** | +| **The swipe feedback loop** (engagement → updated taste, no Kafka lag) | **tidalDB** | + +### Architecture sketch + +``` + ┌──────────────┐ upload ┌─────────────────────────────────────┐ + │ Creator app │────────────▶│ YOUR services │ + └──────────────┘ │ • blob store + CDN + transcode │ + │ • multimodal model → embedding │ + │ • auth / identity / moderation │ + └───────────────┬─────────────────────┘ + │ + write_item_with_metadata(id, meta) + write_item_embedding(id, vector) ← you bring the vector + │ + ▼ + ┌──────────────┐ ┌───────────────────────────────┐ + │ Viewer app │ GET feed │ tidalDB │ + │ (player UI) │◀──── retrieve ────│ HNSW + signal ledgers + │ + │ │ │ preference vectors + │ + │ swipe ─────┼──── signals ─────▶│ ranking profiles + diversity │ + │ (complete, │ signal_with_ │ │ + │ like,skip) │ context(…) │ one process, one query API │ + └──────────────┘ └───────────────────────────────┘ +``` + +The viewer app does two things against tidalDB: it **reads** a ranked feed, and as +the user swipes it **writes** signals back. The write updates that user's taste in +the same process that serves the next read — there is no Kafka topic, no feature +store, and no batch job in between (section 4). + +> tidalDB does **not** generate embeddings. Your model turns the video +> (frames + audio + caption) into a vector; tidalDB L2-normalizes it, inserts it +> into the HNSW index, and ranks over it. See [docs/guides/embeddings.md](./embeddings.md). + +--- + +## 2. Model the signals for short video + +A signal is a typed, timestamped event stream with native decay, velocity, and +windowed aggregation. You declare each signal once in the schema; the engine then +maintains its running decay score, per-window counts, and (optionally) velocity for +free. For short video, six signals capture the surface: + +| Signal | Decay half-life | `positive_engagement` | Why | +|---|---|---|---| +| `view` | 7 days | no | The raw impression. Low-intent — a glance, not a preference. Drives trending velocity, not taste. | +| `completion` | 1 year (barely decays) | **yes** | Finishing a short video is the strongest, most durable taste signal there is. Weight = fraction watched ∈ [0,1]. | +| `like` | 30 days | **yes** | An explicit, durable positive. | +| `share` | 3 days (bursty) | no | Tracks `velocity` — shares spike, then fade; great for trending, not a private taste signal. | +| `skip` | 1 day | **no** | The swipe-away negative. Must **never** pull taste toward the skipped video, and marks it as a hard negative for that user. | +| `replay` | 14 days | **yes** | Re-watching is a meaningful, deliberate positive. | + +### The `positive_engagement(true)` rule — read this twice + +This is the single most important correctness rule for a personalized feed: + +> **Preference-vector personalization fires only for signals declared +> `.positive_engagement(true)`.** When you record an engagement through +> `signal_with_context`, tidalDB folds the *item's content embedding* into the +> *acting user's* preference (taste) vector — but only if that signal type is a +> declared positive-engagement signal. + +Why each classification matters: + +- **`completion`, `like`, `replay` are `positive_engagement(true)`** → engaging + with a video nudges the user's taste vector toward that video's content. This is + the learning step. Get it right and the feed converges on what the user loves. +- **`view` is NOT positive-engagement** → a view is just "the video was shown." + If views folded into taste, a user's vector would be dragged toward everything + the feed happened to surface — a feedback bubble that learns nothing. +- **`skip` is NOT positive-engagement** → this is the load-bearing negative. A + swipe-away must never pull taste *toward* the rejected content. Routed through + `signal_with_context`, `skip` instead marks the video as a **hard negative** and + **seen** for that user, so the feed stops surfacing it. + +If you accidentally mark `skip` or `view` as positive-engagement, the feed slowly +poisons every user's taste vector. Declare them honestly. + +> **Subtle but important — the rule is all-or-nothing per database.** If *any* +> signal in your schema declares `positive_engagement(true)`, then **only** the +> signals you explicitly flagged fold the preference vector; nothing else does. +> If *no* signal declares it, tidalDB falls back to a built-in name allowlist +> (`like`, `share`, `completion`, `search_click`). The embedded schema below +> declares the flags explicitly, so it is fully in control. This matters for the +> HTTP path — see the caveat in section 9. + +### Schema (embedded Rust) + +This is lifted directly from +[`tidal/examples/foryou_feed.rs`](../../tidal/examples/foryou_feed.rs): + +```rust +use std::time::Duration; +use tidaldb::schema::{DecaySpec, EntityKind, SchemaBuilder, Window}; + +let mut schema = SchemaBuilder::new(); + +// view: 7-day half-life, velocity + windows. Low-intent — NOT positive engagement. +let _ = schema + .signal("view", EntityKind::Item, + DecaySpec::Exponential { half_life: Duration::from_secs(7 * 24 * 3600) }) + .windows(&[Window::OneHour, Window::TwentyFourHours, Window::AllTime]) + .velocity(true) + .add(); + +// completion: ~1-year half-life. Weight = fraction watched. POSITIVE engagement. +let _ = schema + .signal("completion", EntityKind::Item, + DecaySpec::Exponential { half_life: Duration::from_secs(365 * 24 * 3600) }) + .windows(&[Window::AllTime]) + .positive_engagement(true) + .add(); + +// like: 30-day half-life. POSITIVE engagement. +let _ = schema + .signal("like", EntityKind::Item, + DecaySpec::Exponential { half_life: Duration::from_secs(30 * 24 * 3600) }) + .windows(&[Window::AllTime]) + .positive_engagement(true) + .add(); + +// share: 3-day half-life, bursty → velocity. NOT positive engagement. +let _ = schema + .signal("share", EntityKind::Item, + DecaySpec::Exponential { half_life: Duration::from_secs(3 * 24 * 3600) }) + .windows(&[Window::TwentyFourHours, Window::AllTime]) + .velocity(true) + .add(); + +// skip: fast 1-day decay — the swipe-away negative. NOT positive engagement. +let _ = schema + .signal("skip", EntityKind::Item, + DecaySpec::Exponential { half_life: Duration::from_secs(24 * 3600) }) + .windows(&[Window::OneHour, Window::TwentyFourHours]) + .add(); + +// replay: 14-day half-life. POSITIVE engagement. +let _ = schema + .signal("replay", EntityKind::Item, + DecaySpec::Exponential { half_life: Duration::from_secs(14 * 24 * 3600) }) + .windows(&[Window::AllTime]) + .positive_engagement(true) + .add(); + +// Content embedding slot for items: 128 dimensions (your model's output size). +let _ = schema.embedding_slot("content", EntityKind::Item, 128); + +let schema = schema.build()?; +``` + +`DecaySpec` has three variants: `Exponential { half_life }`, `Linear { lifetime }`, +and `Permanent`. Available windows: `OneHour`, `TwentyFourHours`, `SevenDays`, +`ThirtyDays`, `AllTime`. Only declare the windows a profile actually reads — +each one is real bookkeeping. + +--- + +## 3. Ingest videos (metadata + embeddings) + +After your services have uploaded, transcoded, and embedded a video, write two +records into tidalDB: the metadata and the embedding vector. + +```rust +use std::collections::HashMap; +use tidaldb::{TidalDb, schema::{EntityId, Timestamp}}; + +let db = TidalDb::builder().ephemeral().with_schema(schema).open()?; +// For durability: .with_data_dir("/var/lib/tidaldb") instead of .ephemeral() + +let now = Timestamp::now(); + +let mut meta = HashMap::new(); +meta.insert("title".to_string(), "Lo-fi piano loop".to_string()); +meta.insert("category".to_string(), "music".to_string()); +meta.insert("format".to_string(), "short".to_string()); +meta.insert("creator_id".to_string(), "42".to_string()); +meta.insert("duration".to_string(), "27".to_string()); // seconds +meta.insert("created_at".to_string(), now.as_nanos().to_string()); + +db.write_item_with_metadata(EntityId::new(1001), &meta)?; + +// You bring the vector. This MUST be your model's output for THIS video. +let embedding: Vec = your_model.embed(&video); // length == slot dimensions (128) +db.write_item_embedding(EntityId::new(1001), &embedding)?; +``` + +Indexed metadata keys the engine understands: `title`, `category`, `format`, +`creator_id`, `tags`, `duration`, `created_at`. `creator_id` is what the +`max_per_creator` diversity cap (section 5) keys on, so always set it. + +**Embedding rules (strict — read [docs/guides/embeddings.md](./embeddings.md) for +real vectors):** + +- `write_item_embedding` L2-normalizes the vector and inserts it into the HNSW index. +- Dimensions are strict: **min 2, max 4096**, and the length **must equal the slot's + declared dimensions** (128 here) or the insert fails. +- **Zero-norm vectors are rejected.** +- **Multi-slot caveat:** `RETRIEVE`/`SEARCH` route through the **first declared slot + only**. A multi-modal app (separate text and image vectors) must fuse them offline + into one vector per slot, or model the modalities as separate entity kinds. + +The example seeds 128-D *random* vectors purely so it runs standalone. A real app +must use genuine embeddings — random vectors give you random nearest-neighbors. + +--- + +## 4. The swipe feedback loop (no Kafka, no feature-store lag) + +This is what makes tidalDB a *feed* engine and not just a vector store. As the user +swipes, you record each engagement through `signal_with_context`, passing the user +id and the video's creator id. tidalDB writes the signal, updates windowed counts +and velocity, marks seen/hard-negative state, and — for positive-engagement signals — +folds the video's embedding into *that user's* preference vector. The next +`retrieve` call in the same process sees the updated taste. No queue, no batch job, +no feature store to keep in sync. + +```rust +let user_id: u64 = 1001; +let creator_id: u64 = 42; +let video = EntityId::new(2001); +let now = Timestamp::now(); + +// User watched the whole thing → completion weight is the fraction watched (1.0). +// completion is positive-engagement → folds the video's embedding into the user's taste. +db.signal_with_context("completion", video, 1.0, now, Some(user_id), Some(creator_id))?; + +// User tapped like → another positive fold, plus strengthens the (user, creator) edge. +db.signal_with_context("like", video, 1.0, now, Some(user_id), Some(creator_id))?; + +// Different video — user swiped away after a glance. Partial view, then skip. +let rejected = EntityId::new(2002); +db.signal_with_context("view", rejected, 0.1, now, Some(user_id), Some(creator_id))?; +// skip is NOT positive-engagement: it does NOT pull taste toward `rejected`. +// It marks `rejected` as a hard negative + seen for this user, so the feed drops it. +db.signal_with_context("skip", rejected, 1.0, now, Some(user_id), Some(creator_id))?; +``` + +Contrast the two write APIs: + +- `db.signal(type, id, weight, ts)` — **global** signal, no user context. Use it for + aggregate counters that feed `trending` (e.g. a server-side view counter). It does + **not** touch any preference vector or seen-state. +- `db.signal_with_context(type, id, weight, ts, Some(user_id), Some(creator_id))` — + **personalized**. Updates the user's preference vector (positive-engagement only), + seen-state, and hard negatives (skip/hide/block). This is the swipe-loop API. + +A `completion` weight is the **fraction watched** in `[0.0, 1.0]` — 0.3 for a +three-second glance at a ten-second clip, 1.0 for a full watch (or a watch past the +end, if you clamp). The decay score blends these naturally: many partial completions +sum to less than a few full ones. + +You can verify the loop landed by reading the live decay score: + +```rust +let score = db.read_decay_score(EntityId::new(2001), "completion", 0)?; // window index 0 +``` + +--- + +## 5. Serve the three feed surfaces + +tidalDB ships **25 built-in ranking profiles**, registered automatically on +`open()`. A short-video app needs three of them. None require any extra schema — you +just name the profile in the query. + +### For You — `profile("for_you")` + +The personalized home feed. It blends interaction-weighted decay scores +(`view·1 + like·2 + share-velocity·1.5`) with the user's preference vector, sorts +with a Hot decay (`gravity = 1.5`), then enforces diversity and exploration. The +built-in defaults are: + +- `max_per_creator = 2` — no single creator can dominate the feed. +- `format_mix_max_fraction = 0.4` — no one format swamps the mix. +- `exploration = 0.1` — **10% of slots are exploration**: unseen/random items + injected to break the filter bubble and let *new creators surface*. Without this, + a feed converges to a narrow loop and new content never gets a chance. + +Seen items and hard negatives (the videos this user already watched or skipped) are +filtered out automatically — a feed must surface fresh content. + +```rust +use tidaldb::query::retrieve::Retrieve; + +let for_you = Retrieve::builder() + .for_user(user_id) // personalization + seen/hard-negative filtering + .profile("for_you") + .limit(10) + .build()?; + +let results = db.retrieve(&for_you)?; +for item in &results.items { + println!("#{} video={} score={:.4}", item.rank, item.entity_id.as_u64(), item.score); +} +``` + +Want to override the defaults for one query? Use `.diversity(...)`: + +```rust +use tidaldb::ranking::diversity::DiversityConstraints; + +let q = Retrieve::builder() + .for_user(user_id) + .profile("for_you") + .diversity(DiversityConstraints { + max_per_creator: Some(1), // stricter: one video per creator + format_mix_max_fraction: Some(0.5), + ..DiversityConstraints::new() + }) + .limit(20) + .build()?; +``` + +### Following — `profile("following")` + +Content from creators the user follows, sourced from the relationship graph and +ranked by recency with a view-velocity boost (`max_per_creator = 3`). This requires +that you have recorded follow relationships (a follows edge) for the user — see +[USE_CASES.md](../../USE_CASES.md) for the relationship model. + +```rust +let following = Retrieve::builder() + .for_user(user_id) + .profile("following") + .limit(20) + .build()?; +let results = db.retrieve(&following)?; +``` + +### Discover / Trending — `profile("trending")` + +The global, *non-personalized* tab — what a brand-new visitor with no taste history +sees. It ranks every item by velocity (`view-velocity + 2·share-velocity` over the +24h window), capped at `max_per_creator = 1` for maximum variety. Note: velocity +needs signals arriving over real elapsed time to populate hour-level buckets, so a +freshly-seeded demo will show sparse trending scores — that is expected. + +```rust +let trending = Retrieve::builder() // no .for_user — trending is global + .profile("trending") + .limit(20) + .build()?; +let results = db.retrieve(&trending)?; +``` + +> The payoff to look for in [`foryou_feed.rs`](../../tidal/examples/foryou_feed.rs): +> after a user completes + likes music videos and skips everything else, the top of +> their `for_you` feed is the *fresh* music videos — surfaced because the swipe +> session built a strong taste signal, **not** because of the (random) embeddings. +> The ordering is driven by signals, which is why it is stable across runs. + +--- + +## 6. The whole thing, embedded (Rust) + +The complete, verified program is at +[`tidal/examples/foryou_feed.rs`](../../tidal/examples/foryou_feed.rs). Run it: + +```bash +cargo run -p tidaldb --example foryou_feed +``` + +It walks the full arc end to end: + +1. Builds the six-signal short-video schema from section 2. +2. Opens an ephemeral DB and ingests ~24 videos across 5 creators. +3. Simulates one user's swipe session — completions + likes + a share + a replay on + music videos, skips on the rest — all via `signal_with_context`. +4. Retrieves `for_you` (diversity-capped, 10% exploration) and prints the ranked feed. +5. Retrieves global `trending` for contrast. + +`Arc` is `Send + Sync`, so the same handle backs every request thread in a +real server: clone the `Arc` into your axum/actix handlers, write signals on the +swipe endpoint, and read feeds on the feed endpoint. See the embedding-integration +examples (`axum_embedding`, `actix_embedding`, `cli_embedding`) in `tidal/examples/` +for the web-handler wiring. + +A `Results` value contains `items: Vec`; each `FeedItem` exposes `.rank`, +`.entity_id.as_u64()`, `.score`, and `.signals` (per-item signal snapshots for +explainability), plus `results.total_candidates` for the pre-diversity pool size. + +For text + vector hybrid search (a search box rather than a feed), use the parallel +`Search::builder().query("...").vector(vec).for_user(uid).limit(n).build()?` then +`db.search(&q)?` — covered in [QUICKSTART.md](../../QUICKSTART.md) and +[API.md](../../API.md). + +--- + +## 7. The whole thing, over HTTP + +The same database behind a JSON HTTP API is `tidal-server`. Start a standalone +node with your schema: + +```bash +cargo run -p tidal-server -- standalone \ + --listen 0.0.0.0:9400 \ + --schema ./short-video-schema.yaml \ + --data-dir /var/lib/tidaldb \ + --metrics 127.0.0.1:9091 +``` + +Set `TIDAL_API_KEY` to require Bearer auth on the data routes. **If `TIDAL_API_KEY` +is unset, the server runs fully unauthenticated and logs a WARN** — never do that +outside a private dev box. + +```bash +export TIDAL_API_KEY="$(openssl rand -hex 32)" +``` + +### Schema YAML + +The HTTP server loads its schema from YAML. The short-video schema looks like: + +```yaml +signals: + - name: view + entity: item + decay: { exponential: { half_life_seconds: 604800 } } # 7 days + windows: [one_hour, twenty_four_hours, all_time] + velocity: true + - name: completion + entity: item + decay: { exponential: { half_life_seconds: 31536000 } } # 1 year + windows: [all_time] + positive_engagement: true + - name: like + entity: item + decay: { exponential: { half_life_seconds: 2592000 } } # 30 days + windows: [all_time] + positive_engagement: true + - name: share + entity: item + decay: { exponential: { half_life_seconds: 259200 } } # 3 days + windows: [twenty_four_hours, all_time] + velocity: true + - name: skip + entity: item + decay: { exponential: { half_life_seconds: 86400 } } # 1 day + windows: [one_hour, twenty_four_hours] + - name: replay + entity: item + decay: { exponential: { half_life_seconds: 1209600 } } # 14 days + windows: [all_time] + positive_engagement: true +text_fields: + - { name: title, kind: text } + - { name: category, kind: keyword } +embedding_slots: + - { name: content, entity: item, dimensions: 128 } +``` + +> **`positive_engagement` works in YAML too.** Declaring it per signal (as +> above) gives the HTTP server the same explicit control as the embedded +> `SchemaBuilder` — `completion`, `like`, and `replay` fold into the user's +> preference vector; `view` and `skip` do not. If you omit the field on *every* +> signal, tidalDB falls back to a built-in name allowlist (`like`, `share`, +> `completion`, `search_click`); declaring it explicitly (recommended) removes +> any ambiguity about which signals shape taste. + +The 25 built-in profiles (`for_you`, `following`, `trending`, …) are available on +the HTTP server with no `profiles:` block — they are registered automatically. Add a +`profiles:` block only to override a built-in or define your own. + +### Routes + +All routes are JSON. Data routes require `Authorization: Bearer $TIDAL_API_KEY` when +the key is set; `GET /health*` and `GET /openapi.json` are always unauthenticated. + +```bash +# Ingest a video's metadata → 201 Created +curl -X POST http://localhost:9400/items \ + -H "Authorization: Bearer $TIDAL_API_KEY" \ + -H 'Content-Type: application/json' \ + -d '{ "entity_id": 1001, + "metadata": { "title": "Lo-fi piano loop", "category": "music", + "format": "short", "creator_id": "42", "duration": "27" } }' + +# Ingest its embedding (length MUST equal slot dimensions) → 204 No Content +curl -X POST http://localhost:9400/embeddings \ + -H "Authorization: Bearer $TIDAL_API_KEY" \ + -H 'Content-Type: application/json' \ + -d '{ "entity_id": 1001, "values": [0.013, -0.041, /* … 128 floats … */ 0.092] }' + +# The swipe loop: record engagement with user + creator context → 204 No Content +curl -X POST http://localhost:9400/signals \ + -H "Authorization: Bearer $TIDAL_API_KEY" \ + -H 'Content-Type: application/json' \ + -d '{ "entity_id": 1001, "signal": "completion", "weight": 1.0, + "user_id": 1001, "creator_id": 42 }' + +curl -X POST http://localhost:9400/signals \ + -H "Authorization: Bearer $TIDAL_API_KEY" \ + -H 'Content-Type: application/json' \ + -d '{ "entity_id": 1001, "signal": "like", "weight": 1.0, + "user_id": 1001, "creator_id": 42 }' + +# The swipe-away negative on a different video +curl -X POST http://localhost:9400/signals \ + -H "Authorization: Bearer $TIDAL_API_KEY" \ + -H 'Content-Type: application/json' \ + -d '{ "entity_id": 1002, "signal": "skip", "weight": 1.0, + "user_id": 1001, "creator_id": 7 }' + +# Serve the For You feed +curl -H "Authorization: Bearer $TIDAL_API_KEY" \ + "http://localhost:9400/feed?user_id=1001&profile=for_you&limit=20" + +# Following feed and the Discover/Trending tab +curl -H "Authorization: Bearer $TIDAL_API_KEY" \ + "http://localhost:9400/feed?user_id=1001&profile=following&limit=20" +curl -H "Authorization: Bearer $TIDAL_API_KEY" \ + "http://localhost:9400/feed?profile=trending&limit=20" # no user_id — global +``` + +When `?user_id=` and `?profile=` are omitted, `/feed` defaults to `profile=for_you`. +Passing `user_id` and `creator_id` on `/signals` is what activates the personalized +loop — omit them and the signal is recorded globally (the `signal` vs. +`signal_with_context` distinction from section 4). + +Middleware on the data routes: 30s timeout (`408`), 100 max in-flight (`429`), 2 MB +body limit (`413`), and an `x-request-id` echoed on every response. The canonical, +always-current HTTP reference is the served spec at `GET /openapi.json` (unauthenticated). + +--- + +## 8. Going to production — checklist + +- [ ] **Durable storage.** Open with `.with_data_dir(...)` (embedded) or + `--data-dir` (server), not ephemeral. The WAL persists signals across restarts. +- [ ] **Real embeddings.** Replace the random vectors with your multimodal model's + output. Same model for ingest and query. See + [docs/guides/embeddings.md](./embeddings.md). +- [ ] **Auth on.** Set `TIDAL_API_KEY`. An unset key means an unauthenticated server + (it logs a WARN). Terminate TLS at your ingress. +- [ ] **Lock down metrics.** The `--metrics` Prometheus endpoint is **unauthenticated** + — bind it to loopback or a cluster-internal address only, never the public + interface. +- [ ] **Graceful shutdown.** On `SIGTERM` the server flips readiness to `503`, + drains in-flight requests, then checkpoints and fsyncs the WAL. Wire your + orchestrator's `preStop`/termination grace period to allow the drain. +- [ ] **Health probes.** Use `GET /health` for readiness (`200` ok / `503` draining), + `GET /health/startup` and `GET /health/live` for startup/liveness. +- [ ] **Observability.** Scrape the metrics endpoint; import the dashboard and alerts + from [docs/ops/](../ops/) (`grafana-dashboard.json`, `prometheus-alerts.yaml`, + `monitoring.md`). Plan headroom with [docs/ops/capacity-planning.md](../ops/capacity-planning.md). +- [ ] **Backup & recovery.** Rehearse restore with [docs/ops/recovery.md](../ops/recovery.md). +- [ ] **Deployment.** Follow [docs/guides/server-deployment.md](./server-deployment.md) + for packaging and rollout, and the Kubernetes runbook in + [docs/runbooks/](../runbooks/) for cluster orchestration. The container images + live at `docker/` (`standalone` / `cluster` / `deploy`) — build from the repo + root so `COPY . .` sees the whole workspace. + +> **On cluster mode — be honest with yourself.** tidalDB's cluster mode is +> **experimental** and currently runs **all regions in a single process** over +> loopback gRPC. It is *not* production multi-process HA (true multi-process HA is +> tracked as m8p10). It also replicates **global signals only** — `user_id` / +> `creator_id` contexts are rejected in cluster mode, so the personalized swipe loop +> (section 4) is a **single-node** feature today. Ship the For You feed on a +> vertically-scaled standalone node; reach for the experimental cluster only for +> read-scale experiments, per the [cluster runbook](../runbooks/cluster.md). + +--- + +## See also + +- [QUICKSTART.md](../../QUICKSTART.md) — the 5-minute ingest → rank loop. +- [USE_CASES.md → UC-01](../../USE_CASES.md#uc-01--personalized-feed--for-you) — product framing for the For You surface. +- [docs/guides/embeddings.md](./embeddings.md) — generating and writing real vectors. +- [docs/guides/server-deployment.md](./server-deployment.md) — packaging and rollout. +- [API.md](../../API.md) — full embedded + HTTP API reference. +- [`tidal/examples/foryou_feed.rs`](../../tidal/examples/foryou_feed.rs) — the verified, runnable version of this guide. diff --git a/docs/guides/embeddings.md b/docs/guides/embeddings.md new file mode 100644 index 0000000..7914d2f --- /dev/null +++ b/docs/guides/embeddings.md @@ -0,0 +1,436 @@ +# Wiring a Real Embedding Model into tidalDB + +tidalDB does **not** generate embeddings. It stores them, indexes them into HNSW, +and ranks over them. You bring the model; tidalDB owns retrieval and ranking. This +guide shows how to wire a production embedding model — OpenAI, Cohere, or a local +sentence-transformers / CLIP model — into the embedded engine and the HTTP server. + +> If you only need a runnable end-to-end demo with random vectors first, run +> `cargo run -p tidaldb --example foryou_feed` and read [QUICKSTART.md](../../QUICKSTART.md). +> This guide is the next step: replacing those random vectors with a real model. + +**See also:** [QUICKSTART.md](../../QUICKSTART.md) · [API.md — Writing Embeddings](../../API.md#writing-embeddings) · [API.md — `POST /embeddings`](../../API.md#post-embeddings) · [docs/guides/build-a-feed-app.md](build-a-feed-app.md) (companion: the full app around this). + +--- + +## 1. The contract + +tidalDB enforces a small, strict contract on every vector you write. Honoring it is +not optional — violations are rejected at the write boundary, not silently coerced. + +| Rule | What tidalDB does | Failure mode | +|------|-------------------|--------------| +| **You generate, tidalDB stores** | `write_item_embedding` / `POST /embeddings` accept a `&[f32]` you computed. No model runs inside the database. | — | +| **L2 normalization is automatic** | The vector is L2-normalized on write so that L2 distance on the HNSW index equals cosine distance. You may pass an un-normalized vector; tidalDB normalizes it. | — | +| **Dimensions are strict: `[2, 4096]`** | A slot must declare dimensions in `[2, 4096]`. Out-of-range slot declarations fail at `schema.build()`. | `InvalidEmbeddingDimensions { slot, dimensions }` — "must be in [2, 4096]" | +| **Vector length must equal the slot** | Every inserted vector's `.len()` must equal the slot's declared dimensions exactly. | `DimensionMismatch { expected, got }` at insert | +| **Zero-norm vectors are rejected** | A vector whose squared sum is below `f32::EPSILON` (all zeros / underflow) has no direction and cannot do cosine similarity. | `ZeroNormVector` | + +Three more facts that shape your integration: + +- **Normalization is automatic, but you should still hand tidalDB a clean vector.** + Most embedding APIs already return unit-norm vectors (OpenAI and Cohere do). + If yours does not, tidalDB will normalize it — but a zero vector will be rejected + outright, so guard against degenerate output (empty input text, blank frames). +- **Multi-slot routing caveat.** A schema may declare several embedding slots, but + `RETRIEVE` and `SEARCH` route ANN through the **first declared Item slot only** + (resolved by `item_embedding_slot()`, defaulting to `"content"`). Multi-modal apps + must fuse modalities offline into one vector per item, or model each modality as a + separate entity kind. See [§7 Failure modes](#7-failure-modes). +- **Personalization needs `positive_engagement(true)`.** The per-user preference + vector that folds item embeddings into a user's taste is updated **only** for + signals declared `.positive_engagement(true)` and written via `signal_with_context`. + A `view` or `skip` that is not positive-engagement does not move the taste vector. + See [§4 The query side](#4-the-query-side). + +--- + +## 2. Declare an embedding slot sized to your model + +The slot's dimensions must equal your model's output dimensionality. Pick the slot +size **before** opening the database — it is baked into the on-disk schema fingerprint, +and changing it later is a new vector space (see [§6 Versioning](#6-model-versioning)). + +### Embedded (Rust) + +```rust +use tidaldb::schema::{SchemaBuilder, EntityKind}; + +let mut s = SchemaBuilder::new(); + +// ... declare your signals first (see QUICKSTART.md) ... + +// One Item content slot, sized to text-embedding-3-small (1536D). +let _ = s.embedding_slot("content", EntityKind::Item, 1536); + +let schema = s.build()?; // fails here if dimensions are outside [2, 4096] +``` + +Common model dimensions: + +| Model | Native dims | Slot declaration | +|-------|-------------|------------------| +| OpenAI `text-embedding-3-small` | 1536 | `embedding_slot("content", EntityKind::Item, 1536)` | +| OpenAI `text-embedding-3-large` | 3072 (shortenable) | `embedding_slot("content", EntityKind::Item, 3072)` — or pass `dimensions: 1024` to the API and declare `1024` (see §3) | +| Cohere `embed-english-v3.0` / `embed-multilingual-v3.0` | 1024 | `embedding_slot("content", EntityKind::Item, 1024)` | +| `sentence-transformers/all-MiniLM-L6-v2` | 384 | `embedding_slot("content", EntityKind::Item, 384)` | +| `sentence-transformers/all-mpnet-base-v2` | 768 | `embedding_slot("content", EntityKind::Item, 768)` | +| OpenAI CLIP `ViT-B/32` (image/frame) | 512 | `embedding_slot("content", EntityKind::Item, 512)` | + +> The 3072D of `text-embedding-3-large` is within the `[2, 4096]` ceiling, so it works +> directly. Larger models do not fit — if you need them, project down to ≤4096 (most +> have a documented `dimensions`/Matryoshka truncation parameter; re-normalize after). + +### HTTP server (config YAML) + +The server loads the same schema from YAML via `--schema`: + +```yaml +embedding_slots: + - name: content + entity: item + dimensions: 1536 # must match your model and be in [2, 4096] +``` + +Start the server pointed at it: + +```bash +tidal-server standalone \ + --listen 0.0.0.0:9400 \ + --schema ./schema.yaml \ + --data-dir /var/lib/myapp/tidaldb \ + --metrics 127.0.0.1:9091 +``` + +> Set `TIDAL_API_KEY` to require `Authorization: Bearer ` on the data routes. +> If it is unset the server runs **unauthenticated** and logs a WARN. The `--metrics` +> endpoint is always unauthenticated — bind it to loopback / cluster-internal only. + +--- + +## 3. Worked wiring per model + +The shape is identical for every model: **your app calls its model, then hands the +vector to tidalDB.** Below, `db` is an `Arc` opened against a schema whose +`content` slot matches the model's dimensions. + +### OpenAI `text-embedding-3-small` (1536D) + +```rust +use std::collections::HashMap; +use tidaldb::schema::{EntityId, Timestamp}; + +// Your model client. tidalDB never sees your API key or HTTP client. +async fn embed_openai(text: &str) -> anyhow::Result> { + let resp: serde_json::Value = reqwest::Client::new() + .post("https://api.openai.com/v1/embeddings") + .bearer_auth(std::env::var("OPENAI_API_KEY")?) + .json(&serde_json::json!({ + "model": "text-embedding-3-small", + "input": text, + })) + .send().await? + .json().await?; + let v: Vec = resp["data"][0]["embedding"] + .as_array().ok_or_else(|| anyhow::anyhow!("no embedding"))? + .iter().map(|x| x.as_f64().unwrap_or(0.0) as f32).collect(); + anyhow::ensure!(v.len() == 1536, "expected 1536D, got {}", v.len()); + Ok(v) +} + +// Ingest one item: metadata first, then its embedding. +async fn ingest(db: &tidaldb::TidalDb, id: u64, title: &str) -> anyhow::Result<()> { + let mut meta = HashMap::new(); + meta.insert("title".into(), title.to_string()); + meta.insert("created_at".into(), Timestamp::now().as_nanos().to_string()); + db.write_item_with_metadata(EntityId::new(id), &meta)?; + + let vec = embed_openai(title).await?; // app computes the vector + db.write_item_embedding(EntityId::new(id), &vec)?; // tidalDB stores + indexes it + Ok(()) +} +``` + +curl, against the HTTP server: + +```bash +# 1. Compute the vector with your model (here, OpenAI). +VEC=$(curl -s https://api.openai.com/v1/embeddings \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model":"text-embedding-3-small","input":"Introduction to Jazz Piano"}' \ + | jq -c '.data[0].embedding') + +# 2. Write metadata, then the embedding, to tidalDB. +curl -s -X POST http://localhost:9400/items \ + -H "Authorization: Bearer $TIDAL_API_KEY" -H "Content-Type: application/json" \ + -d '{"entity_id":1,"metadata":{"title":"Introduction to Jazz Piano"}}' + +curl -s -X POST http://localhost:9400/embeddings \ + -H "Authorization: Bearer $TIDAL_API_KEY" -H "Content-Type: application/json" \ + -d "{\"entity_id\":1,\"values\":$VEC}" # 204 No Content on success +``` + +### OpenAI `text-embedding-3-large` (3072D, shortenable) + +`text-embedding-3-large` is 3072D natively — within the `[2, 4096]` ceiling, so it +works directly with a `dimensions: 3072` slot. It is also **Matryoshka-shortenable**: +pass the `dimensions` parameter to trade a little accuracy for a smaller, faster index. +**The slot must match whatever you choose** — and once chosen, it is fixed for that +slot (see [§6](#6-model-versioning)). + +```rust +async fn embed_openai_large(text: &str, dims: u32) -> anyhow::Result> { + let resp: serde_json::Value = reqwest::Client::new() + .post("https://api.openai.com/v1/embeddings") + .bearer_auth(std::env::var("OPENAI_API_KEY")?) + .json(&serde_json::json!({ + "model": "text-embedding-3-large", + "input": text, + "dimensions": dims, // e.g. 1024 to shorten; omit for native 3072 + })) + .send().await?.json().await?; + let v: Vec = resp["data"][0]["embedding"].as_array() + .ok_or_else(|| anyhow::anyhow!("no embedding"))? + .iter().map(|x| x.as_f64().unwrap_or(0.0) as f32).collect(); + anyhow::ensure!(v.len() == dims as usize, "asked {dims}D, got {}", v.len()); + Ok(v) +} +// Slot must be declared with the SAME dims you pass here: +// s.embedding_slot("content", EntityKind::Item, 1024); // if dims = 1024 +// s.embedding_slot("content", EntityKind::Item, 3072); // if native +``` + +### Cohere `embed-english-v3.0` (1024D) + +Cohere distinguishes `input_type` between indexing documents and embedding queries — +use `search_document` when ingesting items and `search_query` at query time. **Both +must be the same model.** + +```rust +async fn embed_cohere(text: &str, input_type: &str) -> anyhow::Result> { + let resp: serde_json::Value = reqwest::Client::new() + .post("https://api.cohere.com/v1/embed") + .bearer_auth(std::env::var("COHERE_API_KEY")?) + .json(&serde_json::json!({ + "model": "embed-english-v3.0", + "texts": [text], + "input_type": input_type, // "search_document" for items + "embedding_types": ["float"], + })) + .send().await?.json().await?; + let v: Vec = resp["embeddings"]["float"][0].as_array() + .ok_or_else(|| anyhow::anyhow!("no embedding"))? + .iter().map(|x| x.as_f64().unwrap_or(0.0) as f32).collect(); + anyhow::ensure!(v.len() == 1024, "expected 1024D, got {}", v.len()); + Ok(v) +} + +// At ingest: +let vec = embed_cohere(title, "search_document").await?; +db.write_item_embedding(EntityId::new(id), &vec)?; +``` + +### Local model — sentence-transformers (text) or CLIP (video frames) + +A local model is the same contract: produce a `Vec`, write it. Run the model in +your own service (Python, ONNX Runtime, Candle, etc.) and pass tidalDB the result. + +**sentence-transformers (384D MiniLM):** your embedding service exposes, say, +`POST /embed -> {"vector": [...]}`. The tidalDB integration is just: + +```rust +let vec: Vec = my_embed_service.embed(title).await?; // 384 floats +db.write_item_embedding(EntityId::new(id), &vec)?; // slot declared 384D +``` + +**CLIP for video frames (512D):** a short-video item has no single text to embed — +embed the content directly. tidalDB stores one vector per item, so fuse frames into +one vector before writing (sample N frames, CLIP each, mean-pool, then write): + +```rust +// In your encoder service (pseudocode of the offline step): +// frames = sample_keyframes(video, n = 8) +// vecs = frames.map(clip_image_encode) // each 512D +// fused = mean_pool(vecs) // one 512D vector +// Hand the fused vector to tidalDB: +let fused: Vec = encoder.embed_video(&video_id).await?; // 512 floats, non-zero +db.write_item_embedding(EntityId::new(id), &fused)?; // slot declared 512D +``` + +> Guard against degenerate output: a blank frame or empty caption can yield an +> all-zero vector, which tidalDB rejects with `ZeroNormVector`. Skip or backfill those +> items rather than writing a zero. + +curl is identical regardless of which model produced the vector — the server only sees +floats: + +```bash +curl -s -X POST http://localhost:9400/embeddings \ + -H "Authorization: Bearer $TIDAL_API_KEY" -H "Content-Type: application/json" \ + -d '{"entity_id":7,"values":[0.013,-0.044, ... 512 floats ...]}' +``` + +--- + +## 4. The query side + +**The query vector MUST come from the same model and version as the item vectors.** +A vector from a different model lives in a different geometric space; cosine similarity +across spaces is meaningless. There is no cross-space conversion — re-embed the query +text/image with the exact model you indexed with. + +### Hybrid search — `Search::builder().vector(...)` + +`SEARCH` fuses BM25 text relevance with ANN semantic similarity (Reciprocal Rank +Fusion). Provide both the query text and a query vector you computed: + +```rust +use tidaldb::query::search::Search; + +// SAME model as the items. For Cohere, use input_type = "search_query" here. +let qvec: Vec = embed_openai("relaxing jazz piano").await?; // 1536D, matches slot + +let q = Search::builder() + .query("relaxing jazz piano") // BM25 side + .vector(qvec) // ANN side — same model/version as items + .for_user(42) // personalization (optional) + .limit(20) + .build()?; +let results = db.search(&q)?; +``` + +curl (hybrid): compute the query vector with your model, then pass it on the search. +The HTTP `GET /search` route takes `query`, `user_id`, and `limit`; for a query vector, +use the library API or the `POST`-bodied search if your deployment exposes one. Text-only +search is always available: + +```bash +curl -s "http://localhost:9400/search?query=relaxing%20jazz%20piano&user_id=42&limit=20" \ + -H "Authorization: Bearer $TIDAL_API_KEY" +``` + +### More like this — `.similar_to(item_id)` + +To find items near an existing item, you do not need to embed anything — tidalDB uses +the stored vector of `item_id` as the query, through the first content slot: + +```rust +let q = Search::builder() + .similar_to(EntityId::new(1)) // "more like item 1" + .for_user(42) + .exclude(vec![EntityId::new(1)]) // don't return the seed item + .limit(20) + .build()?; +let results = db.search(&q)?; +``` + +`Search::builder()` also takes `.diversity(DiversityConstraints { max_per_creator, +format_mix_max_fraction })` and `.exclude([ids])`. See +[API.md — SEARCH](../../API.md#search--text--semantic-retrieval). + +> Pure feeds use `Retrieve` (`profile("for_you")`, etc.), which leans on signals and +> the per-user preference vector rather than an ad-hoc query vector. The preference +> vector is built from item embeddings folded in by `positive_engagement(true)` signals +> — so the query side of personalization is implicitly "the same model as your items." + +--- + +## 5. Batching 1M items + +The write path is one vector per call, and **re-writes are idempotent** — +`write_item_embedding` overwrites the stored vector and re-inserts into the index, so +re-running a batch is safe (use it for resumable backfills). Batch the **model** calls +(that is where the latency and cost are), then write each result individually. + +```rust +use tidaldb::schema::EntityId; + +// Embedding APIs accept many inputs per request — batch the model, not the DB write. +for chunk in items.chunks(256) { + // One model call for up to 256 inputs (OpenAI/Cohere accept arrays). + let vectors: Vec> = + embed_batch_openai(chunk.iter().map(|i| i.title.as_str())).await?; + + for (item, vec) in chunk.iter().zip(vectors) { + // Length is checked here against the slot — a bad chunk fails fast. + db.write_item_embedding(EntityId::new(item.id), &vec)?; // idempotent + } +} +db.flush_text_index()?; // make freshly written items searchable (ephemeral mode) +``` + +Operational notes for a 1M backfill: + +- **Resumability.** Track the last successfully written id. On restart, re-run from + there — already-written vectors overwrite harmlessly. No dedup bookkeeping needed. +- **Index promotion is automatic.** A cold slot uses an exact brute-force index; once a + slot's durable vector count crosses the internal threshold, the next process restart + promotes it to HNSW (`UsearchIndex`). No config flag — just let the backfill complete + and restart. +- **Cost.** The model calls dominate cost and wall-clock. tidalDB's per-vector write is + cheap; size your batch to the model API's limits, not the database's. +- **HTTP path.** Over the server, the same shape: batch your model calls, then fire one + `POST /embeddings` per item (`{entity_id, values}` → 204). Mind the server's 2MB body + limit (413), 100 max in-flight (429), and 30s timeout (408) — a single 3072D vector is + well under 2MB, but do not pack many vectors into one body. + +--- + +## 6. Model versioning + +**A new embedding model is a new vector space. You cannot mix vectors from two models +in one slot** — their geometry is incomparable, and ANN results would be silently wrong +(no error, just nonsense neighbors). The slot's dimensions are part of the on-disk +schema fingerprint, so even a dimension change is a hard break: opening an existing +data directory with a different slot dimension fails schema validation. + +When you upgrade or switch models, choose one of: + +1. **Re-embed everything (clean cut).** Re-run the full backfill (§5) with the new + model, overwriting every item's vector in the same slot, then switch query-side + embedding to the new model atomically. Simplest; requires one full re-embed pass and + a brief window where new and old must not be queried interleaved. If dimensions + changed, you also need a fresh data directory (or a slot of the new size). + +2. **Separate slot / entity kind (parallel spaces).** Declare a second embedding slot + (or model items as a distinct entity kind) for the new model, backfill it alongside + the old, and cut queries over once it is fully populated. This lets old and new + coexist during migration. **Caveat:** `RETRIEVE`/`SEARCH` ANN routes through the + **first** declared Item slot only — so the new slot only takes effect for default + queries once it is the first slot (i.e. on a schema that declares it first). Until + then, query the new space explicitly via your own search path. Plan the cutover + schema so the slot you want serving default queries is declared first. + +Never write a v2 vector into a v1 slot "to save a re-embed." There is no compatibility +between spaces, and tidalDB cannot detect the mistake — the dimensions may even match. + +--- + +## 7. Failure modes + +| Symptom | Cause | Fix | +|---------|-------|-----| +| Insert error `DimensionMismatch { expected, got }` | Vector length ≠ slot dimensions. | Make the model output match the slot, or re-declare the slot. A truncated/padded vector is wrong even if it inserts — fix the model call. | +| `schema.build()` fails: "invalid dimensions … must be in [2, 4096]" | Slot declared outside `[2, 4096]`. | Pick a model ≤4096D, or project the vector down (Matryoshka `dimensions` param) and re-normalize before writing. | +| Insert error `ZeroNormVector` | All-zero / underflowing vector (blank input text, empty frame, model error returning zeros). | Guard the encoder; skip or backfill degenerate items. Never write a zero placeholder. | +| ANN returns nonsense neighbors, no error | Query vector from a different model/version than the items, **or** v2 vectors written into a v1 slot. | Re-embed the query with the exact item model (§4); never mix models in a slot (§6). | +| Multi-modal app: image search ignores the text slot (or vice-versa) | `RETRIEVE`/`SEARCH` route ANN through the **first** declared Item slot only. | Fuse modalities offline into one vector per item, or model each modality as a separate entity kind and query it explicitly. Plan slot order for the cutover. | +| `for_you` doesn't reflect a user's taste despite engagement | The engaged signal is not `positive_engagement(true)`, or you wrote `signal` instead of `signal_with_context`. | Declare taste-bearing signals (completion, like, replay) `.positive_engagement(true)` and write them via `signal_with_context(.., Some(user_id), ..)`. | + +--- + +## What tidalDB owns vs. what you own + +tidalDB owns **retrieval and ranking** over the vectors you give it: the HNSW index, +L2 normalization, signal-driven scoring, per-user preference vectors, diversity, and +the feed ordering — one process, one query. + +You own **everything upstream**: embedding generation (running the model), the model's +API keys and clients, video/blob storage, CDN/transcoding, auth, moderation, and the +player UI. tidalDB never generates embeddings and never sees your model. + +To build the full application around this — schema design, the signal loop, feeds, and +the HTTP surface — see [docs/guides/build-a-feed-app.md](build-a-feed-app.md), the +runnable [`foryou_feed` example](../../tidal/examples/foryou_feed.rs), and +[QUICKSTART.md](../../QUICKSTART.md). diff --git a/docs/guides/server-deployment.md b/docs/guides/server-deployment.md new file mode 100644 index 0000000..5ea6ba4 --- /dev/null +++ b/docs/guides/server-deployment.md @@ -0,0 +1,364 @@ +# Server Deployment + +Operational reference for running `tidal-server` — the HTTP wrapper around the embeddable `tidaldb` engine. This is the doc to read before exposing tidalDB over the network. + +`tidal-server` is a thin Axum surface over the engine. It does not change the engine's correctness model: every write still goes through the WAL, every shutdown still checkpoints and fsyncs. What the server adds is an HTTP contract, Bearer auth, request middleware (timeouts, concurrency cap, body limit, request IDs), and a YAML-driven schema/profile loader so you can deploy without writing Rust. + +> tidalDB owns **retrieval and ranking only**. It does **not** generate embeddings, store video/blobs, run a CDN/transcoder, or provide auth/moderation/payments. The server brings vectors in over `POST /embeddings`; your application produces them. + +**Cross-references** + +| Topic | Doc | +|-------|-----| +| HTTP + Rust API contract (handwritten) | [API.md](../../API.md) | +| Get-started walkthrough | [QUICKSTART.md](../../QUICKSTART.md) | +| Metrics, alert thresholds, Grafana | [docs/ops/monitoring.md](../ops/monitoring.md) | +| Crash recovery, WAL replay, backup/restore | [docs/ops/recovery.md](../ops/recovery.md) | +| Sizing memory/disk/CPU, scrape budgets | [docs/ops/capacity-planning.md](../ops/capacity-planning.md) | +| Kubernetes deployment (probes, volumes) | [docs/runbooks/kubernetes.md](../runbooks/kubernetes.md) | +| Cluster operation (experimental) | [docs/runbooks/cluster.md](../runbooks/cluster.md) | + +--- + +## 1. Run Modes + +`tidal-server` has two subcommands. Pick `standalone` unless you have a specific reason not to. + +### Standalone — the recommended path + +tidalDB is **single-node-first**. One process wraps one `TidalDb` instance, serves the full data + health surface, and scales vertically. This is the production-ready mode. It is the default in every shipped Docker image and the only mode covered by the durability and recovery guarantees in [docs/ops/recovery.md](../ops/recovery.md). + +```bash +tidal-server standalone \ + --listen 0.0.0.0:9400 \ + --schema ./schema.yaml \ + --data-dir /var/lib/tidaldb \ + --metrics 127.0.0.1:9091 +``` + +### Cluster — EXPERIMENTAL, not production HA + +Cluster mode runs multiple "regions" behind one HTTP surface and replicates between them over the **real `tidal-net` gRPC transport**. The critical caveat: **all regions run inside this one process over loopback** — there is no host or process isolation, so a process crash takes down every region at once. **This is not production high availability.** True multi-process HA is tracked as **m8p10**. + +The mode refuses to start unless you opt in explicitly (`--experimental-cluster` or `TIDAL_ALLOW_EXPERIMENTAL_CLUSTER=1`) and emits a loud `WARN` describing exactly what it is and is not. Cluster mode currently replicates **global signals only** — `user_id`/`creator_id` signal context is dropped on the cluster path so followers stay consistent with the leader's WAL. + +For everything cluster-specific (topology file, leader promotion, partition/heal, scatter-gather routes), see the **[cluster runbook](../runbooks/cluster.md)**. The rest of this guide focuses on standalone. + +--- + +## 2. CLI Flags and Environment Variables + +### Standalone flags + +| Flag | Env override | Default | Meaning | +|------|--------------|---------|---------| +| `--listen ` | `PORT` | `127.0.0.1:9400` | HTTP bind address. A bare `PORT` value (e.g. `PORT=8080`) is normalized to `0.0.0.0:8080`. A `host:port` string is used verbatim. | +| `--schema ` | — | compiled-in default | Path to the schema/profile YAML (see [section 3](#3-schema-yaml)). | +| `--config-dir ` | `TIDAL_CONFIG` | unset | Directory holding `default-schema.yaml`. Used only when `--schema` is omitted; the explicit flag always wins. If the dir is set but the file is missing, startup **fails loudly** rather than silently serving compiled-in defaults. | +| `--data-dir ` | — | **unset → EPHEMERAL** | Persistent data directory. **If omitted, the server runs in-memory and loses every write on restart.** Always set this in production. | +| `--metrics ` | — | unset (disabled) | Bind address for the Prometheus endpoint. **Bind to loopback only — it is unauthenticated** (see [section 7](#7-metrics-and-shutdown)). | + +### Cluster flags + +| Flag | Env override | Default | Meaning | +|------|--------------|---------|---------| +| `--listen ` | `PORT` | `127.0.0.1:9500` | HTTP bind address (same bare-port normalization). | +| `--schema ` | — | compiled-in default | Schema/profile YAML. | +| `--topology ` | — | compiled-in default | Cluster topology YAML (`regions`, `leader`). See the [cluster runbook](../runbooks/cluster.md). | +| `--config-dir ` | `TIDAL_CONFIG` | unset | Directory holding `default-schema.yaml` and `default-cluster.yaml`. | +| `--experimental-cluster` | `TIDAL_ALLOW_EXPERIMENTAL_CLUSTER` | off | Required opt-in. Without it (flag or env `1`/`true`/`yes`), cluster mode refuses to start. | + +### Environment variables (all modes) + +| Variable | Default | Meaning | +|----------|---------|---------| +| `TIDAL_API_KEY` | unset | Bearer token for data routes. **If unset, the server is UNAUTHENTICATED** and logs a WARN at startup. See [section 4](#4-authentication). | +| `TIDAL_CONFIG` | unset | Config directory (backs `--config-dir`). | +| `PORT` | unset | Listen port/address (backs `--listen`); bare port → `0.0.0.0:PORT`. | +| `TIDAL_SERVER_LOG` | `info` | `tracing` env-filter directive, e.g. `TIDAL_SERVER_LOG=tidal_server=debug,info`. | +| `TIDAL_ALLOW_EXPERIMENTAL_CLUSTER` | unset | Truthy (`1`/`true`/`yes`) opts in to cluster mode. | + +--- + +## 3. Schema YAML + +The server loads its schema and ranking profiles from YAML at startup. This is the YAML projection of the Rust `SchemaBuilder` API. Four top-level keys: `signals` (required), `text_fields`, `embedding_slots`, `profiles` (all optional). At least one signal must be defined or startup fails. + +### Top-level shape + +```yaml +signals: # required, >= 1 +text_fields: # optional +embedding_slots: # optional +profiles: # optional — built-in profiles are available regardless +``` + +### `signals` + +```yaml +signals: + - name: view + entity: item # item | user | creator + decay: + exponential: + half_life_seconds: 604800 # exclusive: exponential | linear | permanent + windows: [one_hour, twenty_four_hours, seven_days] + velocity: true + positive_engagement: true # optional; folds into the user preference vector +``` + +- `decay` is exactly one of: + - `exponential: { half_life_seconds: }` + - `linear: { lifetime_seconds: }` + - `permanent: true` +- `windows` accepts: `one_hour` (`1h`), `twenty_four_hours` (`24h`), `seven_days` (`7d`), `thirty_days` (`30d`), `all_time` (`alltime`). +- `velocity: true` enables velocity tracking for the signal. +- `positive_engagement: true` makes engaging with an item via this signal fold + the item's embedding into the user's preference vector (personalization). + Omit it and the engine falls back to name-based detection — declare it + explicitly on signals like `view`/`like`/`completion` for predictable taste. + +### `text_fields` + +```yaml +text_fields: + - name: title + kind: text # text (analyzed, BM25) | keyword (exact) + - name: category + kind: keyword +``` + +### `embedding_slots` + +```yaml +embedding_slots: + - name: content_vector + entity: item + dimensions: 128 +``` + +Embedding rules to respect from the engine side: +- The app **brings vectors**; the server's `POST /embeddings` L2-normalizes them and inserts into the HNSW index. tidalDB does not generate embeddings. +- Dimensions are **strict**: min 2, max 4096, and the submitted vector length **must equal the slot's declared `dimensions`** or the insert fails. Zero-norm vectors are rejected. +- **Multi-slot caveat:** `RETRIEVE`/`SEARCH` route through the **first declared slot only**. Multi-modal apps must fuse offline or use separate entity kinds. + +### `profiles` + +Optional. The 25 built-in profiles (`for_you`, `trending`, `hot`, `new`, `following`, `related`, `top_week`, `most_viewed`, `controversial`, `hidden_gems`, `shuffle`, `cohort_trending`, etc.) are available without declaring anything. Use `profiles` to add custom ones or override behavior. + +```yaml +profiles: + - name: for_you + version: 1 + candidate_strategy: # default: scan(created_at) + ann: + slot: content_vector + limit: 200 + sort: # string OR parameterized map + hot: + gravity: 1.5 + boosts: + - { signal: like, agg: decay_score, window: all_time, weight: 2.5 } + - { signal: view, agg: velocity, window: one_hour, weight: 1.0 } + gates: + - { signal: skip, agg: value, window: all_time, min_threshold: 0.0 } + penalties: + - { signal: skip, agg: decay_score, window: seven_days, weight: 1.5 } + excludes: + - { signal: skip, agg: value, window: all_time, above: 1.0 } + diversity: + max_per_creator: 2 + format_mix_max_fraction: 0.4 + exploration: 0.1 +``` + +Field reference: +- **`candidate_strategy`** — `scan` | `relationship` | `hybrid` | `cohort_trending`, or a map: `ann: { slot, limit }`, `signal_ranked: { signal, window }`, `scan: { sort_field }`. Default: `scan` over `created_at`. +- **`sort`** — string: `trending`, `rising`, `controversial`, `hidden_gems`, `shuffle`, `new`, `most_followed`, `creator_engagement_rate`, `alphabetical_asc`, `alphabetical_desc`, `shortest`, `longest`, `live_viewer_count`, `date_saved`; or map: `hot: { gravity }`, `top_window: { window }`, `most_viewed: { window }`, `most_liked: { window }`, `most_commented: { window }`, `most_shared: { window }`. +- **`boosts` / `penalties`** — `{ signal, agg, window, weight }`. `agg` ∈ `value | velocity | decay_score | ratio | relative_velocity`. +- **`gates`** — `{ signal, agg, window, min_threshold }` (drop candidates below the threshold). +- **`excludes`** — `{ signal, agg, window, above }` (drop candidates above the threshold; e.g. hard-negative skip/hide/block). +- **`diversity`** — `{ max_per_creator, format_mix_max_fraction }`. +- **`exploration`** — `f64` in `[0,1]`. + +Every `signal` named in a profile's boosts/gates/penalties/excludes/decay **must be declared in `signals`**, or startup fails with a clear error naming the offending profile and signal. + +> **Correctness rule — preference-vector personalization.** The user "taste vector" that drives `for_you` is folded from item embeddings **only for signals declared `positive_engagement: true`** (the YAML field above; equivalently `SignalBuilder::positive_engagement(true)` in embedded mode). Declare it on your engaging signals (`view`/`like`/`completion`); if you omit it on *every* signal, the engine falls back to a name-based allowlist. If your personalization looks flat, confirm the engaging signals carry the flag. See [API.md](../../API.md) for the underlying `signal_with_context` semantics. + +### Worked example (the shipped default) + +```yaml +signals: + - name: view + entity: item + decay: + exponential: + half_life_seconds: 604800 # 7 days + windows: [one_hour, twenty_four_hours, seven_days] + velocity: true + - name: like + entity: item + decay: + exponential: + half_life_seconds: 1209600 # 14 days + windows: [twenty_four_hours, seven_days, thirty_days, all_time] + velocity: false + - name: skip + entity: item + decay: + permanent: true + velocity: false +text_fields: + - name: title + kind: text + - name: category + kind: keyword +embedding_slots: + - name: content_vector + entity: item + dimensions: 128 +``` + +This is the compiled-in default at `tidal-server/config/default-schema.yaml`, used when neither `--schema` nor a `TIDAL_CONFIG` directory provides one. + +--- + +## 4. Authentication + +Data routes (`/items`, `/embeddings`, `/signals`, `/feed`, `/search`) require an `Authorization: Bearer ` header **when `TIDAL_API_KEY` is set**. The comparison is constant-time. A missing or wrong token returns `401` with `WWW-Authenticate: Bearer`. + +Health probes (`/health`, `/health/startup`, `/health/live`) and the OpenAPI document (`/openapi.json`) are **always unauthenticated** — a client needs the spec to learn how to authenticate, and probes must work for orchestrators that cannot present a token. + +> **WARNING — no key means an open server.** If `TIDAL_API_KEY` is unset (or empty), **every data route is served without authentication** and the server logs a startup WARN. This is fine for local development. **Never expose an unauthenticated server to any network.** Set `TIDAL_API_KEY` before binding to anything other than `127.0.0.1`. + +```bash +export TIDAL_API_KEY="$(openssl rand -hex 32)" +curl -H "Authorization: Bearer $TIDAL_API_KEY" \ + "http://localhost:9400/feed?user_id=42&profile=for_you&limit=20" +``` + +--- + +## 5. The Served OpenAPI Reference (`GET /openapi.json`) + +`tidal-server` serves a machine-readable **OpenAPI 3.1** document at `GET /openapi.json`, unauthenticated. **This is the canonical HTTP API contract** — it is derived directly from the handler signatures and DTOs, so it can never drift from the running code (unlike a handwritten spec). [API.md](../../API.md) remains the prose companion, but `/openapi.json` is the source of truth for request/response shapes. + +Inspect it: + +```bash +curl -s localhost:9400/openapi.json | jq .info +# { +# "title": "tidalDB HTTP API", +# "version": "", +# "description": "Embeddable, single-node-first database for the personalized content ranking problem..." +# } +``` + +You can load `/openapi.json` into any OpenAPI viewer (Swagger UI, Redoc, Stoplight, Postman) or feed it to a client generator (`openapi-generator`, `oapi-codegen`, etc.) to produce typed clients in any language. The standalone document describes the data + health surface; the cluster document is a superset that also describes the `/cluster/*` and `/sharded/*` routes. The `bearerAuth` security scheme is declared on the data routes so generated clients know to send the token; `/openapi.json` and the probes carry no security requirement. + +### Route summary (standalone) + +| Method + path | Auth | Success | Notes | +|---------------|------|---------|-------| +| `GET /health` | no | `200` ready / `503` draining | Readiness probe; reports item count. | +| `GET /health/startup` | no | `200` always | Startup probe. | +| `GET /health/live` | no | `200` always | Liveness probe. | +| `GET /openapi.json` | no | `200` | OpenAPI 3.1 document. | +| `POST /items` | yes | `201` | Body: `{ "entity_id": , "metadata": { ... } }`. | +| `POST /embeddings` | yes | `204` | Body: `{ "entity_id": , "values": [, ...] }`. | +| `POST /signals` | yes | `204` | Body: `{ "entity_id", "signal", "weight", "user_id"?, "creator_id"? }`. | +| `GET /feed` | yes | `200` | `?user_id=&profile=for_you&limit=20`. | +| `GET /search` | yes | `200` | `?query=&user_id=&limit=20`. | + +`limit` is clamped to `1000` at the trust boundary. Request middleware: 30 s timeout (`408`), 100 max in-flight (`429`), 2 MB body cap (`413`), and an `x-request-id` echoed on every response. + +--- + +## 6. Running the Server + +### Via cargo (development) + +```bash +# Ephemeral, loopback, default schema — quickest smoke test: +cargo run -p tidal-server -- standalone + +# Durable, real schema, metrics on loopback: +cargo run -p tidal-server -- standalone \ + --listen 0.0.0.0:9400 \ + --schema tidal-server/config/default-schema.yaml \ + --data-dir /var/lib/tidaldb \ + --metrics 127.0.0.1:9091 +``` + +For an end-to-end engine walkthrough (embedded, no server), run the verified example: `cargo run -p tidaldb --example foryou_feed`. See [QUICKSTART.md](../../QUICKSTART.md). + +### Via Docker + +Three images ship, all built from the **repository root** as build context: + +| Image | Dockerfile | Toolchain | Use | +|-------|-----------|-----------|-----| +| standalone | `docker/standalone/Dockerfile` | `rust:1.91-bookworm` | General single-node; bundles `curl` for healthcheck. | +| deploy | `docker/deploy/Dockerfile` | `rust:1.91-slim-bookworm`, uid 10001 | Slim production single-node; bakes the schema, `WORKDIR /data`. | +| cluster | `docker/cluster/Dockerfile` | — | Experimental cluster mode; see the [cluster runbook](../runbooks/cluster.md). | + +Both single-node images: +- Run as a **non-root** `tidal` user. +- Declare `VOLUME ["/data"]` and pass `--data-dir /data` in the default `CMD`. **Durability lives on `/data` — you must mount a real volume there or every write is lost on restart** (the container layer is ephemeral). +- Use `ENTRYPOINT ["tidal-server"]` + a `CMD` of subcommand args, so `docker run ...` overrides work (e.g. running an inspect command). +- Set `ENV TIDAL_CONFIG` to the baked config dir (`/etc/tidal-server` for standalone, `/config` for deploy) so the container is self-contained. +- Expose `9400` (HTTP) and `9091` (metrics). + +```bash +# Build (from repo root): +docker build -f docker/standalone/Dockerfile -t tidaldb:standalone . + +# Run with a named volume for durability and an API key: +docker run -d --name tidaldb \ + -p 9400:9400 \ + -p 127.0.0.1:9091:9091 \ + -e TIDAL_API_KEY="$(openssl rand -hex 32)" \ + -v tidaldb-data:/data \ + tidaldb:standalone +``` + +Note the metrics port is published to `127.0.0.1` only — it is unauthenticated (see [section 7](#7-metrics-and-shutdown)). + +### Via docker-compose + +`docker/docker-compose.yml` brings up the standalone image plus a Prometheus sidecar, with a named `tidaldb-data` volume mounted at `/data`: + +```bash +cd docker +export TIDAL_API_KEY="$(openssl rand -hex 32)" +docker compose up -d +``` + +The compose healthcheck calls `GET /health` with the Bearer header. Adjust the `tidaldb-data` volume to a host bind mount if you want the data outside Docker's volume store. For Kubernetes equivalents (probes wired to `/health/startup`, `/health/live`, `/health`; a `PersistentVolumeClaim` at `/data`), see the [Kubernetes runbook](../runbooks/kubernetes.md). + +--- + +## 7. Metrics and Shutdown + +### Metrics + +`--metrics ` enables the engine's Prometheus endpoint on a **separate port** (default image: `9091`). It serves `/metrics` (Prometheus text format) and `/healthz` (JSON). **This endpoint is UNAUTHENTICATED — bind it to loopback or a cluster-internal interface only.** The engine logs a WARN if you bind it to a non-loopback address. For the full metric catalog, scrape config, alert thresholds, and Grafana dashboard, see [docs/ops/monitoring.md](../ops/monitoring.md). For sizing the scrape budget and memory/disk/CPU, see [docs/ops/capacity-planning.md](../ops/capacity-planning.md). + +### Graceful shutdown + +On `SIGTERM` (or Ctrl-C), the server: +1. **Flips readiness to `503`** — `GET /health` reports `{ "ok": false, "cause": "shutting down" }` so a load balancer stops routing new traffic. Startup/liveness probes still return `200`. +2. **Drains** in-flight requests (axum graceful shutdown). +3. **Closes the database** — checkpoints state and **fsyncs the WAL** before the process exits. The standalone path reclaims sole ownership and runs this deterministically; if a flush fails it is logged at error level. There is a `Drop` backstop that runs the same shutdown if ownership is unexpectedly still shared. + +Give orchestrators a `terminationGracePeriod` long enough to cover drain + checkpoint (a few seconds is typically ample for single-node; size against your WAL volume per [capacity planning](../ops/capacity-planning.md)). For crash recovery, WAL replay, and backup/restore, see [docs/ops/recovery.md](../ops/recovery.md). + +### Reverse proxy and TLS termination + +`tidal-server` speaks **plain HTTP** and does not terminate TLS. In production, front it with a reverse proxy (nginx, Caddy, Envoy, a cloud load balancer, or a Kubernetes Ingress) that: +- Terminates TLS and forwards to the server's `--listen` address. +- Performs health checks against `GET /health` (it returns `503` during drain, so the proxy stops sending traffic before the process exits). +- Optionally adds its own rate limiting in front of the server's 100-in-flight / 30 s-timeout middleware. +- Preserves or sets `x-request-id` if you want end-to-end tracing — the server generates one when absent and echoes it on the response. + +Keep the metrics port off the public proxy entirely; scrape it over the internal network only. diff --git a/docs/runbooks/cluster.md b/docs/runbooks/cluster.md index dbcf725..52b87fb 100644 --- a/docs/runbooks/cluster.md +++ b/docs/runbooks/cluster.md @@ -1,113 +1,268 @@ # tidalDB Cluster Runbook -This runbook describes how to operate the simulated multi-region tidalDB -cluster that ships with `tidal-server`. The cluster reuses the -`SimulatedCluster` fabric — it runs multiple in-process nodes, replays the -real WAL + CRDT reconciliation paths, and exposes a single HTTP surface -for microservices. +Operating the multi-region `tidal-server cluster` surface: launch, the +operational API, replication transport facts, failover and partition drills, +and the honest write-durability contract. -> **Important limitations** +> ## STATUS: EXPERIMENTAL — SINGLE-PROCESS, NOT PRODUCTION HA > -> - Cluster mode currently replicates global signals only. `user_id` / -> `creator_id` contexts are rejected so followers stay consistent with the -> leader’s WAL stream. -> - All metadata and embedding writes are broadcast to every region up front. -> There is no separate replication log for items yet. +> Cluster mode runs **every region inside one `tidal-server` process**. The +> regions replicate over the **real `tidal-net` gRPC transport** (`ShipSegment` +> RPC, serialization, circuit breaker, HTTP/2 — see [§4](#4-grpc-replication-transport-tidal-net)), +> but that transport is a **loopback self-loop**: a follower's gRPC server and +> its only peer entry are the same loopback address. So you get faithful +> multi-region replication *semantics* over a real wire, with **no host or +> process isolation**. +> +> **A process crash, OOM, or host failure takes the entire "cluster" down at +> once.** There is no second machine to fail over to. This is a development / +> staging / demo fabric and a correctness harness for the replication paths — +> **it is NOT production high availability.** +> +> True multi-process, multi-host HA (process isolation, real network between +> regions, quorum-acked writes) is tracked as **m8p10** in +> [docs/planning/ROADMAP.md](../planning/ROADMAP.md). Until then, for a real +> production deployment run a **single `tidal-server standalone`** node and back +> it with host-level redundancy and disk durability. +> +> **Cluster mode refuses to start** unless you explicitly opt in with either the +> `--experimental-cluster` flag or the `TIDAL_ALLOW_EXPERIMENTAL_CLUSTER=1` +> environment variable. On start it emits a loud `WARN` restating exactly the +> above. Do not wire it into a production load balancer. + +## Scope: what cluster mode does and does not own + +tidalDB owns **retrieval and ranking only**. Cluster mode adds multi-region +replication of that, nothing more. It does **not** generate embeddings, store +video/blobs, run a CDN or transcoder, do auth/moderation/payments, or provide a +distributed consensus log. Bring your vectors; tidalDB retrieves and ranks over +them. See [VISION.md](../../VISION.md) for the scope boundary. ## Prerequisites -- Rust toolchain ≥ 1.91 if running directly. +- Rust toolchain ≥ 1.91 if running directly (the Docker build pins + `rust:1.91-bookworm`). +- `protobuf-compiler` (`protoc`) and a C++ toolchain (`g++`) on the build host — + `tidal-net`'s build script compiles the WAL-shipping `.proto`, and USearch's + HNSW core is C++. The Docker image installs both. - Docker 25+ if running via container. -- Port 9500 available (default cluster listener). +- Port `9500` available (default cluster HTTP listener). Each follower region + also binds an **OS-assigned loopback port** for its gRPC replication server + unless you pin one in the topology (see [§3](#3-topology-yaml)). ## 1. Launch the cluster locally +Cluster mode is gated. Pass `--experimental-cluster` (or set +`TIDAL_ALLOW_EXPERIMENTAL_CLUSTER=1`) or the server exits with an error +explaining why. + ```bash +TIDAL_ALLOW_EXPERIMENTAL_CLUSTER=1 \ cargo run -p tidal-server -- \ cluster \ --listen 127.0.0.1:9500 \ --schema tidal-server/config/default-schema.yaml \ - --topology tidal-server/config/default-cluster.yaml + --topology tidal-server/config/default-cluster.yaml \ + --experimental-cluster ``` -The default topology spins up three regions (`us-east`, `eu-west`, -`ap-south`) with `us-east` as leader. +The default topology spins up three regions (`us-east`, `eu-west`, `ap-south`) +with `us-east` as leader. On a clean start you will see a `WARN` line stating +this is experimental and single-process, an `info` line per follower +(`follower gRPC transport ready`), and finally `listening on http://127.0.0.1:9500`. + +**Auth:** set `TIDAL_API_KEY=` to require `Authorization: Bearer ` +on the data and `/cluster/*` mutation routes. If it is **unset the server runs +UNAUTHENTICATED and logs a WARN** — never expose an unauthenticated cluster +beyond loopback. Health probes and `/openapi.json` are always unauthenticated. + +Useful environment variables (shared with standalone): + +| Var | Effect | +|-----|--------| +| `TIDAL_ALLOW_EXPERIMENTAL_CLUSTER=1` | Opt in to cluster mode (alternative to `--experimental-cluster`). | +| `TIDAL_API_KEY` | Bearer token for protected routes. Unset ⇒ unauthenticated + WARN. | +| `TIDAL_CONFIG` | Config dir holding `default-schema.yaml` / `default-cluster.yaml` (used when `--schema` / `--topology` omitted). | +| `PORT` | Listen address. A bare port (`9500`) normalises to `0.0.0.0:9500`. | +| `TIDAL_SERVER_LOG` | `tracing` filter (default `info`). | ## 2. Launch via Docker ```bash -# Build the image once -docker build -f docker/cluster/Dockerfile -t tidal-cluster . +# Build the image once (build context is the repo root). +docker build -f docker/cluster/Dockerfile -t tidaldb:cluster . -# Run (press Ctrl+C to stop) -docker run --rm -p 9500:9500 tidal-cluster +# Run (press Ctrl+C to stop). The image already sets +# TIDAL_ALLOW_EXPERIMENTAL_CLUSTER=1 so cluster mode starts; it still logs the +# loud experimental WARN. +docker run --rm -p 9500:9500 tidaldb:cluster ``` -To supply custom schema/topology files: +The image bakes the default schema/topology under `/etc/tidal-server` and uses +`ENTRYPOINT ["tidal-server"]` + a `cluster …` `CMD`, so you can override the +subcommand (e.g. `docker run tidaldb:cluster standalone …`) or supply your own +config files: ```bash docker run --rm -p 9500:9500 \ - -v $PWD/configs/my-schema.yaml:/srv/schema.yaml \ - -v $PWD/configs/my-topology.yaml:/srv/topology.yaml \ - tidal-cluster \ - tidal-server cluster \ + -e TIDAL_API_KEY=changeme \ + -v "$PWD/configs/my-schema.yaml:/srv/schema.yaml:ro" \ + -v "$PWD/configs/my-topology.yaml:/srv/topology.yaml:ro" \ + tidaldb:cluster \ + cluster \ --listen 0.0.0.0:9500 \ --schema /srv/schema.yaml \ --topology /srv/topology.yaml ``` -## 3. Core API calls +The container ships a `HEALTHCHECK` that curls `/health`. For Kubernetes/Compose +deployment patterns see [Cross-references](#cross-references). -All routes are JSON unless noted. +## 3. Topology YAML + +The topology file declares regions and the leader. Minimal default +(`tidal-server/config/default-cluster.yaml`): + +```yaml +regions: + - name: us-east + - name: eu-west + - name: ap-south +leader: us-east +# Optional: OS worker threads serving cluster write/heal requests (gRPC ship). +# Defaults to available parallelism when omitted. +# write_workers: 4 +``` + +Fields: + +| Field | Required | Meaning | +|-------|----------|---------| +| `regions[].name` | yes | Region name used everywhere in the HTTP API (`?region=`, `/cluster/promote`, etc.). Must be unique. | +| `regions[].grpc_addr` | no | This region's gRPC replication bind address, e.g. `"127.0.0.1:9601"`. **Omit it** for the single-process default — the server allocates a free loopback port and self-heals a bind race by retrying on a fresh port. An explicit address is only needed for the future multi-process deployment (m8p10) and is tried exactly once. | +| `leader` | yes | Must name one of the declared regions. Writes route here. | +| `write_workers` | no | Size of the runtime-free OS-thread pool that runs the blocking gRPC segment-ship on `/signals` and `/cluster/heal`. Bounds write concurrency on the hottest path. | + +The schema YAML (signals / text_fields / embedding_slots / profiles) is the same +format the standalone server loads — see [API.md](../../API.md) and +[QUICKSTART.md](../../QUICKSTART.md) for the full schema/profile grammar and the +25 built-in ranking profiles. Topology only adds the region layout above. + +> **Personalization correctness note.** Preference-vector personalization (the +> taste vector that powers `for_you`) only updates for signals declared +> `positive_engagement: true` in the schema. The cluster `/signals` route writes +> **global** signals (no `user_id` / `creator_id` context), so it never folds +> item embeddings into a user's taste vector regardless of the +> `positive_engagement` flag. Personalized writes go through the embedded engine +> (`signal_with_context`), not this HTTP route. + +## 4. gRPC replication transport (tidal-net) + +Replication between regions is the real `tidal-net` `WalShipping` gRPC service, +not in-process channels. The facts you need to operate and firewall it: + +| Property | Value | Notes | +|----------|-------|-------| +| Service | `WalShipping` (proto `tidal.replication.v1`) | RPCs: **`ShipSegment`** (unary; the production replication path) and **`Heartbeat`** (ControlPlane health). `StreamSegments` is declared but **not implemented** — segment delivery is the unary `ShipSegment`. | +| Default port | `59529` | In tidalDB's reserved dev band `59520–59529`. In single-process cluster mode each follower binds an **auto-allocated** loopback port instead (or the topology's `grpc_addr`). | +| Transport security | mTLS via `TlsConfig` (`ca_cert`, `server_cert`, `server_key`, optional `client_cert` / `client_key`) | `insecure = true` (plaintext) is the single-process loopback default. A peer with no TLS config **must** set `insecure`. | +| Max payload | **64 MiB** | Both encoder and decoder codec limits are raised to this; pinned by a compile-time assert to the engine's `InProcessTransport` limit so both ends agree. WAL segments default to 16 MiB. | +| Circuit breaker | per-peer, **threshold 5**, **reset 30s** | After 5 consecutive ship failures the breaker opens; it stays open for 30s, then the next attempt probes (HalfOpen → Closed on success, Open again on failure). Backpressure (channel full) does **not** trip it. | +| Timeouts (defaults) | connect 5s, request 10s, keep-alive PING every 10s / 5s ACK | A blackholed peer fails fast instead of stalling the single-threaded shipper. | + +`ShipSegment` carries the WAL segment id, BLAKE3-validated payload bytes, event +count, and the leader's authoritative `leader_last_seq` (so a follower can +advance its replication-lag high-water-mark even for an all-local segment that +filters to empty). The follower's segment-receiver thread drains and applies it +on arrival. + +> **Operational guard:** the gRPC replication ports and the Prometheus `/metrics` +> endpoint are **UNAUTHENTICATED**. Bind them to loopback or a cluster-internal +> network only — never the public interface. + +## 5. Core HTTP API + +All routes are JSON unless noted. Examples assume `BASE=http://localhost:9500` +and, when `TIDAL_API_KEY` is set, `AUTH='-H "Authorization: Bearer $TIDAL_API_KEY"'`. +Drop the `-H` header when running unauthenticated. + +Middleware mirrors standalone: 30s request timeout (408), 100 max in-flight +(429), 2 MB body limit (413), and an `x-request-id` on every response. Health +probes sit **outside** the load-shedding stack so liveness/readiness are never +queued or timed out under saturation. ### Health ```bash -curl http://localhost:9500/health +curl "$BASE/health" # 200 ok / 503 while draining; reports leader + item count +curl "$BASE/health/startup" # always 200 +curl "$BASE/health/live" # always 200 +curl "$BASE/openapi.json" # served OpenAPI 3.1, UNAUTHENTICATED — canonical HTTP reference ``` -Returns overall status and item count on the leader. +The cluster `/openapi.json` is a superset that documents the `/cluster/*` and +`/sharded/*` routes. It is the machine-readable source of truth for request / +response shapes. -### Register items & embeddings +### Register items & embeddings (leader-routed) ```bash -curl -X POST http://localhost:9500/items \ +curl -X POST "$BASE/items" \ -H 'Content-Type: application/json' \ -d '{ "entity_id": 1, "metadata": { "title": "Jazz Piano", "category": "music" } }' +# → 201 Created -curl -X POST http://localhost:9500/embeddings \ +curl -X POST "$BASE/embeddings" \ -H 'Content-Type: application/json' \ -d '{ "entity_id": 1, "values": [0.1, 0.2, 0.3, 0.4] }' +# → 204 No Content ``` -### Record signals (cluster mode = global only) +Embeddings: tidalDB does **not** generate vectors — the caller brings them. The +write L2-normalizes and inserts into the HNSW index. Dimensions are **strict**: +they must equal the slot's declared dimensions (min 2, max 4096) or the insert +fails, and **zero-norm vectors are rejected**. `RETRIEVE` / `SEARCH` route +through the **first declared embedding slot only**; multi-modal apps must fuse +offline or use separate entity kinds. + +### Record signals (cluster `/signals` = global only) ```bash -curl -X POST http://localhost:9500/signals \ +curl -X POST "$BASE/signals" \ -H 'Content-Type: application/json' \ -d '{ "entity_id": 1, "signal": "view", "weight": 1.0 }' +# → 204 No Content (see the durability contract in §8) ``` -### Retrieve and search +The signal name must be declared in the schema. This route records a **global** +signal on the leader and ships it to followers; it does not personalize (see the +personalization note in [§3](#3-topology-yaml)). + +### Retrieve and search (region-pinned reads) ```bash -curl "http://localhost:9500/feed?user_id=42&profile=trending&limit=10" -curl "http://localhost:9500/search?query=jazz%20piano&limit=5" +# Default read region is the leader. +curl "$BASE/feed?user_id=42&profile=for_you&limit=20" +curl "$BASE/search?query=jazz%20piano&user_id=42&limit=5" -# Target a specific region (followers may lag during partitions) -curl "http://localhost:9500/feed?profile=trending®ion=eu-west" +# Pin a read to a specific region. Followers may lag the leader (and lag jumps +# during a partition) — use this for canary reads and lag verification. +curl "$BASE/feed?profile=trending®ion=eu-west" ``` -## 4. Cluster operations +`?region=` accepts any declared region name; an unknown name returns 400. Omit +it and the read serves from the current leader. `limit` is clamped to 1000 at the +trust boundary. The `for_you` profile applies the built-in diversity defaults +(`max_per_creator=2`, `format_mix_max_fraction=0.4`, `exploration=0.1`). + +## 6. Cluster management API ### Check cluster status ```bash -curl http://localhost:9500/cluster/status | jq +curl "$BASE/cluster/status" | jq ``` -Sample response: - ```json { "leader": "us-east", @@ -120,47 +275,170 @@ Sample response: } ``` +`lag_events` is `relay_log_len − applied_events` (saturating). A non-zero, +*growing* lag on a region is the signal that it is partitioned or its +segment-receiver is wedged. + ### Promote a new leader ```bash -curl -X POST http://localhost:9500/cluster/promote \ +curl -X POST "$BASE/cluster/promote" \ -H 'Content-Type: application/json' \ -d '{ "region": "eu-west" }' +# → { "ok": true, "leader": "eu-west" } ``` -`/cluster/status` will now report `eu-west` as leader. New writes are routed -there and replayed to the other regions. +After promotion `/cluster/status` reports the new leader; new writes route there +and replay to the other regions. An unknown region returns 400. ### Simulate a partition & heal ```bash -# Isolate ap-south (writes will skip this follower) -curl -X POST http://localhost:9500/cluster/partition \ +# Isolate ap-south: leader ships skip this follower, so its lag climbs. +curl -X POST "$BASE/cluster/partition" \ -H 'Content-Type: application/json' \ -d '{ "region": "ap-south" }' +# → { "ok": true, "partitioned": "ap-south" } -# Heal the partition (missed batches are replayed automatically) -curl -X POST http://localhost:9500/cluster/heal \ +# Heal: missed segments are re-shipped over gRPC (blocking, offloaded to the +# write pool — a saturated pool degrades to 429, not 500). +curl -X POST "$BASE/cluster/heal" \ -H 'Content-Type: application/json' \ -d '{ "region": "ap-south" }' +# → { "ok": true, "healed": "ap-south" } ``` -Monitor `/cluster/status` to confirm lag drops back to zero after healing. +## 7. Sharded scatter-gather API -## 5. Runbook checklist +The `/sharded/*` routes hash-partition entities across regions +(`hash(entity_id) % num_shards`, using the engine's own `ShardRouter` so writes +and reads never disagree) and fan a query out to **all** shards, K-way merging +the per-shard results by score. -1. **Startup** — launch `tidal-server cluster …` (or Docker). Confirm log line - `listening on http://…`. -2. **Baseline health** — `GET /health` and `GET /cluster/status` return `200`. -3. **Seed data** — `POST /items`, `/embeddings`, `/signals` for initial items. -4. **Traffic** — microservices call `/signals`, `/feed`, `/search`. Add `region` - query param to pin to a follower for canary reads. -5. **Failover** — to move traffic during maintenance, `POST /cluster/promote` - to the target region. Verify status before proceeding. -6. **Partition drill** — `POST /cluster/partition` to isolate a follower, - observe lag, then `POST /cluster/heal`. -7. **Shutdown** — send SIGINT (Ctrl+C) or stop the container. The server logs - `shutdown signal received` and exits cleanly. +Write routes (each writes to the single owning shard): + +```bash +curl -X POST "$BASE/sharded/items" -d '{ "entity_id": 7, "metadata": { "title": "..." } }' # 201 +curl -X POST "$BASE/sharded/embeddings" -d '{ "entity_id": 7, "values": [0.1,0.2,0.3,0.4] }' # 204 +curl -X POST "$BASE/sharded/signals" -d '{ "entity_id": 7, "signal": "view", "weight": 1.0 }' # 204 +``` + +Read routes (scatter-gather across all shards): + +```bash +curl "$BASE/sharded/feed?profile=for_you&limit=20&deadline_ms=50" +curl "$BASE/sharded/search?query=jazz&limit=10&deadline_ms=100" +``` + +Each sharded read accepts an optional `deadline_ms` — the **total** scatter +budget (default 50ms, server-clamped to 10s). The per-shard deadline is +`deadline_ms − 5ms` network overhead. The response includes a `scatter_gather` +block: + +```json +{ + "items": [ /* merged, deduped, diversity-enforced, re-ranked */ ], + "total_candidates": 4210, + "scatter_gather": { + "degraded": false, + "shards_queried": 3, + "elapsed_ms": 12, + "shard_deadline_ms": 45 + } +} +``` + +Degraded semantics: a shard that is partitioned, errors, or misses the deadline +is reported in `unavailable_shards` (a name list) and flips `degraded: true` — +it is **never silently dropped**. The merge dedups replicated copies of an entity +(keeping the best-scoring copy), reconciles `total_candidates` so replicated +shards are not counted multiple times, and re-enforces `max_per_creator` across +the merged set. + +## 8. Write-durability contract (read this before trusting a 204) + +A `204 No Content` from `/signals` (and the data writes) means **the write is +durably applied on the leader** — storage updated plus **WAL fsync** — and +nothing more. + +- The follower ship is **best-effort**. A ship that fails is logged engine-side + and queued for re-delivery via the heal / convergence path. The 204 does + **NOT** assert quorum and does **NOT** assert any follower acknowledged the + write. +- If the single process crashes after the 204 but before followers caught up, + the leader's WAL still has the write (it survives restart from disk); the + followers will reconcile on the next ship/heal. But because there is **only + one process**, a crash is total downtime, not a failover — see the status box. +- A quorum-ack write contract is explicitly future work, tracked with the + multi-process cluster effort (**m8p10**). + +In short: **204 = leader durability, not cluster durability.** Design your +client retries accordingly (the writes are idempotent on `entity_id` + signal). + +## 9. Failover drill + +1. **Baseline.** `GET /cluster/status`; confirm the expected leader and + `lag_events: 0` on every region. +2. **Pre-seed reads.** Issue a region-pinned read against the target region + (`?region=eu-west`) to confirm it is serving and roughly caught up. +3. **Promote.** `POST /cluster/promote { "region": "eu-west" }`. +4. **Verify.** `GET /cluster/status` now reports `eu-west` as leader. Send a + write (`POST /signals`) and confirm `relay_log_len` advances and the other + regions' `applied_events` follow within a heartbeat. +5. **Cut over traffic.** Point your client's writes at the same `$BASE` (the + single surface routes writes to whoever is leader) — no client change needed. + +> Reminder: this is a *leadership move within one process*, not a host failover. +> It is the right drill for "move the write region during maintenance," not for +> "survive a machine dying." + +## 10. Partition drill + +1. **Baseline.** `GET /cluster/status`; all `lag_events: 0`, all + `partitioned: false`. +2. **Inject.** `POST /cluster/partition { "region": "ap-south" }`. Confirm + `partitioned: true` for that region. +3. **Write through it.** Send several `POST /signals`. Watch `ap-south`'s + `lag_events` climb while `applied_events` stalls — the leader's ships to it + are dropped. +4. **Read the stale follower.** `GET /feed?region=ap-south` should return the + pre-partition view (demonstrating eventual, not strong, read consistency). +5. **Heal.** `POST /cluster/heal { "region": "ap-south" }`. Missed segments + re-ship over gRPC. +6. **Verify convergence.** Poll `GET /cluster/status` until `ap-south`'s + `lag_events` returns to 0 and `partitioned: false`. +7. **Scatter-gather degradation.** While partitioned, a `GET /sharded/feed` + should report `degraded: true` with `ap-south` in `unavailable_shards` — not + an error. Confirm it clears after heal. + +## 11. Shutdown + +Send `SIGTERM` (or `SIGINT` / Ctrl+C, or stop the container). The server flips +readiness to 503, drains in-flight requests, then drops the cluster — which runs +each region's `TidalDb` shutdown path: checkpoint in-memory signal state → flush +storage → write the WAL checkpoint marker + **fsync** → join the WAL, sweeper, +checkpoint, and text-syncer threads. The drop is idempotent. You will see +`cluster shutdown: closing all nodes (checkpoint + WAL fsync)`. + +## Cross-references + +- **Kubernetes deployment** — `docs/runbooks/kubernetes.md` *(planned for the + m8p10 multi-process work; not yet written — until then deploy a single + standalone node per the production guidance in the status box, and use the + Docker `HEALTHCHECK` / `/health` probe as the readiness gate).* +- **Server deployment guide** — `docs/guides/server-deployment.md` *(planned; + not yet written — see [API.md](../../API.md) and [QUICKSTART.md](../../QUICKSTART.md) + for the current server config and route reference).* +- **Monitoring & alerts** — [docs/ops/monitoring.md](../ops/monitoring.md) + (Prometheus scrape config, replication-lag and WAL metrics, recommended + alerts; remember the `/metrics` endpoint is unauthenticated — bind it + internally). +- **Roadmap / m8p10 (true multi-process HA)** — + [docs/planning/ROADMAP.md](../planning/ROADMAP.md) for the distributed-fabric + status, the m8p1–m8p9 completed phases, and the outstanding multi-process / + tier-3 chaos work. +- **API & schema reference** — [API.md](../../API.md), + [QUICKSTART.md](../../QUICKSTART.md), and the live `/openapi.json` document. +- **Scope & vision** — [VISION.md](../../VISION.md). +``` -Refer to `docs/planning/ROADMAP.md` for the underlying distributed -fabric guarantees and property tests. diff --git a/docs/runbooks/kubernetes.md b/docs/runbooks/kubernetes.md new file mode 100644 index 0000000..dcf03ba --- /dev/null +++ b/docs/runbooks/kubernetes.md @@ -0,0 +1,206 @@ +# tidalDB on Kubernetes + +How to run tidalDB on Kubernetes: a hardened, single-node **standalone** +deployment as a `StatefulSet` with a durable volume, the three health probes, +metrics, secret-backed auth, and graceful rolling updates. The manifests live in +[`k8s/`](../../k8s/) and apply with `kubectl apply -k k8s/`. + +> ## Deploy the STANDALONE server, one replica +> +> tidalDB is single-node-first: the server wraps one embedded engine whose state +> (WAL + checkpoints + indexes) lives on a data dir. It scales **vertically** +> (a bigger pod), not by adding replicas — there is no shared-storage multi-writer +> mode, so `replicas: 1` in the StatefulSet is load-bearing. The experimental +> multi-region `cluster` subcommand runs every region inside **one process** over +> loopback gRPC ([NOT production HA](cluster.md)), so it does not make this an HA +> story either. Run one standalone pod, back it with a durable `PersistentVolume`, +> and recover from the WAL on restart (see [recovery](../ops/recovery.md)). True +> multi-process HA is tracked as **m8p10** in +> [ROADMAP.md](../planning/ROADMAP.md). + +## What's in `k8s/` + +| File | Purpose | +|------|---------| +| `namespace.yaml` | The `tidaldb` namespace | +| `schema-configmap.yaml` | The schema YAML the server loads (`--schema`); edit for your signals | +| `statefulset.yaml` | The server: durable PVC, probes, security context, resources | +| `service.yaml` | Headless `Service` for stable DNS + in-cluster clients | +| `poddisruptionbudget.yaml` | `maxUnavailable: 0` — a drain can't silently kill the single node | +| `secret.example.yaml` | Template for the API-key secret (create the real one out-of-band) | +| `servicemonitor.yaml` | Optional Prometheus-Operator scrape config (apply separately) | +| `kustomization.yaml` | Ties the core resources together for `kubectl apply -k` | + +## Prerequisites + +- A Kubernetes cluster (1.25+) and `kubectl` pointed at it. For local testing, + [`kind`](https://kind.sigs.k8s.io/) is used in the walkthrough below. +- A container registry the cluster can pull from (for real clusters), or a local + image loaded into the node (for `kind`). The image is built from + [`docker/deploy/Dockerfile`](../../docker/deploy/Dockerfile). +- A default `StorageClass` (for dynamic `PersistentVolumeClaim` provisioning). + `kind`, GKE, EKS, and AKS all ship one. + +## Deploy + +### 1. Build and publish the image + +```bash +# From the repo root — the build context must be the workspace root. +docker build -f docker/deploy/Dockerfile -t /tidaldb: . +docker push /tidaldb: +``` + +Set that reference in `k8s/statefulset.yaml` (`image:`), pinned by digest in +production (`@sha256:...`). + +### 2. Create the namespace and the API-key secret + +The secret is deliberately **not** in the kustomization so no key lands in git. +Create it directly: + +```bash +kubectl create namespace tidaldb +kubectl -n tidaldb create secret generic tidaldb-api-key \ + --from-literal=api-key="$(openssl rand -hex 32)" +``` + +In production, manage it with External Secrets Operator, Sealed Secrets, or +Vault Agent instead. The StatefulSet injects it as `TIDAL_API_KEY` — clients +then send `Authorization: Bearer ` on every data route. **If the secret is +empty the server runs unauthenticated and logs a WARN — never do that on a +shared network.** + +### 3. Edit the schema (optional) + +`k8s/schema-configmap.yaml` carries the schema the server loads. Edit it to model +your signals, text fields, embedding slots, and (optionally) ranking profiles — +the format is documented in [server-deployment.md](../guides/server-deployment.md). +The schema is read once at boot; roll the StatefulSet to apply changes. + +### 4. Apply + +```bash +kubectl apply -k k8s/ +kubectl -n tidaldb rollout status statefulset/tidaldb --timeout=180s +``` + +### 5. Verify + +```bash +kubectl -n tidaldb port-forward statefulset/tidaldb 9400:9400 & +KEY=$(kubectl -n tidaldb get secret tidaldb-api-key -o jsonpath='{.data.api-key}' | base64 -d) + +curl -s localhost:9400/health # {"ok":true,...} +curl -s localhost:9400/openapi.json | jq .info # served API contract +curl -s -X POST localhost:9400/items \ + -H "Authorization: Bearer $KEY" -H 'Content-Type: application/json' \ + -d '{"entity_id":1,"metadata":{"title":"hello","category":"demo","created_at":"1700000000"}}' +curl -s -H "Authorization: Bearer $KEY" "localhost:9400/feed?profile=trending&limit=5" +``` + +## How the health probes map + +The server exposes three unauthenticated endpoints, wired to the three probe +types in `statefulset.yaml`: + +| Probe | Endpoint | Behavior | +|-------|----------|----------| +| `startupProbe` | `GET /health/startup` | 200 once the HTTP listener is up; high `failureThreshold` covers slow WAL replay / index load on large data dirs (see [capacity-planning](../ops/capacity-planning.md)) | +| `livenessProbe` | `GET /health/live` | 200 while the process is alive; restart if it stops answering | +| `readinessProbe` | `GET /health` | 200 ready / **503 while draining** — on SIGTERM the pod leaves the Service endpoints before it stops accepting | + +## Rolling updates and graceful shutdown + +On `kubectl rollout restart` (or any pod delete), the kubelet sends SIGTERM. The +server flips readiness to 503 (so it leaves the Service), drains in-flight +requests, then checkpoints and fsyncs the WAL before exit. +`terminationGracePeriodSeconds: 60` gives that room — raise it if your data dir +is large. Because there is one replica, a restart is a brief planned outage +while the new pod replays the WAL; the `PodDisruptionBudget` (`maxUnavailable: +0`) prevents an *involuntary* drain from taking the node down without operator +intent. + +## Persistence and backup + +The `volumeClaimTemplate` provisions a `PersistentVolumeClaim` (`/data`, 10Gi by +default — size it from [capacity-planning](../ops/capacity-planning.md)). The WAL ++ checkpoints there are the source of truth and survive pod restarts. For backup +and disaster recovery (snapshotting the PVC, restoring a corrupt data dir), see +[recovery](../ops/recovery.md). + +## Metrics + +The pod exposes Prometheus metrics on `:9091/metrics` (**unauthenticated** — it is +not exposed by the headless Service externally; keep it cluster-internal). Scrape +it one of two ways: + +- **Prometheus Operator:** `kubectl apply -f k8s/servicemonitor.yaml` (requires the + `monitoring.coreos.com` CRDs). +- **Plain Prometheus:** the pod carries `prometheus.io/scrape`, `prometheus.io/port`, + and `prometheus.io/path` annotations. + +Alert rules and a dashboard ship in [`docs/ops/prometheus-alerts.yaml`](../ops/prometheus-alerts.yaml) +and [`docs/ops/grafana-dashboard.json`](../ops/grafana-dashboard.json); see +[monitoring](../ops/monitoring.md). + +## Local walkthrough with `kind` + +This is the exact flow used to verify the manifests end-to-end: + +```bash +# 1. Create a local cluster. +kind create cluster --name tidaldb + +# 2. Build the image and load it into the kind node (no registry needed). +docker build -f docker/deploy/Dockerfile -t tidaldb:deploy . +kind load docker-image tidaldb:deploy --name tidaldb + +# 3. Namespace + API-key secret. +kubectl create namespace tidaldb +kubectl -n tidaldb create secret generic tidaldb-api-key \ + --from-literal=api-key="$(openssl rand -hex 32)" + +# 4. Apply and wait for ready. +kubectl apply -k k8s/ +kubectl -n tidaldb rollout status statefulset/tidaldb --timeout=240s + +# 5. Verify, then tear down. +kubectl -n tidaldb port-forward statefulset/tidaldb 9400:9400 & +curl -s localhost:9400/health +kind delete cluster --name tidaldb +``` + +The manifests set `image: tidaldb:deploy` with `imagePullPolicy: IfNotPresent`, +which is exactly what `kind load` + a local tag need. For a real cluster, swap in +your registry image. + +> **`kind create cluster` fails with "could not find a log line that matches … +> Multi-User System"?** On Docker Desktop the node's `systemd` can die at boot +> with `Failed to create control group inotify object: Too many open files` +> (`docker logs -control-plane` shows it). The Docker VM's inotify +> limits are too low; raise them in the VM kernel, then recreate: +> +> ```bash +> docker run --rm --privileged alpine sysctl -w fs.inotify.max_user_instances=8192 +> kind delete cluster --name tidaldb && kind create cluster --name tidaldb +> ``` +> +> This is a kind-on-Docker-Desktop prerequisite, unrelated to tidalDB. + +## Troubleshooting + +| Symptom | Likely cause | +|---------|--------------| +| Pod `Pending` | No default `StorageClass`, or the PVC can't bind — `kubectl -n tidaldb describe pvc data-tidaldb-0` | +| Pod `CrashLoopBackOff` at boot | Bad schema YAML in the ConfigMap, or a data dir from an incompatible schema — check logs; see [recovery § schema mismatch](../ops/recovery.md) | +| Pod never `Ready`, but `Running` | Readiness probe failing — `kubectl -n tidaldb logs statefulset/tidaldb`; a large data dir may need a longer `startupProbe` | +| `401 Unauthorized` on data routes | Wrong/empty `tidaldb-api-key` secret; clients must send `Authorization: Bearer ` | +| Writes lost after restart | Data dir not on the PVC — confirm `--data-dir /data` and the `data` volume mount | + +## See also + +- [Server deployment guide](../guides/server-deployment.md) — config, auth, the served OpenAPI spec +- [Build a feed app](../guides/build-a-feed-app.md) — what to run against this server +- [Cluster runbook](cluster.md) — the experimental multi-region mode (not production HA) +- [Monitoring](../ops/monitoring.md) · [Capacity planning](../ops/capacity-planning.md) · [Recovery](../ops/recovery.md) diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml new file mode 100644 index 0000000..2478942 --- /dev/null +++ b/k8s/kustomization.yaml @@ -0,0 +1,26 @@ +# Core, cluster-agnostic tidalDB deployment. Apply with: +# kubectl apply -k k8s/ +# +# Create the API-key secret FIRST (it is deliberately excluded here so no key +# is committed — see secret.example.yaml): +# kubectl -n tidaldb create secret generic tidaldb-api-key \ +# --from-literal=api-key="$(openssl rand -hex 32)" +# +# servicemonitor.yaml is also excluded (needs the Prometheus Operator); apply it +# separately if you run the Operator. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: tidaldb + +resources: + - namespace.yaml + - schema-configmap.yaml + - service.yaml + - statefulset.yaml + - poddisruptionbudget.yaml + +labels: + - pairs: + app.kubernetes.io/part-of: tidaldb + includeSelectors: false diff --git a/k8s/namespace.yaml b/k8s/namespace.yaml new file mode 100644 index 0000000..7f23e93 --- /dev/null +++ b/k8s/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: tidaldb + labels: + app.kubernetes.io/name: tidaldb diff --git a/k8s/poddisruptionbudget.yaml b/k8s/poddisruptionbudget.yaml new file mode 100644 index 0000000..cbaa3c8 --- /dev/null +++ b/k8s/poddisruptionbudget.yaml @@ -0,0 +1,20 @@ +# A single-node DB cannot stay available across a voluntary disruption (node +# drain / autoscaler scale-down) — there is no second replica to serve while +# the pod reschedules. This PDB makes that explicit: `maxUnavailable: 0` blocks +# VOLUNTARY evictions so a drain cannot kill tidalDB without an operator first +# scaling it down or accepting the downtime. It does NOT protect against +# involuntary loss (node crash) — for that you need durable PVC storage + the +# recovery path in docs/ops/recovery.md. Zero-downtime requires true +# multi-process HA, which is not yet available (tracked as m8p10). +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: tidaldb + namespace: tidaldb + labels: + app.kubernetes.io/name: tidaldb +spec: + maxUnavailable: 0 + selector: + matchLabels: + app.kubernetes.io/name: tidaldb diff --git a/k8s/schema-configmap.yaml b/k8s/schema-configmap.yaml new file mode 100644 index 0000000..61c585b --- /dev/null +++ b/k8s/schema-configmap.yaml @@ -0,0 +1,50 @@ +# The tidalDB schema (signal types + decay, text fields, embedding slots) the +# server loads at startup with `--schema`. Mounting it as a ConfigMap lets you +# evolve the schema without rebuilding the image. This mirrors the in-image +# tidal-server/config/default-schema.yaml; edit it to model YOUR signals. +# +# Schema format reference: docs/guides/server-deployment.md and API.md. +# NOTE: editing this ConfigMap does NOT hot-reload a running pod — the schema is +# read once at boot. Roll the StatefulSet (kubectl rollout restart) to apply +# changes, and only make backward-compatible schema changes on a populated +# data dir (see docs/ops/recovery.md § schema mismatch). +apiVersion: v1 +kind: ConfigMap +metadata: + name: tidaldb-schema + namespace: tidaldb + labels: + app.kubernetes.io/name: tidaldb +data: + schema.yaml: | + signals: + - name: view + entity: item + decay: + exponential: + half_life_seconds: 604800 # 7 days + windows: [one_hour, twenty_four_hours, seven_days] + velocity: true + positive_engagement: true # folds into the user preference vector + - name: like + entity: item + decay: + exponential: + half_life_seconds: 1209600 # 14 days + windows: [twenty_four_hours, seven_days, thirty_days, all_time] + velocity: false + positive_engagement: true + - name: skip + entity: item + decay: + permanent: true + velocity: false + text_fields: + - name: title + kind: text + - name: category + kind: keyword + embedding_slots: + - name: content_vector + entity: item + dimensions: 128 diff --git a/k8s/secret.example.yaml b/k8s/secret.example.yaml new file mode 100644 index 0000000..547416e --- /dev/null +++ b/k8s/secret.example.yaml @@ -0,0 +1,25 @@ +# EXAMPLE ONLY — do not apply this as-is and do not commit a real key. +# +# The server reads TIDAL_API_KEY and requires `Authorization: Bearer ` on +# every data route when it is set. If the key is empty/unset the server runs +# UNAUTHENTICATED and logs a loud startup WARN — never do that on a real network. +# +# This file is intentionally LEFT OUT of kustomization.yaml. Create the real +# secret out-of-band so the key never lands in git: +# +# kubectl -n tidaldb create secret generic tidaldb-api-key \ +# --from-literal=api-key="$(openssl rand -hex 32)" +# +# or manage it with your secret store (External Secrets Operator, Sealed +# Secrets, Vault Agent, etc.). The StatefulSet mounts key `api-key` from the +# secret named `tidaldb-api-key`. +apiVersion: v1 +kind: Secret +metadata: + name: tidaldb-api-key + namespace: tidaldb + labels: + app.kubernetes.io/name: tidaldb +type: Opaque +stringData: + api-key: "CHANGE-ME-use-a-32-byte-random-hex-string" diff --git a/k8s/service.yaml b/k8s/service.yaml new file mode 100644 index 0000000..aee074b --- /dev/null +++ b/k8s/service.yaml @@ -0,0 +1,24 @@ +# Headless Service: gives the StatefulSet pod a stable DNS name +# (tidaldb-0.tidaldb.tidaldb.svc.cluster.local) and backs in-cluster clients. +# Headless (clusterIP: None) is the right default for a single stateful pod; +# readiness gating still applies, so the pod drops out of DNS A-records while +# draining. Front it with an Ingress/Gateway for external traffic — do NOT +# expose the metrics port (9091) outside the cluster (it is unauthenticated). +apiVersion: v1 +kind: Service +metadata: + name: tidaldb + namespace: tidaldb + labels: + app.kubernetes.io/name: tidaldb +spec: + clusterIP: None + selector: + app.kubernetes.io/name: tidaldb + ports: + - name: http + port: 9400 + targetPort: http + - name: metrics + port: 9091 + targetPort: metrics diff --git a/k8s/servicemonitor.yaml b/k8s/servicemonitor.yaml new file mode 100644 index 0000000..06d322e --- /dev/null +++ b/k8s/servicemonitor.yaml @@ -0,0 +1,21 @@ +# OPTIONAL — requires the Prometheus Operator (monitoring.coreos.com CRDs). +# It is intentionally NOT in kustomization.yaml so `kubectl apply -k k8s/` works +# on a vanilla cluster. Apply it separately only if you run the Operator: +# kubectl apply -f k8s/servicemonitor.yaml +# Without the Operator, use the pod's prometheus.io/* scrape annotations instead. +# Alert rules + dashboard: docs/ops/prometheus-alerts.yaml, docs/ops/grafana-dashboard.json. +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: tidaldb + namespace: tidaldb + labels: + app.kubernetes.io/name: tidaldb +spec: + selector: + matchLabels: + app.kubernetes.io/name: tidaldb + endpoints: + - port: metrics + path: /metrics + interval: 30s diff --git a/k8s/statefulset.yaml b/k8s/statefulset.yaml new file mode 100644 index 0000000..ba35de5 --- /dev/null +++ b/k8s/statefulset.yaml @@ -0,0 +1,142 @@ +# tidalDB standalone server as a StatefulSet. +# +# WHY A STATEFULSET (not a Deployment): tidalDB is single-node-first and +# embeddable — the server wraps ONE engine instance whose state (WAL + +# checkpoints + indexes) lives on a durable data dir. It scales VERTICALLY +# (bigger node), not by adding replicas. `replicas: 1` is intentional and load- +# bearing: there is no shared-storage multi-writer mode. The experimental +# multi-region `cluster` subcommand runs every region in ONE process (no host +# isolation — NOT production HA, tracked as m8p10), so it does not change this. +# See docs/runbooks/kubernetes.md and docs/runbooks/cluster.md. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: tidaldb + namespace: tidaldb + labels: + app.kubernetes.io/name: tidaldb + app.kubernetes.io/component: server +spec: + serviceName: tidaldb # the headless Service in service.yaml — stable network id + replicas: 1 # single-node-first: scale up, not out (see header) + selector: + matchLabels: + app.kubernetes.io/name: tidaldb + template: + metadata: + labels: + app.kubernetes.io/name: tidaldb + app.kubernetes.io/component: server + annotations: + # Plain-Prometheus scrape hints (the PodMonitor/ServiceMonitor in + # servicemonitor.yaml is the Operator-native alternative). Metrics are + # unauthenticated — keep :9091 cluster-internal (see ops/monitoring.md). + prometheus.io/scrape: "true" + prometheus.io/port: "9091" + prometheus.io/path: "/metrics" + spec: + # SIGTERM flips readiness to 503 (pod leaves Endpoints), drains in-flight + # requests, then checkpoints + fsyncs the WAL before exit. Give that room. + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 10001 # the `tidal` user baked into docker/deploy/Dockerfile + runAsGroup: 10001 + fsGroup: 10001 # makes the mounted PVC group-writable by the runtime user + seccompProfile: + type: RuntimeDefault + containers: + - name: tidaldb + # For a real cluster, replace with your registry image pinned by digest + # (e.g. registry.example.com/tidaldb@sha256:...) and set + # imagePullPolicy: IfNotPresent. `tidaldb:deploy` is the local image + # built from docker/deploy/Dockerfile and loaded via `kind load`. + image: tidaldb:deploy + imagePullPolicy: IfNotPresent + # ENTRYPOINT is the bare binary; these args override the image CMD so + # the schema comes from the mounted ConfigMap, not the baked default. + args: + - standalone + - --listen + - 0.0.0.0:9400 + - --schema + - /etc/tidaldb/schema/schema.yaml + - --data-dir + - /data + - --metrics + - 0.0.0.0:9091 + env: + - name: TIDAL_API_KEY + valueFrom: + secretKeyRef: + name: tidaldb-api-key + key: api-key + - name: TIDAL_SERVER_LOG + value: info + ports: + - name: http + containerPort: 9400 + - name: metrics + containerPort: 9091 + # Three distinct probes map to the three health endpoints: + # - /health/startup : always 200 once the HTTP listener is up + # - /health/live : always 200 while the process is alive + # - /health : 200 ready / 503 while draining on SIGTERM + # All are unauthenticated by design, so the probes need no token. + startupProbe: + httpGet: + path: /health/startup + port: http + periodSeconds: 5 + failureThreshold: 60 # up to ~5 min for large-DB WAL replay / index load (capacity-planning.md) + livenessProbe: + httpGet: + path: /health/live + port: http + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health # 503 during drain -> removed from Service Endpoints + port: http + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + resources: + requests: + cpu: "250m" + memory: 256Mi + limits: + cpu: "2" + memory: 2Gi # size from docs/ops/capacity-planning.md for your item/embedding count + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true # the server only writes /data (PVC) and /tmp (emptyDir) + capabilities: + drop: ["ALL"] + volumeMounts: + - name: data + mountPath: /data + - name: schema + mountPath: /etc/tidaldb/schema + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: schema + configMap: + name: tidaldb-schema + - name: tmp + emptyDir: {} + volumeClaimTemplates: + - metadata: + name: data + labels: + app.kubernetes.io/name: tidaldb + spec: + accessModes: ["ReadWriteOnce"] + # storageClassName: "" # uncomment + set to pin a class; omitted = cluster default + resources: + requests: + storage: 10Gi diff --git a/scripts/check-docs.sh b/scripts/check-docs.sh index a6c40ce..7ac5616 100755 --- a/scripts/check-docs.sh +++ b/scripts/check-docs.sh @@ -35,6 +35,17 @@ for p in tidal/docs tidal/ai-lookup tidal/site .ai .agents/skills; do done [ "$mirror_hit" -eq 0 ] && ok "no doc-mirror trees present" +# --------------------------------------------------------------------------- +# 1b. No per-crate docker mirror. Container images live only at the repo-root +# docker/ (what .woodpecker.yaml builds); tidal/docker/ was a drifting +# duplicate and was consolidated away. +# --------------------------------------------------------------------------- +if [ -e "tidal/docker" ] || git ls-files --error-unmatch "tidal/docker" >/dev/null 2>&1; then + err "docker mirror reappeared: 'tidal/docker' — Dockerfiles live only at repo-root docker/ (build from the repo root so COPY . . sees the workspace)" +else + ok "no docker-mirror tree present" +fi + # --------------------------------------------------------------------------- # 2. CLAUDE.md Repository Structure must list every sibling workspace crate. # --------------------------------------------------------------------------- diff --git a/tidal-server/Cargo.toml b/tidal-server/Cargo.toml index 4f4404a..e8ca918 100644 --- a/tidal-server/Cargo.toml +++ b/tidal-server/Cargo.toml @@ -48,6 +48,12 @@ thiserror = "2" tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal", "sync"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } +# utoipa 5.x derives the OpenAPI 3.1 document (ApiDoc) and per-handler path +# attributes that back the unauthenticated GET /openapi.json route. JSON-only: +# no swagger-ui asset bundle (that crate's vendored JS does not pass our +# -D warnings posture). The `axum_extras` feature is the axum 0.8-compatible +# integration surface. +utoipa = { version = "5", features = ["axum_extras"] } tidaldb = { path = "../tidal", features = ["test-utils"] } tidal-net = { path = "../tidal-net" } diff --git a/tidal-server/config/default-schema.yaml b/tidal-server/config/default-schema.yaml index 3ad7f83..48afbb9 100644 --- a/tidal-server/config/default-schema.yaml +++ b/tidal-server/config/default-schema.yaml @@ -6,6 +6,7 @@ signals: half_life_seconds: 604800 # 7 days windows: [one_hour, twenty_four_hours, seven_days] velocity: true + positive_engagement: true # folds into the user preference vector - name: like entity: item decay: @@ -13,6 +14,7 @@ signals: half_life_seconds: 1209600 # 14 days windows: [twenty_four_hours, seven_days, thirty_days, all_time] velocity: false + positive_engagement: true - name: skip entity: item decay: diff --git a/tidal-server/src/cluster.rs b/tidal-server/src/cluster.rs index 738528e..b3ce83e 100644 --- a/tidal-server/src/cluster.rs +++ b/tidal-server/src/cluster.rs @@ -60,6 +60,7 @@ use tidaldb::{ }; use tower::{ServiceBuilder, limit::ConcurrencyLimitLayer}; use tower_http::timeout::TimeoutLayer; +use utoipa::ToSchema; use crate::{ dto::{ @@ -596,6 +597,9 @@ pub fn build_cluster_router(state: Arc, api_key: Option>) .route("/health/startup", get(crate::health::health_startup)) .route("/health/live", get(crate::health::health_live)) .route("/cluster/status", get(cluster_status)) + // Cluster-superset OpenAPI document (adds the /cluster/* and /sharded/* + // routes). Unauthenticated, like the probes — contract only, no data. + .route("/openapi.json", get(crate::openapi::serve_cluster)) .with_state(Arc::clone(&state)); let protected = Router::new() @@ -687,22 +691,39 @@ async fn cluster_health( // ── Cluster management ───────────────────────────────────────────────────── -#[derive(Serialize)] -struct ClusterStatusResponse { +/// `GET /cluster/status` response body. +#[derive(Serialize, ToSchema)] +pub(crate) struct ClusterStatusResponse { + /// Current leader region name. leader: String, + /// Length of the leader's replication relay log. relay_log_len: u64, + /// Per-region replication status. regions: Vec, } -#[derive(Serialize)] -struct RegionStatus { +/// One region's replication status within a [`ClusterStatusResponse`]. +#[derive(Serialize, ToSchema)] +pub(crate) struct RegionStatus { + /// Region name. name: String, + /// Replication events applied on this region. applied_events: u64, + /// Events the region lags the leader by. lag_events: u64, + /// Whether this region is currently partitioned from the leader. partitioned: bool, } -async fn cluster_status( +#[utoipa::path( + get, + path = "/cluster/status", + tag = "cluster", + responses( + (status = 200, description = "Cluster replication status", body = ClusterStatusResponse), + ), +)] +pub(crate) async fn cluster_status( State(state): State>, ) -> std::result::Result, ClusterAppError> { let cluster = state.cluster().map_err(ClusterAppError)?; @@ -731,12 +752,28 @@ async fn cluster_status( })) } -#[derive(Deserialize)] -struct RegionRequest { +/// Body for the cluster region-management routes (`/cluster/promote`, +/// `/cluster/partition`, `/cluster/heal`). +#[derive(Deserialize, ToSchema)] +pub(crate) struct RegionRequest { + /// Target region name. + #[schema(example = "us-east")] region: String, } -async fn cluster_promote( +#[utoipa::path( + post, + path = "/cluster/promote", + tag = "cluster", + request_body = RegionRequest, + responses( + (status = 200, description = "Region promoted to leader"), + (status = 400, description = "Unknown region"), + (status = 401, description = "Missing or invalid API key"), + ), + security(("bearerAuth" = [])), +)] +pub(crate) async fn cluster_promote( State(state): State>, Json(req): Json, ) -> std::result::Result, ClusterAppError> { @@ -751,7 +788,19 @@ async fn cluster_promote( }))) } -async fn cluster_partition( +#[utoipa::path( + post, + path = "/cluster/partition", + tag = "cluster", + request_body = RegionRequest, + responses( + (status = 200, description = "Region partitioned from the leader"), + (status = 400, description = "Unknown region"), + (status = 401, description = "Missing or invalid API key"), + ), + security(("bearerAuth" = [])), +)] +pub(crate) async fn cluster_partition( State(state): State>, Json(req): Json, ) -> std::result::Result, ClusterAppError> { @@ -766,7 +815,20 @@ async fn cluster_partition( }))) } -async fn cluster_heal( +#[utoipa::path( + post, + path = "/cluster/heal", + tag = "cluster", + request_body = RegionRequest, + responses( + (status = 200, description = "Region healed (missed segments re-shipped)"), + (status = 400, description = "Unknown region"), + (status = 401, description = "Missing or invalid API key"), + (status = 429, description = "Write pool saturated"), + ), + security(("bearerAuth" = [])), +)] +pub(crate) async fn cluster_heal( State(state): State>, Json(req): Json, ) -> std::result::Result, ClusterAppError> { @@ -790,7 +852,19 @@ async fn cluster_heal( // ── Data routes (cluster mode) ───────────────────────────────────────────── -async fn create_item( +#[utoipa::path( + post, + path = "/items", + tag = "data", + request_body = ItemRequest, + responses( + (status = 201, description = "Item created on the leader"), + (status = 400, description = "Invalid request"), + (status = 401, description = "Missing or invalid API key"), + ), + security(("bearerAuth" = [])), +)] +pub(crate) async fn create_item( State(state): State>, Json(req): Json, ) -> std::result::Result { @@ -802,7 +876,19 @@ async fn create_item( Ok(StatusCode::CREATED) } -async fn write_embedding( +#[utoipa::path( + post, + path = "/embeddings", + tag = "data", + request_body = EmbeddingRequest, + responses( + (status = 204, description = "Embedding written on the leader"), + (status = 400, description = "Invalid request"), + (status = 401, description = "Missing or invalid API key"), + ), + security(("bearerAuth" = [])), +)] +pub(crate) async fn write_embedding( State(state): State>, Json(req): Json, ) -> std::result::Result { @@ -822,7 +908,20 @@ async fn write_embedding( /// `heal_region`, but the 204 does NOT assert quorum or follower acknowledgement /// — it asserts leader durability only. A future quorum-ack contract is tracked /// with the broader multi-process cluster work (m8p10). -async fn write_signal( +#[utoipa::path( + post, + path = "/signals", + tag = "data", + request_body = SignalRequest, + responses( + (status = 204, description = "Signal durably applied on the leader"), + (status = 400, description = "Invalid request"), + (status = 401, description = "Missing or invalid API key"), + (status = 429, description = "Write pool saturated"), + ), + security(("bearerAuth" = [])), +)] +pub(crate) async fn write_signal( State(state): State>, Json(req): Json, ) -> std::result::Result { @@ -846,7 +945,19 @@ async fn write_signal( Ok(StatusCode::NO_CONTENT) } -async fn feed( +#[utoipa::path( + get, + path = "/feed", + tag = "data", + params(FeedQuery), + responses( + (status = 200, description = "Ranked feed from the target region", body = FeedResponse), + (status = 400, description = "Unknown region or invalid request"), + (status = 401, description = "Missing or invalid API key"), + ), + security(("bearerAuth" = [])), +)] +pub(crate) async fn feed( State(state): State>, Query(query): Query, ) -> std::result::Result, ClusterAppError> { @@ -885,7 +996,19 @@ async fn feed( })) } -async fn search( +#[utoipa::path( + get, + path = "/search", + tag = "data", + params(SearchQueryParams), + responses( + (status = 200, description = "Ranked search from the target region", body = SearchResponse), + (status = 400, description = "Unknown region or invalid request"), + (status = 401, description = "Missing or invalid API key"), + ), + security(("bearerAuth" = [])), +)] +pub(crate) async fn search( State(state): State>, Query(query): Query, ) -> std::result::Result, ClusterAppError> { @@ -932,7 +1055,19 @@ async fn search( // ── Sharded (scatter-gather) routes ───────────────────────────────────────── -async fn sharded_create_item( +#[utoipa::path( + post, + path = "/sharded/items", + tag = "sharded", + request_body = ItemRequest, + responses( + (status = 201, description = "Item written to its owning shard"), + (status = 400, description = "Invalid request"), + (status = 401, description = "Missing or invalid API key"), + ), + security(("bearerAuth" = [])), +)] +pub(crate) async fn sharded_create_item( State(state): State>, Json(req): Json, ) -> std::result::Result { @@ -952,7 +1087,19 @@ async fn sharded_create_item( Ok(StatusCode::CREATED) } -async fn sharded_write_embedding( +#[utoipa::path( + post, + path = "/sharded/embeddings", + tag = "sharded", + request_body = EmbeddingRequest, + responses( + (status = 204, description = "Embedding written to its owning shard"), + (status = 400, description = "Invalid request"), + (status = 401, description = "Missing or invalid API key"), + ), + security(("bearerAuth" = [])), +)] +pub(crate) async fn sharded_write_embedding( State(state): State>, Json(req): Json, ) -> std::result::Result { @@ -969,7 +1116,19 @@ async fn sharded_write_embedding( Ok(StatusCode::NO_CONTENT) } -async fn sharded_write_signal( +#[utoipa::path( + post, + path = "/sharded/signals", + tag = "sharded", + request_body = SignalRequest, + responses( + (status = 204, description = "Signal written to its owning shard"), + (status = 400, description = "Invalid request"), + (status = 401, description = "Missing or invalid API key"), + ), + security(("bearerAuth" = [])), +)] +pub(crate) async fn sharded_write_signal( State(state): State>, Json(req): Json, ) -> std::result::Result { @@ -987,32 +1146,50 @@ async fn sharded_write_signal( Ok(StatusCode::NO_CONTENT) } -#[derive(Deserialize)] -struct ShardedFeedQuery { +/// `GET /sharded/feed` query parameters. +#[derive(Deserialize, ToSchema, utoipa::IntoParams)] +#[into_params(parameter_in = Query)] +pub(crate) struct ShardedFeedQuery { + /// Optional user to personalize ranking for. #[serde(default)] user_id: Option, + /// Named ranking profile; defaults to `for_you`. #[serde(default = "default_profile")] + #[param(example = "for_you")] profile: String, + /// Page size; clamped to `MAX_LIMIT` (1000). #[serde(default = "default_limit")] + #[param(example = 20)] limit: u32, + /// Per-shard deadline in milliseconds for the scatter-gather. #[serde(default)] deadline_ms: Option, } -#[derive(Serialize)] -struct ShardedFeedResponse { +/// `GET /sharded/feed` response body. +#[derive(Serialize, ToSchema)] +pub(crate) struct ShardedFeedResponse { + /// Merged ranked items across all shards. items: Vec, + /// Total candidates considered across shards. total_candidates: usize, + /// Scatter-gather execution metadata. scatter_gather: ScatterGatherInfo, } -#[derive(Serialize)] -struct ScatterGatherInfo { +/// Scatter-gather execution metadata returned with sharded responses. +#[derive(Serialize, ToSchema)] +pub(crate) struct ScatterGatherInfo { + /// True when one or more shards were unavailable and results are partial. degraded: bool, + /// Names of shards that did not respond within the deadline. #[serde(skip_serializing_if = "Vec::is_empty")] unavailable_shards: Vec, + /// Number of shards queried. shards_queried: usize, + /// Wall-clock time the scatter-gather took, in milliseconds. elapsed_ms: u64, + /// Per-shard deadline that was applied, in milliseconds. shard_deadline_ms: u64, } @@ -1031,7 +1208,19 @@ impl From for ScatterGatherInfo { } } -async fn sharded_feed( +#[utoipa::path( + get, + path = "/sharded/feed", + tag = "sharded", + params(ShardedFeedQuery), + responses( + (status = 200, description = "Scatter-gather ranked feed", body = ShardedFeedResponse), + (status = 400, description = "Invalid request"), + (status = 401, description = "Missing or invalid API key"), + ), + security(("bearerAuth" = [])), +)] +pub(crate) async fn sharded_feed( State(state): State>, Query(query): Query, ) -> std::result::Result, ClusterAppError> { @@ -1073,25 +1262,49 @@ async fn sharded_feed( })) } -#[derive(Deserialize)] -struct ShardedSearchQuery { +/// `GET /sharded/search` query parameters. +#[derive(Deserialize, ToSchema, utoipa::IntoParams)] +#[into_params(parameter_in = Query)] +pub(crate) struct ShardedSearchQuery { + /// Free-text query string. + #[param(example = "jazz piano")] query: String, + /// Optional user to personalize ranking for. #[serde(default)] user_id: Option, + /// Page size; clamped to `MAX_LIMIT` (1000). #[serde(default = "default_limit")] + #[param(example = 20)] limit: u32, + /// Per-shard deadline in milliseconds for the scatter-gather. #[serde(default)] deadline_ms: Option, } -#[derive(Serialize)] -struct ShardedSearchResponse { +/// `GET /sharded/search` response body. +#[derive(Serialize, ToSchema)] +pub(crate) struct ShardedSearchResponse { + /// Merged ranked search results across all shards. items: Vec, + /// Total candidates considered across shards. total_candidates: usize, + /// Scatter-gather execution metadata. scatter_gather: ScatterGatherInfo, } -async fn sharded_search( +#[utoipa::path( + get, + path = "/sharded/search", + tag = "sharded", + params(ShardedSearchQuery), + responses( + (status = 200, description = "Scatter-gather ranked search", body = ShardedSearchResponse), + (status = 400, description = "Invalid request"), + (status = 401, description = "Missing or invalid API key"), + ), + security(("bearerAuth" = [])), +)] +pub(crate) async fn sharded_search( State(state): State>, Query(query): Query, ) -> std::result::Result, ClusterAppError> { @@ -1155,7 +1368,7 @@ where // ── Error handling ───────────────────────────────────────────────────────── -struct ClusterAppError(ServerError); +pub(crate) struct ClusterAppError(ServerError); impl IntoResponse for ClusterAppError { fn into_response(self) -> Response { diff --git a/tidal-server/src/config.rs b/tidal-server/src/config.rs index e815bdc..16cc4b4 100644 --- a/tidal-server/src/config.rs +++ b/tidal-server/src/config.rs @@ -92,6 +92,9 @@ pub fn load_schema(path: Option<&Path>) -> Result<(Schema, Vec)> if let Some(velocity) = signal.velocity { sig = sig.velocity(velocity); } + if let Some(positive_engagement) = signal.positive_engagement { + sig = sig.positive_engagement(positive_engagement); + } let _ = sig.add(); } @@ -156,6 +159,13 @@ struct SignalSpec { windows: Option>, #[serde(default)] velocity: Option, + /// Whether engaging with an item via this signal should fold the item's + /// embedding into the user's preference vector (personalization). Mirrors + /// `SignalBuilder::positive_engagement`. When omitted, the engine falls back + /// to name-based detection; declare it explicitly for predictable taste + /// updates (see docs/guides/build-a-feed-app.md). + #[serde(default)] + positive_engagement: Option, } #[derive(Debug, Deserialize)] @@ -693,6 +703,30 @@ profiles: assert_eq!(profile.diversity.max_per_creator, Some(2)); } + #[test] + fn parse_signal_positive_engagement() { + // `positive_engagement` is now a first-class YAML field so server-mode + // schemas get the same preference-vector control as the embedded + // SchemaBuilder. When omitted it stays None (engine name-based fallback). + let yaml = r" +signals: + - name: like + entity: item + decay: + exponential: + half_life_seconds: 1209600 + windows: [all_time] + positive_engagement: true + - name: skip + entity: item + decay: + permanent: true +"; + let spec: SchemaSpec = serde_yml::from_str(yaml).unwrap(); + assert_eq!(spec.signals[0].positive_engagement, Some(true)); + assert_eq!(spec.signals[1].positive_engagement, None); + } + #[test] fn parse_simple_sort_modes() { let cases = vec![ diff --git a/tidal-server/src/dto.rs b/tidal-server/src/dto.rs index 355ab0a..0c3c505 100644 --- a/tidal-server/src/dto.rs +++ b/tidal-server/src/dto.rs @@ -12,20 +12,28 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; use tidaldb::query::{retrieve::RetrieveResult, search::SearchResultItem}; +use utoipa::ToSchema; // ── Request DTOs ───────────────────────────────────────────────────────────── /// `POST /items` body. -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, ToSchema)] pub struct ItemRequest { + /// Entity ID of the item to create. + #[schema(example = 1)] pub entity_id: u64, + /// Arbitrary string→string metadata (e.g. `title`, `category`, `created_at`). pub metadata: HashMap, } /// `POST /embeddings` body. -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, ToSchema)] pub struct EmbeddingRequest { + /// Entity ID the embedding belongs to. + #[schema(example = 1)] pub entity_id: u64, + /// Dense embedding vector. The database retrieves and ranks over these + /// vectors; it does not generate them. pub values: Vec, } @@ -35,13 +43,21 @@ pub struct EmbeddingRequest { /// standalone path; the cluster path ignores them (its replication layer does /// not yet carry per-signal context), so defaulting them keeps both modes' /// wire contract identical. -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, ToSchema)] pub struct SignalRequest { + /// Entity the signal is recorded against. + #[schema(example = 1)] pub entity_id: u64, + /// Signal type name (e.g. `view`, `like`, `share`). + #[schema(example = "view")] pub signal: String, + /// Signal weight applied to the running decay score. + #[schema(example = 1.0)] pub weight: f64, + /// Optional originating user context (standalone path only). #[serde(default)] pub user_id: Option, + /// Optional originating creator context (standalone path only). #[serde(default)] pub creator_id: Option, } @@ -61,14 +77,21 @@ pub struct SignalRequest { pub const MAX_LIMIT: u32 = 1000; /// `GET /feed` query parameters. -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, ToSchema, utoipa::IntoParams)] +#[into_params(parameter_in = Query)] pub struct FeedQuery { + /// Optional user to personalize ranking for. #[serde(default)] pub user_id: Option, + /// Named ranking profile; defaults to `for_you`. #[serde(default = "default_profile")] + #[param(example = "for_you")] pub profile: String, + /// Page size; clamped to `MAX_LIMIT` (1000) at the trust boundary. #[serde(default = "default_limit")] + #[param(example = 20)] pub limit: u32, + /// Target region (cluster mode only; rejected with 400 standalone). #[serde(default)] pub region: Option, } @@ -90,13 +113,20 @@ impl FeedQuery { } /// `GET /search` query parameters. -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, ToSchema, utoipa::IntoParams)] +#[into_params(parameter_in = Query)] pub struct SearchQueryParams { + /// Free-text query string (BM25 + ANN hybrid). + #[param(example = "jazz piano")] pub query: String, + /// Optional user to personalize ranking for. #[serde(default)] pub user_id: Option, + /// Page size; clamped to `MAX_LIMIT` (1000) at the trust boundary. #[serde(default = "default_limit")] + #[param(example = 20)] pub limit: u32, + /// Target region (cluster mode only; rejected with 400 standalone). #[serde(default)] pub region: Option, } @@ -132,46 +162,63 @@ pub const fn default_limit() -> u32 { // ── Response DTOs ──────────────────────────────────────────────────────────── /// `GET /feed` response body. -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, ToSchema)] pub struct FeedResponse { + /// Ranked items, best-first. pub items: Vec, + /// Total candidates considered before ranking/diversity. pub total_candidates: usize, + /// Region the feed was served from (cluster mode); `null` standalone. pub region: Option, } /// One ranked item in a feed response. -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, ToSchema)] pub struct FeedItem { + /// Entity ID of the ranked item. pub entity_id: u64, + /// Final ranking score. pub score: f64, + /// 1-based rank within the page. pub rank: usize, + /// Signal snapshot for explainability; omitted when empty. #[serde(skip_serializing_if = "Option::is_none")] pub signals: Option>, } /// A signal value snapshot attached to a ranked item, for explainability. -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, ToSchema)] pub struct SignalValue { + /// Signal type name. pub name: String, + /// Decayed signal value at query time. pub value: f64, } /// `GET /search` response body. -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, ToSchema)] pub struct SearchResponse { + /// Ranked search results, best-first. pub items: Vec, + /// Total candidates considered before ranking/diversity. pub total_candidates: usize, + /// Region the search was served from (cluster mode); `null` standalone. pub region: Option, } /// One ranked item in a search response. -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, ToSchema)] pub struct SearchItem { + /// Entity ID of the ranked item. pub entity_id: u64, + /// Final fused ranking score. pub score: f64, + /// 1-based rank within the page. pub rank: usize, + /// BM25 lexical component; omitted when the query had no text match path. #[serde(skip_serializing_if = "Option::is_none")] pub bm25_score: Option, + /// Semantic (ANN) component; omitted when no vector match path ran. #[serde(skip_serializing_if = "Option::is_none")] pub semantic_score: Option, } diff --git a/tidal-server/src/lib.rs b/tidal-server/src/lib.rs index b34e461..d08af14 100644 --- a/tidal-server/src/lib.rs +++ b/tidal-server/src/lib.rs @@ -19,6 +19,7 @@ pub mod dto; pub mod error; pub mod health; pub mod offload; +pub mod openapi; pub mod router; pub mod scatter_gather; pub mod state; diff --git a/tidal-server/src/openapi.rs b/tidal-server/src/openapi.rs new file mode 100644 index 0000000..d866d90 --- /dev/null +++ b/tidal-server/src/openapi.rs @@ -0,0 +1,225 @@ +//! Machine-readable `OpenAPI` 3.1 specification for the tidalDB HTTP API. +//! +//! Two [`OpenApi`](utoipa::OpenApi) documents are derived from the +//! `#[utoipa::path(...)]` attributes on the handlers and the +//! `#[derive(ToSchema)]` on the DTOs: +//! +//! * [`StandaloneApiDoc`] — the data + health surface served by +//! [`crate::router::build_router`]. +//! * [`ClusterApiDoc`] — the same surface PLUS the cluster-management and +//! sharded (scatter-gather) routes served by +//! [`crate::cluster::build_cluster_router`]. +//! +//! Both are served UNAUTHENTICATED at `GET /openapi.json` (sibling to the +//! `/health/*` probes): the document describes the API contract only — it +//! carries no entity data, signals, or secrets — so gating it behind the same +//! bearer token a client needs the spec to learn how to send would be +//! self-defeating. The `bearerAuth` security scheme is declared via the +//! [`SecurityAddon`] modifier so generated clients know the data routes require +//! a token; the spec endpoint itself is exempt. + +use axum::Json; +use utoipa::{ + Modify, OpenApi, + openapi::security::{Http, HttpAuthScheme, SecurityScheme}, +}; + +/// `GET /openapi.json` handler for the **standalone** surface. +/// +/// Mounted unauthenticated alongside the `/health/*` probes by +/// [`crate::router::build_router`]. Returns the derived `OpenAPI` 3.1 document +/// as JSON (`utoipa::openapi::OpenApi` is `Serialize`). +pub(crate) async fn serve_standalone() -> Json { + Json(StandaloneApiDoc::openapi()) +} + +/// `GET /openapi.json` handler for the **cluster** surface — the superset +/// document that also describes the `/cluster/*` and `/sharded/*` routes. +/// Mounted by [`crate::cluster::build_cluster_router`]. +pub(crate) async fn serve_cluster() -> Json { + Json(ClusterApiDoc::openapi()) +} + +/// Injects the `bearerAuth` HTTP security scheme into the components. +/// +/// The per-handler `security(("bearerAuth" = []))` attributes reference this +/// scheme by name; without registering it here the generated document would +/// name an undefined scheme. Health and `/openapi.json` carry no `security` +/// requirement, so they remain open. +struct SecurityAddon; + +impl Modify for SecurityAddon { + fn modify(&self, openapi: &mut utoipa::openapi::OpenApi) { + // `components` is always `Some` once any schema is registered, but guard + // rather than unwrap so an empty-component build degrades gracefully + // (the crate forbids `unwrap`/`expect` in non-test code). + let components = openapi.components.get_or_insert_with(Default::default); + components.add_security_scheme( + "bearerAuth", + SecurityScheme::Http(Http::new(HttpAuthScheme::Bearer)), + ); + } +} + +/// `OpenAPI` document for the standalone HTTP surface. +#[derive(OpenApi)] +#[openapi( + info( + title = "tidalDB HTTP API", + version = env!("CARGO_PKG_VERSION"), + description = "Embeddable, single-node-first database for the personalized \ + content ranking problem. Write items, embeddings, and signals; \ + retrieve ranked feeds and run hybrid search. Data routes require \ + a Bearer token when TIDAL_API_KEY is configured; health probes \ + and this spec do not.", + ), + paths( + crate::router::health, + crate::router::create_item, + crate::router::write_embedding, + crate::router::write_signal, + crate::router::feed, + crate::router::search, + ), + components(schemas( + crate::dto::ItemRequest, + crate::dto::EmbeddingRequest, + crate::dto::SignalRequest, + crate::dto::FeedResponse, + crate::dto::FeedItem, + crate::dto::SignalValue, + crate::dto::SearchResponse, + crate::dto::SearchItem, + )), + modifiers(&SecurityAddon), + tags( + (name = "health", description = "Liveness / readiness probes (unauthenticated)."), + (name = "data", description = "Item, embedding, signal writes and ranked reads."), + ), +)] +pub struct StandaloneApiDoc; + +/// `OpenAPI` document for the cluster HTTP surface. +/// +/// Superset of [`StandaloneApiDoc`]: the same data + health routes plus the +/// cluster-management (`/cluster/*`) and sharded scatter-gather +/// (`/sharded/*`) routes. The cluster handlers register the SAME `/items`, +/// `/embeddings`, `/signals`, `/feed`, `/search` paths as the standalone ones; +/// because this is a distinct document the shared paths are described once here +/// from the cluster handlers' attributes (region-aware response text), so there +/// is no path-key collision. +#[derive(OpenApi)] +#[openapi( + info( + title = "tidalDB HTTP API", + version = env!("CARGO_PKG_VERSION"), + description = "Cluster-mode tidalDB HTTP surface: the standalone data + health \ + routes plus multi-region cluster management and sharded \ + scatter-gather routes. EXPERIMENTAL — all regions run in one \ + process (no host/process isolation).", + ), + paths( + crate::cluster::cluster_status, + crate::cluster::cluster_promote, + crate::cluster::cluster_partition, + crate::cluster::cluster_heal, + crate::cluster::create_item, + crate::cluster::write_embedding, + crate::cluster::write_signal, + crate::cluster::feed, + crate::cluster::search, + crate::cluster::sharded_create_item, + crate::cluster::sharded_write_embedding, + crate::cluster::sharded_write_signal, + crate::cluster::sharded_feed, + crate::cluster::sharded_search, + ), + components(schemas( + crate::dto::ItemRequest, + crate::dto::EmbeddingRequest, + crate::dto::SignalRequest, + crate::dto::FeedResponse, + crate::dto::FeedItem, + crate::dto::SignalValue, + crate::dto::SearchResponse, + crate::dto::SearchItem, + crate::cluster::ClusterStatusResponse, + crate::cluster::RegionStatus, + crate::cluster::RegionRequest, + crate::cluster::ScatterGatherInfo, + crate::cluster::ShardedFeedResponse, + crate::cluster::ShardedSearchResponse, + )), + modifiers(&SecurityAddon), + tags( + (name = "health", description = "Liveness / readiness probes (unauthenticated)."), + (name = "data", description = "Item, embedding, signal writes and ranked reads."), + (name = "cluster", description = "Multi-region cluster management."), + (name = "sharded", description = "Scatter-gather routes fanning across all shards."), + ), +)] +pub struct ClusterApiDoc; + +#[cfg(test)] +mod tests { + use super::*; + + /// The standalone document must enumerate the data + health paths and carry + /// the bearerAuth scheme so generated clients know how to authenticate. + #[test] + fn standalone_doc_lists_core_paths() { + let doc = StandaloneApiDoc::openapi(); + let paths = &doc.paths.paths; + for p in [ + "/health", + "/items", + "/embeddings", + "/signals", + "/feed", + "/search", + ] { + assert!(paths.contains_key(p), "standalone doc missing path {p}"); + } + // The cluster-only routes must NOT leak into the standalone document. + assert!( + !paths.contains_key("/cluster/status"), + "standalone doc must not advertise cluster routes" + ); + let schemes = &doc + .components + .as_ref() + .expect("components present after schema registration") + .security_schemes; + assert!( + schemes.contains_key("bearerAuth"), + "SecurityAddon must register the bearerAuth scheme" + ); + } + + /// The cluster document is a superset: data + health + cluster + sharded. + #[test] + fn cluster_doc_lists_cluster_and_sharded_paths() { + let doc = ClusterApiDoc::openapi(); + let paths = &doc.paths.paths; + for p in [ + "/feed", + "/cluster/status", + "/cluster/promote", + "/cluster/partition", + "/cluster/heal", + "/sharded/items", + "/sharded/feed", + "/sharded/search", + ] { + assert!(paths.contains_key(p), "cluster doc missing path {p}"); + } + } + + /// The served version string is the crate version, exposed for the route's + /// contract test and the verification report. + #[test] + fn doc_version_is_crate_version() { + let doc = StandaloneApiDoc::openapi(); + assert_eq!(doc.info.version, env!("CARGO_PKG_VERSION")); + } +} diff --git a/tidal-server/src/router.rs b/tidal-server/src/router.rs index d270abb..cea0329 100644 --- a/tidal-server/src/router.rs +++ b/tidal-server/src/router.rs @@ -97,6 +97,10 @@ pub fn build_router(state: Arc, api_key: Option>) -> Route .route("/health", get(health)) .route("/health/startup", get(crate::health::health_startup)) .route("/health/live", get(crate::health::health_live)) + // The OpenAPI document describes the contract only (no data/secrets), so + // it is served unauthenticated next to the probes — a client needs the + // spec to learn how to authenticate the data routes. + .route("/openapi.json", get(crate::openapi::serve_standalone)) .with_state(Arc::clone(&state)); // Protected routes — gated by Bearer token when a key is configured. @@ -194,7 +198,19 @@ pub async fn bearer_auth(request: Request, next: Next, expected_key: &str) -> Re } } -async fn create_item( +#[utoipa::path( + post, + path = "/items", + tag = "data", + request_body = ItemRequest, + responses( + (status = 201, description = "Item created"), + (status = 400, description = "Invalid request"), + (status = 401, description = "Missing or invalid API key"), + ), + security(("bearerAuth" = [])), +)] +pub(crate) async fn create_item( State(state): State>, Json(req): Json, ) -> Result { @@ -204,7 +220,19 @@ async fn create_item( Ok(StatusCode::CREATED) } -async fn write_embedding( +#[utoipa::path( + post, + path = "/embeddings", + tag = "data", + request_body = EmbeddingRequest, + responses( + (status = 204, description = "Embedding written"), + (status = 400, description = "Invalid request"), + (status = 401, description = "Missing or invalid API key"), + ), + security(("bearerAuth" = [])), +)] +pub(crate) async fn write_embedding( State(state): State>, Json(req): Json, ) -> Result { @@ -214,7 +242,19 @@ async fn write_embedding( Ok(StatusCode::NO_CONTENT) } -async fn write_signal( +#[utoipa::path( + post, + path = "/signals", + tag = "data", + request_body = SignalRequest, + responses( + (status = 204, description = "Signal recorded"), + (status = 400, description = "Invalid request"), + (status = 401, description = "Missing or invalid API key"), + ), + security(("bearerAuth" = [])), +)] +pub(crate) async fn write_signal( State(state): State>, Json(req): Json, ) -> Result { @@ -230,7 +270,19 @@ async fn write_signal( Ok(StatusCode::NO_CONTENT) } -async fn feed( +#[utoipa::path( + get, + path = "/feed", + tag = "data", + params(FeedQuery), + responses( + (status = 200, description = "Ranked feed", body = FeedResponse), + (status = 400, description = "Invalid request (e.g. region param standalone)"), + (status = 401, description = "Missing or invalid API key"), + ), + security(("bearerAuth" = [])), +)] +pub(crate) async fn feed( State(state): State>, Query(query): Query, ) -> Result, AppError> { @@ -265,7 +317,19 @@ async fn feed( })) } -async fn search( +#[utoipa::path( + get, + path = "/search", + tag = "data", + params(SearchQueryParams), + responses( + (status = 200, description = "Ranked search results", body = SearchResponse), + (status = 400, description = "Invalid request (e.g. region param standalone)"), + (status = 401, description = "Missing or invalid API key"), + ), + security(("bearerAuth" = [])), +)] +pub(crate) async fn search( State(state): State>, Query(query): Query, ) -> Result, AppError> { @@ -300,7 +364,16 @@ async fn search( } /// Readiness probe: 200 when ready, 503 when shutting down. -async fn health( +#[utoipa::path( + get, + path = "/health", + tag = "health", + responses( + (status = 200, description = "Service ready"), + (status = 503, description = "Service shutting down"), + ), +)] +pub(crate) async fn health( State(state): State>, Query(query): Query>, ) -> std::result::Result<(StatusCode, Json), AppError> { @@ -337,7 +410,7 @@ impl From for AppError { } } -struct AppError(ServerError); +pub(crate) struct AppError(ServerError); impl IntoResponse for AppError { fn into_response(self) -> Response { diff --git a/tidal-server/tests/standalone.rs b/tidal-server/tests/standalone.rs index deb932b..3e6addb 100644 --- a/tidal-server/tests/standalone.rs +++ b/tidal-server/tests/standalone.rs @@ -289,3 +289,38 @@ async fn search_rejects_region_param() { "error must explain region routing is cluster-only: {body}" ); } + +/// The `OpenAPI` document is served, unauthenticated, at `GET /openapi.json` +/// and describes the data surface (this is the canonical HTTP API reference). +/// The served document — not just the in-crate `StandaloneApiDoc` unit test — is +/// the contract clients fetch, so pin it end-to-end through the router. +#[tokio::test] +async fn openapi_json_is_served_and_describes_the_data_routes() { + let app = make_app(); + let (status, body) = get_json(&app, "/openapi.json").await; + assert_eq!(status, StatusCode::OK, "body: {body}"); + + // OpenAPI 3.x envelope: a top-level `openapi` version string. + let version = body + .get("openapi") + .and_then(serde_json::Value::as_str) + .unwrap_or(""); + assert!( + version.starts_with("3."), + "expected an OpenAPI 3.x document, got openapi={version:?}" + ); + + // The /feed path must be advertised so generated clients can call the feed. + assert!( + body.pointer("/paths/~1feed").is_some(), + "served spec must document the /feed path: {body}" + ); + + // The bearerAuth scheme must be registered so clients know the data routes + // are token-gated when TIDAL_API_KEY is set. + assert!( + body.pointer("/components/securitySchemes/bearerAuth") + .is_some(), + "served spec must declare the bearerAuth security scheme: {body}" + ); +} diff --git a/tidal/CLAUDE.md b/tidal/CLAUDE.md index cd1fee6..7bc1494 100644 --- a/tidal/CLAUDE.md +++ b/tidal/CLAUDE.md @@ -38,9 +38,11 @@ re-create a `tidal/docs/`, `tidal/ai-lookup/`, `tidal/site/`, or `tidal/README.m | `text/` | Tantivy full-text indexing | | `wal/` | Write-ahead log + crash recovery | -`benches/`, `examples/` (quickstart, axum_embedding, actix_embedding, cli_embedding), -`tests/` (integration + crash-property), and `docker/` (cluster / standalone / deploy) -round out the crate. +`benches/`, `examples/` (quickstart, foryou_feed, axum_embedding, actix_embedding, +cli_embedding), and `tests/` (integration + crash-property) round out the crate. +Container images live at the workspace root under [`../docker/`](../docker/) +(`standalone` / `cluster` / `deploy`), not per-crate — build them from the repo +root so `COPY . .` sees the whole workspace. ## Build & test (workspace-aware) diff --git a/tidal/Cargo.toml b/tidal/Cargo.toml index 04fb009..867b3ca 100644 --- a/tidal/Cargo.toml +++ b/tidal/Cargo.toml @@ -78,6 +78,9 @@ unwrap_used = "deny" [[example]] name = "quickstart" +[[example]] +name = "foryou_feed" + [[example]] name = "axum_embedding" diff --git a/tidal/docker/cluster/Dockerfile b/tidal/docker/cluster/Dockerfile deleted file mode 100644 index 0e8f0a2..0000000 --- a/tidal/docker/cluster/Dockerfile +++ /dev/null @@ -1,73 +0,0 @@ -# Multi-region tidalDB cluster image. -# -# Runs a 3-region cluster (us-east, eu-west, ap-south) behind a single HTTP -# surface on port 9500. Regions replicate over the real tidal-net gRPC transport -# (loopback), but all run in this single container process — there is no host or -# process isolation, so it is NOT production HA (true multi-process deployment is -# m8p10). See docs/runbooks/cluster.md for the operational API. -# -# Build context is this repository's root (the workspace `Cargo.toml` / -# `Cargo.lock` and every member crate live there). Build from the repo root: -# -# docker build -f tidal/docker/cluster/Dockerfile -t tidaldb:cluster . -# -# The repo-root allowlist `.dockerignore` strips target/, node_modules/, .git, -# and .env* from the context, so `COPY . .` brings in exactly the workspace -# sources cargo needs to resolve the member graph and build `-p tidal-server`. -# Pin the builder to bookworm so its glibc matches the bookworm-slim runtime -# below. The default `rust:1.91` tracks Debian trixie (glibc 2.41), whose -# auto-vectorized math pulls in `libmvec.so.1` — a library bookworm's glibc does -# not ship, so a trixie-built binary aborts at startup on bookworm with -# "libmvec.so.1: cannot open shared object file". -FROM rust:1.91-bookworm AS builder - -ARG DEBIAN_FRONTEND=noninteractive - -WORKDIR /app - -# Copy the full workspace and build only the server binary. The unified -# workspace requires every member manifest present to resolve metadata, so we -# copy the whole (already-pruned) context rather than a fragile manifest-only -# subset. -COPY . . - -# g++ builds the usearch C++ HNSW core; protobuf-compiler (protoc) is needed by -# tidal-net's build script (tonic-build compiles the WAL-shipping .proto), which -# tidal-server now depends on for real gRPC cluster replication. -RUN apt-get update && apt-get install -y --no-install-recommends g++ protobuf-compiler && rm -rf /var/lib/apt/lists/* -RUN cargo build -p tidal-server --release --locked - -FROM debian:bookworm-slim - -ARG DEBIAN_FRONTEND=noninteractive - -WORKDIR /srv -RUN useradd --system --home /srv tidal && \ - apt-get update && apt-get install -y --no-install-recommends ca-certificates curl && \ - rm -rf /var/lib/apt/lists/* - -COPY --from=builder /app/target/release/tidal-server /usr/local/bin/tidal-server -COPY --chown=tidal:tidal tidal-server/config /etc/tidal-server - -USER tidal -EXPOSE 9500 - -# Path the server reads its schema/topology from. The binary-side clap wiring -# for this lives in tidal-server (see sibling task T5); keep the name and -# default in sync with it here. -ENV TIDAL_CONFIG=/etc/tidal-server - -# Cluster mode is gated as experimental: replication is real gRPC but all -# regions share one process (no host/process isolation), so it refuses to start -# without an explicit opt-in. The image opts in here so `docker run` works; the -# server still logs a loud WARN that this is not production HA. -ENV TIDAL_ALLOW_EXPERIMENTAL_CLUSTER=1 - -HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \ - CMD curl -f http://localhost:9500/health || exit 1 - -# ENTRYPOINT is just the binary so `docker run tidaldb:cluster ` -# overrides work. CMD carries the default cluster invocation against the baked -# schema + topology. -ENTRYPOINT ["tidal-server"] -CMD ["cluster", "--listen", "0.0.0.0:9500", "--schema", "/etc/tidal-server/default-schema.yaml", "--topology", "/etc/tidal-server/default-cluster.yaml"] diff --git a/tidal/docker/deploy/Dockerfile b/tidal/docker/deploy/Dockerfile deleted file mode 100644 index 68ecf77..0000000 --- a/tidal/docker/deploy/Dockerfile +++ /dev/null @@ -1,64 +0,0 @@ -# Production single-node tidalDB deploy image (slim toolchain, durable /data). -# -# Build context is this repository's root (the workspace `Cargo.toml` / -# `Cargo.lock` and every member crate live there). Build from the repo root: -# -# docker build -f tidal/docker/deploy/Dockerfile -t tidaldb:deploy . -# -# The repo-root allowlist `.dockerignore` strips target/, node_modules/, .git, -# and .env* from the context, so `COPY . .` brings in exactly the workspace -# sources cargo needs to resolve the member graph and build `-p tidal-server`. -# Pin the builder to bookworm so its glibc matches the bookworm-slim runtime. -# The default slim tag tracks Debian trixie, whose vectorized-math -# `libmvec.so.1` is absent on bookworm and aborts the binary at startup. -FROM rust:1.91-slim-bookworm AS builder -ARG DEBIAN_FRONTEND=noninteractive -WORKDIR /build -# protobuf-compiler (protoc) is required by tidal-net's build script (tonic-build -# compiles the WAL-shipping .proto); tidal-server depends on tidal-net for gRPC -# cluster replication. -RUN apt-get update && apt-get install -y --no-install-recommends pkg-config libssl-dev g++ protobuf-compiler && rm -rf /var/lib/apt/lists/* - -# Copy the full workspace and build only the server binary. The unified -# workspace requires every member manifest present to resolve metadata, so we -# copy the whole (already-pruned) context rather than a fragile manifest-only -# subset. -COPY . . -RUN cargo build -p tidal-server --release --locked - -FROM debian:bookworm-slim -ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl && rm -rf /var/lib/apt/lists/* -RUN useradd --system -u 10001 tidal - -COPY --from=builder --chown=tidal:tidal /build/target/release/tidal-server /usr/local/bin/tidal-server -COPY --chown=tidal:tidal tidal-server/config /config - -# Persistent data lives under /data, owned by the runtime user. Without this -# directory + the `--data-dir /data` flag below the server boots EPHEMERAL and -# loses every write on restart. -RUN mkdir -p /data && chown tidal:tidal /data -VOLUME ["/data"] - -USER tidal:tidal -WORKDIR /data -EXPOSE 9400 9091 - -ENV TIDAL_SERVER_LOG=info -# Path the server reads its schema/topology from. The binary-side clap wiring -# for this lives in tidal-server (see sibling task T5); keep the name and -# default in sync with it here. -ENV TIDAL_CONFIG=/config - -HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ - CMD curl -sf http://localhost:9400/health || exit 1 - -# ENTRYPOINT is just the binary so `docker run tidaldb:deploy ` -# overrides work. CMD carries the durable standalone invocation against the -# baked schema, persisting to the /data volume. -ENTRYPOINT ["tidal-server"] -CMD ["standalone", \ - "--listen", "0.0.0.0:9400", \ - "--schema", "/config/default-schema.yaml", \ - "--data-dir", "/data", \ - "--metrics", "0.0.0.0:9091"] diff --git a/tidal/docker/docker-compose.yml b/tidal/docker/docker-compose.yml deleted file mode 100644 index 3df3782..0000000 --- a/tidal/docker/docker-compose.yml +++ /dev/null @@ -1,35 +0,0 @@ -services: - tidaldb: - # Build context is this repository's root (../.. from this compose file: - # docker -> tidal -> root) so the workspace Cargo.toml/lock and every - # member crate are in scope for `cargo build -p tidal-server`. - build: - context: ../.. - dockerfile: tidal/docker/standalone/Dockerfile - ports: - - "9400:9400" - - "9091:9091" - environment: - - TIDAL_API_KEY=${TIDAL_API_KEY} - - TIDAL_SERVER_LOG=info - volumes: - # Matches the standalone image's durable --data-dir + VOLUME ["/data"]. - - tidaldb-data:/data - restart: unless-stopped - healthcheck: - test: ["CMD", "curl", "-f", "-H", "Authorization: Bearer ${TIDAL_API_KEY}", "http://localhost:9400/health"] - interval: 30s - timeout: 5s - retries: 3 - - prometheus: - image: prom/prometheus:v2.53.0 - ports: - - "9090:9090" - volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro - depends_on: - - tidaldb - -volumes: - tidaldb-data: diff --git a/tidal/docker/prometheus.yml b/tidal/docker/prometheus.yml deleted file mode 100644 index 4110900..0000000 --- a/tidal/docker/prometheus.yml +++ /dev/null @@ -1,8 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - - job_name: tidaldb - static_configs: - - targets: ['tidaldb:9091'] diff --git a/tidal/docker/standalone/Dockerfile b/tidal/docker/standalone/Dockerfile deleted file mode 100644 index c6f3b3c..0000000 --- a/tidal/docker/standalone/Dockerfile +++ /dev/null @@ -1,66 +0,0 @@ -# Single-node tidalDB server image. -# -# Build context is this repository's root (the workspace `Cargo.toml` / -# `Cargo.lock` and every member crate live there). Build from the repo root: -# -# docker build -f tidal/docker/standalone/Dockerfile -t tidaldb:standalone . -# -# tidal-server is a workspace member at `tidal-server/` and depends on the -# `tidal/` engine crate plus `tidal-net/` (gRPC cluster transport). The repo-root -# allowlist `.dockerignore` strips target/, node_modules/, .git, and .env* from -# the context, so `COPY . .` brings in exactly the workspace sources cargo needs -# to resolve the member graph and build `-p tidal-server`. -# Pin the builder to bookworm so its glibc matches the bookworm-slim runtime. -# The default `rust:1.91` tracks Debian trixie, whose vectorized-math -# `libmvec.so.1` is absent on bookworm and aborts the binary at startup. -FROM rust:1.91-bookworm AS builder -ARG DEBIAN_FRONTEND=noninteractive -WORKDIR /app - -# g++ builds the usearch C++ HNSW core; protobuf-compiler (protoc) is needed by -# tidal-net's build script (tonic-build compiles the WAL-shipping .proto). -RUN apt-get update && apt-get install -y --no-install-recommends g++ protobuf-compiler && rm -rf /var/lib/apt/lists/* - -# Copy the full workspace and build only the server binary. The unified -# workspace requires every member manifest present to resolve metadata, so we -# copy the whole (already-pruned) context rather than a fragile manifest-only -# subset. -COPY . . -RUN cargo build -p tidal-server --release --locked - -FROM debian:bookworm-slim -ARG DEBIAN_FRONTEND=noninteractive -WORKDIR /srv - -RUN useradd --system --home /srv tidal && \ - apt-get update && apt-get install -y --no-install-recommends ca-certificates curl && \ - rm -rf /var/lib/apt/lists/* - -COPY --from=builder /app/target/release/tidal-server /usr/local/bin/tidal-server -COPY --chown=tidal:tidal tidal-server/config /etc/tidal-server - -# Persistent data lives under /data, owned by the runtime user. Without this -# directory + the `--data-dir /data` flag below the server boots EPHEMERAL and -# loses every write on restart. -RUN mkdir -p /data && chown tidal:tidal /data -VOLUME ["/data"] - -# Path the server reads its schema/topology from. The binary-side clap wiring -# for this lives in tidal-server (see sibling task T5); keep the name and -# default in sync with it here. -ENV TIDAL_CONFIG=/etc/tidal-server - -USER tidal -EXPOSE 9400 9091 - -HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \ - CMD curl -f -H "Authorization: Bearer ${TIDAL_API_KEY:-}" http://localhost:9400/health || exit 1 - -# ENTRYPOINT is just the binary so `docker run tidaldb:standalone ` -# overrides work (e.g. `... cluster ...` or an ad-hoc inspect). CMD carries the -# default standalone invocation, including `--data-dir /data` for durability. -ENTRYPOINT ["tidal-server"] -CMD ["standalone", \ - "--listen", "0.0.0.0:9400", \ - "--data-dir", "/data", \ - "--metrics", "0.0.0.0:9091"] diff --git a/tidal/examples/foryou_feed.rs b/tidal/examples/foryou_feed.rs new file mode 100644 index 0000000..6d2dcc8 --- /dev/null +++ b/tidal/examples/foryou_feed.rs @@ -0,0 +1,472 @@ +#![allow(clippy::unwrap_used)] +//! tidalDB short-video "For You" feed: the TikTok/Reels-shaped use case. +//! +//! Demonstrates the full swipe-loop on the embedded engine — the same shape a +//! short-video app's home feed would drive: +//! +//! 1. Define a schema modeling short-video signals: `view`, `completion`, +//! `like`, `share`, `skip` (the swipe-away negative), and `replay`. +//! 2. Open an ephemeral database and ingest ~24 short videos across 5 creators. +//! 3. Simulate one user's swipe session: completions + likes on music videos, +//! skips on the rest, and a couple of shares — fed back through +//! `signal_with_context` so the engine updates *that user's* taste vector. +//! 4. Retrieve the personalized `for_you` feed for the user (diversity-capped, +//! with a 10% exploration budget), then the global `trending` feed for +//! contrast. +//! +//! The point: the ranking outcome is driven by the *signals*, not by the random +//! embeddings. The user who completes and likes music videos and swipes past +//! everything else gets a music-leaning personalized feed — deterministically, +//! because the feedback loop is real. +//! +//! # Running +//! +//! ```bash +//! cargo run -p tidaldb --example foryou_feed +//! ``` + +use std::{collections::HashMap, time::Duration}; + +use rand::Rng; +use tidaldb::{ + TidalDb, + schema::{DecaySpec, EntityId, EntityKind, SchemaBuilder, Timestamp, Window}, +}; + +/// Generate a random unit-normalized embedding of the given dimensionality. +/// +/// A real short-video app does NOT compute embeddings this way — it runs the +/// video (frames + audio + caption) through a multimodal model and stores the +/// resulting vector here. tidalDB *retrieves and ranks over* embeddings; it +/// does not generate them. See `docs/guides/embeddings.md`. +fn random_unit_vector(dim: usize, rng: &mut impl Rng) -> Vec { + let v: Vec = (0..dim).map(|_| rng.random::() - 0.5).collect(); + let norm = v.iter().map(|x| x * x).sum::().sqrt(); + if norm < f32::EPSILON { + // Degenerate case: return a unit vector along the first axis. + let mut unit = vec![0.0_f32; dim]; + unit[0] = 1.0; + return unit; + } + v.iter().map(|x| x / norm).collect() +} + +#[allow(clippy::too_many_lines)] // Linear walkthrough script; splitting would hurt readability. +fn main() -> Result<(), Box> { + // Initialize tracing so spans emitted by tidalDB are visible. + tracing_subscriber::fmt() + .with_env_filter("tidaldb=info") + .init(); + + // ── 1. Define the short-video schema ──────────────────────────────── + // + // Each signal carries native temporal semantics. The half-lives encode how + // long each kind of engagement should keep mattering to ranking: + // - `view` decays over a week; this is the raw impression. + // - `completion` barely decays — finishing a short video is a strong, + // durable taste signal. Weight is the fraction watched (0..1), and it is + // a *positive-engagement* signal: it pulls the user's preference vector + // toward the video's content embedding. + // - `like` decays over a month and is also positive-engagement. + // - `share` decays fast (3d) and tracks velocity — shares are bursty. + // - `skip` the swipe-away. Fast 1d decay; NOT positive-engagement + // (a swipe must never pull taste toward the skipped video). Routed + // through `signal_with_context` it also marks the item as a hard + // negative / seen for that user. + // - `replay` medium decay; re-watching is a meaningful positive. + + let mut schema = SchemaBuilder::new(); + + // view: 7-day half-life, velocity + 1h/24h/AllTime windows. + let _ = schema + .signal( + "view", + EntityKind::Item, + DecaySpec::Exponential { + half_life: Duration::from_secs(7 * 24 * 3600), + }, + ) + .windows(&[Window::OneHour, Window::TwentyFourHours, Window::AllTime]) + .velocity(true) + .add(); + + // completion: very slow decay (1 year), positive-engagement. + // Weight = fraction of the video watched, in [0.0, 1.0]. + let _ = schema + .signal( + "completion", + EntityKind::Item, + DecaySpec::Exponential { + half_life: Duration::from_secs(365 * 24 * 3600), + }, + ) + .windows(&[Window::AllTime]) + .velocity(false) + .positive_engagement(true) + .add(); + + // like: 30-day half-life, positive-engagement. + let _ = schema + .signal( + "like", + EntityKind::Item, + DecaySpec::Exponential { + half_life: Duration::from_secs(30 * 24 * 3600), + }, + ) + .windows(&[Window::AllTime]) + .velocity(false) + .positive_engagement(true) + .add(); + + // share: 3-day half-life, velocity (bursty), 24h + AllTime windows. + let _ = schema + .signal( + "share", + EntityKind::Item, + DecaySpec::Exponential { + half_life: Duration::from_secs(3 * 24 * 3600), + }, + ) + .windows(&[Window::TwentyFourHours, Window::AllTime]) + .velocity(true) + .add(); + + // skip: fast 1-day decay — the swipe-away negative. NOT positive-engagement. + let _ = schema + .signal( + "skip", + EntityKind::Item, + DecaySpec::Exponential { + half_life: Duration::from_secs(24 * 3600), + }, + ) + .windows(&[Window::OneHour, Window::TwentyFourHours]) + .velocity(false) + .add(); + + // replay: medium decay (14 days), positive-engagement. + let _ = schema + .signal( + "replay", + EntityKind::Item, + DecaySpec::Exponential { + half_life: Duration::from_secs(14 * 24 * 3600), + }, + ) + .windows(&[Window::AllTime]) + .velocity(false) + .positive_engagement(true) + .add(); + + // Embedding slot: 128-dim content vectors for items (the "content" model). + let _ = schema.embedding_slot("content", EntityKind::Item, 128); + + let schema = schema.build()?; + + // ── 2. Open an ephemeral database ─────────────────────────────────── + + let db = TidalDb::builder().ephemeral().with_schema(schema).open()?; + + db.health_check()?; + + println!( + "tidalDB opened (ephemeral, build: {}) — short-video For You demo", + tidaldb::BUILD_HASH + ); + println!(); + + // ── 3. Ingest ~24 short videos across 5 creators ──────────────────── + // + // NOTE: embeddings here are random unit vectors purely so the example is + // self-contained. A production app computes them from the actual video + // (frames/audio/caption) with a multimodal model — tidalDB only retrieves + // and ranks over them. See `docs/guides/embeddings.md`. + + let mut rng = rand::rng(); + let dim = 128; + + // Five creators, each leaning into one category so the feedback loop has a + // clear "taste" to discover. Creator 1 == music, 3 == music; the rest are + // tech / cooking / comedy. + let categories = ["music", "tech", "cooking", "comedy", "music"]; + let now = Timestamp::now(); + + let mut music_items: Vec = Vec::new(); + let mut nonmusic_items: Vec = Vec::new(); + + for i in 1..=24u64 { + // Round-robin items across the 5 creators / categories. + let slot = (i as usize - 1) % categories.len(); + let category = categories[slot]; + let creator_id = (slot + 1) as u64; // creator ids 1..=5 + + let mut metadata = HashMap::new(); + metadata.insert("title".to_string(), format!("Short #{i} — {category}")); + metadata.insert("category".to_string(), category.to_string()); + metadata.insert("format".to_string(), "short".to_string()); + metadata.insert("creator_id".to_string(), creator_id.to_string()); + // Durations typical of short-form video: 15–45s. + metadata.insert("duration".to_string(), format!("{}", 15 + (i % 4) * 10)); + metadata.insert("created_at".to_string(), now.as_nanos().to_string()); + + db.write_item_with_metadata(EntityId::new(i), &metadata)?; + + let embedding = random_unit_vector(dim, &mut rng); + db.write_item_embedding(EntityId::new(i), &embedding)?; + + if category == "music" { + music_items.push(i); + } else { + nonmusic_items.push(i); + } + } + + println!( + "Ingested {} short videos across 5 creators ({} music, {} other), {dim}D embeddings.", + db.item_count(), + music_items.len(), + nonmusic_items.len() + ); + println!(" music videos: {music_items:?}\n non-music videos: {nonmusic_items:?}"); + println!(); + + // ── 4. Simulate one user's swipe session ──────────────────────────── + // + // We route every engagement through `signal_with_context` with the user id + // (1001) and the video's creator id. That is what makes this a *feedback + // loop*: positive-engagement signals (completion, like, replay) fold the + // video's content embedding into user 1001's preference vector, while skips + // mark videos as hard negatives for that user. + // + // IMPORTANT — a session is a *slice* of the catalog, not the whole thing. + // Any item the user engages with (positively OR negatively) is marked SEEN + // for that user, and the personalized `for_you` feed deliberately excludes + // seen + hard-negative items: a feed must surface FRESH content, never + // re-show what was just watched. So we touch only part of the catalog and + // leave fresh videos behind. The taste the user reveals on the watched slice + // (music) is what ranks those fresh videos. + + let user_id: u64 = 1001; + + // Helper to recover a video's creator id from its round-robin slot. + let creator_of = |item: u64| -> u64 { ((item - 1) % categories.len() as u64) + 1 }; + + // Split each category into a "watched this session" slice and a "fresh" + // remainder. The watched slice trains taste; the fresh remainder is what the + // For You feed gets to rank. + let watched_music: Vec = music_items.iter().take(5).copied().collect(); + let fresh_music: Vec = music_items.iter().skip(5).copied().collect(); + let skipped_nonmusic: Vec = nonmusic_items.iter().take(6).copied().collect(); + let fresh_nonmusic: Vec = nonmusic_items.iter().skip(6).copied().collect(); + + let mut completions = 0u32; + let mut likes = 0u32; + let mut skips = 0u32; + let mut shares = 0u32; + let mut replays = 0u32; + + // The user LOVES music: watches the session's music videos to completion and + // likes them. (Positive-engagement → folds each video's embedding into the + // user's preference vector, and strengthens the (user, music-creator) edge.) + for &item in &watched_music { + // completion weight is the fraction watched — these get watched fully. + db.signal_with_context( + "completion", + EntityId::new(item), + 1.0, + now, + Some(user_id), + Some(creator_of(item)), + )?; + completions += 1; + + db.signal_with_context( + "like", + EntityId::new(item), + 1.0, + now, + Some(user_id), + Some(creator_of(item)), + )?; + likes += 1; + } + + // Two shares + one replay on the first couple of watched music videos. + for &item in watched_music.iter().take(2) { + db.signal_with_context( + "share", + EntityId::new(item), + 1.0, + now, + Some(user_id), + Some(creator_of(item)), + )?; + shares += 1; + } + if let Some(&item) = watched_music.first() { + db.signal_with_context( + "replay", + EntityId::new(item), + 1.0, + now, + Some(user_id), + Some(creator_of(item)), + )?; + replays += 1; + } + + // The user SWIPES PAST the session's non-music videos: a brief view, then a + // skip. The skip is the swipe-away negative — it must NOT pull taste toward + // the skipped content (skip is not positive-engagement), and it marks the + // video as a hard negative so the feed never re-surfaces it. + for &item in &skipped_nonmusic { + // A partial view (watched ~10% — a glance) before swiping. + db.signal_with_context( + "view", + EntityId::new(item), + 0.1, + now, + Some(user_id), + Some(creator_of(item)), + )?; + db.signal_with_context( + "skip", + EntityId::new(item), + 1.0, + now, + Some(user_id), + Some(creator_of(item)), + )?; + skips += 1; + } + + println!( + "User {user_id} swipe session: {completions} completions, {likes} likes, \ + {shares} shares, {replays} replay, {skips} skips." + ); + println!( + " watched (now seen): music {watched_music:?}, skipped non-music {skipped_nonmusic:?}" + ); + println!(" FRESH (feed pool): music {fresh_music:?}, non-music {fresh_nonmusic:?}"); + + // Show that the feedback loop landed: the most-completed music video has a + // live completion decay score. + if let Some(&top_music) = watched_music.first() { + let score = db.read_decay_score(EntityId::new(top_music), "completion", 0)?; + println!( + " completion decay score for music video #{top_music}: {:.4}", + score.unwrap_or(0.0) + ); + } + println!(); + + // ── 5. Retrieve the personalized For You feed ─────────────────────── + // + // `for_you` blends interaction-weighted decay scores (view + 2*like + + // 1.5*share velocity) with the user's preference vector, then enforces + // diversity: `max_per_creator = 2` so a single creator cannot dominate the + // feed, plus a 10% exploration budget (`FOR_YOU_EXPLORATION`) that injects + // unseen/random items to avoid a filter bubble. Skipped videos are filtered + // out as hard negatives for this user. + + let for_you = tidaldb::query::retrieve::Retrieve::builder() + .for_user(user_id) + .profile("for_you") + .limit(10) + .build()?; + + let results = db.retrieve(&for_you)?; + + println!( + "RETRIEVE profile=for_you (user={user_id}): {} results from {} candidates", + results.items.len(), + results.total_candidates + ); + print_feed(&results); + + // The payoff: the candidate pool is the FRESH videos only (the 11 watched / + // skipped videos are excluded), and the top of the feed is the fresh MUSIC + // videos — surfaced because the user's session built a strong interaction + // edge with the music creators. The taste was learned from signals, not from + // the (random) embeddings, so this ordering is stable across runs. + let top_ids: Vec = results + .items + .iter() + .take(fresh_music.len()) + .map(|i| i.entity_id.as_u64()) + .collect(); + println!( + " top {} For You slots are the fresh music videos {top_ids:?} (taste leaned music)", + fresh_music.len() + ); + println!(); + + // ── 6. Retrieve the global Trending feed for contrast ─────────────── + // + // `trending` is NOT personalized: it ranks every item globally by + // velocity (view_vel + 2*share_vel over the 24h window), also capped per + // creator. This is what a brand-new visitor with no taste history sees. + + let trending = tidaldb::query::retrieve::Retrieve::builder() + .profile("trending") + .limit(10) + .build()?; + + let trending_results = db.retrieve(&trending)?; + + println!( + "RETRIEVE profile=trending (global, no user): {} results from {} candidates", + trending_results.items.len(), + trending_results.total_candidates + ); + print_feed(&trending_results); + println!(); + + // ── 7. Clean up ───────────────────────────────────────────────────── + + db.close()?; + println!("tidalDB closed. For You demo complete."); + + // ── What tidalDB owns vs what the app owns ────────────────────────── + // + // tidalDB owns: RANKING. Signal ledgers (decay/velocity/windows), per-user + // preference vectors, candidate retrieval, filtering, diversity, and the + // For You / Trending ordering — one process, one query. + // + // The app owns: everything else. Video storage + CDN + transcoding, + // embedding generation (run the video through your multimodal model), + // authentication/identity, and the player UI. The app writes signals as + // users swipe and reads back ranked feeds. + // + // To build the real thing end-to-end, see `docs/guides/build-a-feed-app.md`. + + Ok(()) +} + +/// Print a ranked feed: rank, entity id, score, and per-item signal summary. +fn print_feed(results: &tidaldb::query::Results) { + println!("{:<6} {:<12} {:<10} Signals", "Rank", "Video ID", "Score"); + println!("{}", "-".repeat(64)); + + for item in &results.items { + let signal_summary: String = item + .signals + .iter() + .map(|s| format!("{}={:.3}", s.name, s.value)) + .collect::>() + .join(", "); + + println!( + "{:<6} {:<12} {:<10.4} {}", + item.rank, + item.entity_id.as_u64(), + item.score, + if signal_summary.is_empty() { + "-".to_string() + } else { + signal_summary + } + ); + } +} diff --git a/tidal/examples/quickstart.rs b/tidal/examples/quickstart.rs index d23e0f0..f4df81e 100644 --- a/tidal/examples/quickstart.rs +++ b/tidal/examples/quickstart.rs @@ -58,6 +58,9 @@ fn main() -> Result<(), Box> { ) .windows(&[Window::OneHour, Window::TwentyFourHours, Window::AllTime]) .velocity(true) + // Positive engagement: folds into a user's taste vector when recorded + // with context (see the personalization step below). + .positive_engagement(true) .add(); // Like signal: 30-day half-life, AllTime window. @@ -71,6 +74,7 @@ fn main() -> Result<(), Box> { ) .windows(&[Window::AllTime]) .velocity(false) + .positive_engagement(true) .add(); // Share signal: needed by the trending profile's boost definitions.