diff --git a/.gitignore b/.gitignore index f074d7a..c2b15b1 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ target/ # Dependencies node_modules/ +vendor/ +__pycache__/ # Secrets (never commit) .env @@ -17,11 +19,18 @@ node_modules/ *.key credentials.json service-account*.json +.envault/ # Logs *.log logs/ +# Generic build output (no tracked path uses these names today — verified before adding) +dist/ +build/ +*.exe +*.dmg + # IDE / OS .idea/ .vscode/ @@ -44,6 +53,7 @@ tmp/ test-results/ playwright-report/ playwright-report-demo/ +playwright-report-semantics/ playwright/.auth/ demo/out/ demo/.cache/ diff --git a/demo/capability-inventory.md b/demo/capability-inventory.md index 4665377..3fe6d09 100644 --- a/demo/capability-inventory.md +++ b/demo/capability-inventory.md @@ -17,6 +17,9 @@ touch several. | `docs/ops/observability.md` | The four observability surfaces and the aggregated-status caveat. | | `docs/ops/grafana-tidaldb.json` | The 13 dashboard panels and their PromQL. | | Live cluster (`kubectl`, `curl`) | Actual image digest, actual secret keys, actual behavior. | +| `k8s/cluster/schema-configmap.yaml` | The three declared signals and their decay, the two text fields, and the 1536-dim embedding slot. | +| `tidal/src/ranking/builtins.rs`, `profile.rs` | The 27 built-in ranking profiles and the boost/penalty/gate/exclude mechanisms they do and do not populate. | +| Local standalone node (`tidal-server standalone` + `feed-fixture`) | Actual ranking, decay, and ANN behaviour on a corpus with known ground truth. | **Verified against.** Image `registry.threesix.ai/tidal/server:m12-admin-gate-20260823@sha256:6e220060…`, @@ -332,6 +335,130 @@ absent by construction, while the operator/data credential split from `388e445` - **Coverage.** `existing-green` - **Evidence.** `tests/e2e/features/09-operator-authority.spec.ts` +### CAP-016 — A signal write changes the order of the next query + +- **Area.** `POST /signals` then `GET /feed`; the 60-item fixture catalog on a + local standalone node (`tests/e2e/app/`). +- **Business purpose.** `VISION.md:17` — "Ranking is not a feature. It is a + primitive." Every other capability here proves the *deployment* answers; none + of them proves the database does the thing it exists to do. A ranking database + whose signals do not reorder anything is a document store with extra steps. +- **Personas.** Application developer; operator. +- **Primary workflow.** Seed 60 items with deterministic 1536-dim vectors from + `tidal-stress`'s own `embedding_for`; read `/feed?profile=for_you&limit=60`; + write one unit-weight `like` on the last-ranked item; re-read immediately with + no sleep and no retry; assert the item's index decreased. +- **Edge cases.** Position is asserted, never score — the profile re-normalises + scores across the candidate set (measured: unliked items move 0.5 → 0.0 once + one item is boosted), so a score comparison would be brittle *and* would drag + ranking arithmetic into the test. A freshly seeded corpus has no signals, so + every item ties and the order is the entity-id tie-break; that is correct, and + pre-seeding engagement to make it look livelier would destroy the baseline. +- **Permission boundary.** None on standalone (`tidal-server/tests/standalone.rs:32`). + Against an authenticated target the harness proxy injects the bearer + server-side, so no credential reaches the browser. +- **Dependencies.** `tidal-server standalone`; the `feed-fixture` binary; the + cluster's own schema, extracted from `k8s/cluster/schema-configmap.yaml` + (`dimensions: 1536`) rather than `tidal-server/config/default-schema.yaml`, + which declares 128 and would reject every embedding with a 422. +- **Observability.** `/feed` returns each item's decayed signal values, so the + assertion records *why* the order changed, not just that it did. +- **Coverage.** `new-test` +- **Evidence.** `tests/e2e/features/10-ranking-semantics.spec.ts`; + `tests/e2e/demo/workflows/feed-app.demo.spec.ts` (capture `CAP-016-feed-reorder`) + +### CAP-017 — Decay is computed at query time from the declared half-life + +- **Area.** `signals[].value` on `/feed`; `decay` in the loaded schema. +- **Business purpose.** `CODING_GUIDELINES.md:88` — "Decay is a type, not a + formula you call." If decay were a batch job, a value would be stale between + runs; if it were applied at write time, it could not change without a write. +- **Personas.** Application developer. +- **Primary workflow.** Write `view` and `like` concurrently (so both carry the + same elapsed time), read the feed twice separated by a known interval, and + recover each half-life from the two readings via `H = t·ln2 / −ln(v₂/v₁)`. + Compare against the half-life declared by the schema the node actually loaded. +- **Edge cases.** The interval is the independent variable of the property under + test, not a sleep that makes an assertion pass — the comment says so. The + declared value is parsed from the loaded schema rather than hardcoded, so the + assertion cannot degenerate into a tautology. `skip` is declared permanent, and + permanence is the one decay claim this surface *cannot* show, because no + built-in profile reports a skip term at all (CAP-018) — recorded, not faked. +- **Permission boundary.** None. +- **Dependencies.** CAP-016's fixture. +- **Observability.** Both readings, the elapsed time, and both implied + half-lives are attached. +- **Coverage.** `new-test` +- **Evidence.** `tests/e2e/features/10-ranking-semantics.spec.ts` — measured + 7.0007 d and 14.0014 d against declared 7 d and 14 d, from a 4-second window. + +### CAP-018 — A negative signal is durably accepted and then ignored + +- **Area.** `POST /signals` `skip`; all seven resolvable ranking profiles. +- **Business purpose.** A tripwire in the honest direction, in the same spirit as + CAP-015. `VISION.md:187` claims "Negative signals are equal citizens". They are + not, in any shipped profile — and a walkthrough that implied otherwise would be + the exact dishonesty this inventory exists to prevent. +- **Personas.** Application developer; operator. +- **Primary workflow.** Record the target's position under all seven profiles, + write five unit-weight `skip`s, re-read all seven, assert every position is + unchanged and no profile reports a `skip_penalty` term. +- **Edge cases.** Five writes rather than one, so the result cannot be dismissed + as falling below a rounding threshold. The earlier draft of this check skipped + an item that had just been *liked* and concluded nothing, because the + `like_boost` dominated — the target must be signal-neutral. +- **Permission boundary.** None. +- **Dependencies.** CAP-016's fixture. +- **Observability.** Positions before/after per profile, plus the signal-name + vocabulary each profile reports — which is how the undeclared-signal finding + (BUG-019) surfaced. +- **Coverage.** `new-test` +- **Evidence.** `tests/e2e/features/10-ranking-semantics.spec.ts` + +### CAP-019 — Vector search returns the true nearest neighbours + +- **Area.** `POST /vector_search`; the usearch ANN index. +- **Business purpose.** An approximate index that is quietly *wrong* is worse + than a slow exact one: recommendations degrade with no error and no alert. +- **Personas.** Application developer. +- **Primary workflow.** For one probe per category, POST the item's own raw + vector and assert it returns itself first at ≈0 distance, then assert the whole + top-k equals brute-force cosine computed by `tidal-stress`'s `GroundTruth` + over the same corpus, from the same generator that produced the indexed vectors. +- **Edge cases.** The request field is `vector`, not `values` — a wrong name is + answered 422, which reads like a malformed body. The response field is `items`, + not `matches`. Each category owns an otherwise-unoccupied 100-id embedding + cluster so the corpus has real neighbour structure: `recall.rs:94-101` warns + that uniform-random high-dimensional vectors make recall@k measure "impossible + tie-breaking, not index quality". +- **Permission boundary.** None. +- **Dependencies.** CAP-016's fixture; `GroundTruth::from_ids`. +- **Observability.** Returned ids, truth ids, and the self-distance per probe. +- **Coverage.** `new-test` +- **Evidence.** `tests/e2e/features/10-ranking-semantics.spec.ts` — 10/10 exact + match on all four probes; self-distances 0.0148–0.0197 against a 0.05 tolerance. + +### CAP-020 — Rank is a dense sequence on standalone, and duplicated on the cluster + +- **Area.** `rank` on `/feed` and `/search`; `scatter_merge`. +- **Business purpose.** `rank` is the field a client paginates and displays on. + Duplicated ranks silently corrupt any consumer that keys on it. +- **Personas.** Application developer; operator. +- **Primary workflow.** Assert `rank` is exactly `1..n` on a standalone node. + Separately, assert the deployed cluster still returns duplicates, with the + root cause cited and a "delete this tripwire" instruction in the message. +- **Edge cases.** A limit of 1–2 can be answered from one shard group and would + show no duplicate; the cluster check uses 12. The cluster check also asserts + scores are still correctly ordered — that is what localises the fault to the + missing rank stamp rather than to the merge's sort. +- **Permission boundary.** Read-only against the cluster; nothing is written. +- **Dependencies.** CAP-016's fixture (standalone half); the public ingress and + data bearer (cluster half). +- **Observability.** Both rank arrays and both score arrays are attached. +- **Coverage.** `new-test` +- **Evidence.** `tests/e2e/features/10-ranking-semantics.spec.ts` (dense 1..60); + `tests/e2e/features/11-ranking-integrity.spec.ts` (cluster `[1,1,1,2,2,3,4,3,4,5,6,5]`) + ## Intentionally excluded | Capability | Why lower leverage | Alternate evidence | Owner / revisit | @@ -351,3 +478,6 @@ absent by construction, while the operator/data credential split from `388e445` | BUG-002 | CAP-008, CAP-015 | Medium | A metrics scrape either returns series or fails loudly. | `wget --timeout=6` returned zero lines for `tidaldb-2`, indistinguishable from "this pod exports nothing". | Three retries at `--timeout=10` returned metrics every time; peers returned 332/344 series at 6s. | Client timeout too short for the payload on a loaded node — a measurement artifact, not a product defect. | Harness uses a generous timeout and asserts baseline `tidaldb_` series are present before concluding a specific metric is absent. Runbook §9.1 now specifies `--timeout=15` and explains why. | `04-network-isolation.spec.ts` and `09-operator-authority.spec.ts` both green with the baseline-present guard. | fixed | | BUG-003 | CAP-011, CAP-014 | Low | The boot log reflects the current credential state. | Boot log WARNs `TIDAL_ADMIN_KEY is not set` while the gate is live and enforcing. | Pod started 05:41; WARN at 05:42; kubelet materialized the projected secret at 05:51:43; poller loaded it. | Working as designed — the credential poller exists so a key can be added without a restart. The log is a point-in-time record, not current state. | Documented in runbook §9.2 as an explicit "do not trust the boot WARN, test the behaviour" note; the harness allowlists this WARN with that reason. | `06-logs.spec.ts` allowlists it with the rationale; `09-operator-authority.spec.ts` proves the behaviour. | fixed | | BUG-004 | CAP-013 | Medium | The backup check verifies the fleet backup. | Selecting the newest Backup by timestamp picked `restore-canary-longhorn-rwx-*` (20 items, 1 PVB) — it would report "pass" having verified nothing about the fleet. | Ran the runbook's own command as written; it returned the canary. | The selector sorted all Backups instead of filtering to the schedule that the freshness alert actually watches. | Runbook and harness both now filter on `velero.io/schedule-name=velero-fleet-daily`. | Re-ran: selects `velero-fleet-daily-20260823033025`, 3708/3708 items, 48/48 PVBs Completed. | fixed | +| BUG-018 | CAP-018 | High | A declared negative signal can demote an item. | Five `skip` writes changed nothing under any of the seven resolvable profiles, and no profile reports a skip term. | `10-ranking-semantics.spec.ts` records positions before/after for all seven profiles: identical. | The `Penalty` mechanism is fully implemented (`tidal/src/ranking/profile.rs:227`, applied at `tidal/src/ranking/executor/signal_values.rs:183`, labelled `{signal}_penalty` at `tidal/src/ranking/executor/mod.rs:65`) but never populated: every built-in profile is built from `skeleton()`, which sets `penalties: vec![]` (`tidal/src/ranking/builtins.rs:62`), and none of the 27 overrides it. Same shape as the `scatter_merge` defect — a guard present on one path and absent on its sibling. | Not fixed here. Adding a skip penalty to a shipped profile changes ranking behaviour for every consumer and is a product decision, not a test fix. Routed to `@tidal-engineer` with the inertness pinned by CAP-018. | `10-ranking-semantics.spec.ts` asserts the current inertness and fails with an instruction the day a profile applies a penalty. | routed | +| BUG-019 | CAP-018 | Medium | Every signal a profile reads is declared by the schema. | Three built-in profiles read signals this schema does not declare, so those terms are permanently 0: `trending` reads `share_velocity`, `hidden_gems` reads `completion`, `controversial` reads `dislike`. | Signal-name vocabulary captured per profile in `skip-is-inert.json`. | Built-in profiles reference a richer signal set than the deployed schema declares; the engine reports the term rather than rejecting the profile, so the degradation is silent. `trending` effectively ranks on `view_velocity` alone. | Not fixed here. Either the profiles should validate against the loaded schema at registration, or the schema should declare the full set — a product decision. Routed to `@tidal-engineer`. | The vocabulary is recorded as evidence on every run, so a change is visible. | routed | +| BUG-020 | CAP-016 | Low | A candidate scan honours the sort field it declares. | `for_you` declares `CandidateStrategy::Scan { sort_field: "created_at" }` (`tidal/src/ranking/builtins.rs:49`) but a `created_at` metadata value has no effect: an order chosen to match neither id-ascending nor created_at-descending came back strictly id-ascending. | Probed four items with interleaved timestamps; observed order was id-ascending in every case, including numeric-epoch values. | The scan appears to ignore the metadata string entirely and fall back to entity id. | Not fixed here. Recorded so nobody "fixes" a monotonous fixture feed by writing timestamps that cannot work; the fixture contract documents it at the point of temptation. | Documented in `tests/e2e/app/fixture-contract.ts`. | routed | diff --git a/demo/capture-manifest.json b/demo/capture-manifest.json index fb33918..16652df 100644 --- a/demo/capture-manifest.json +++ b/demo/capture-manifest.json @@ -40,6 +40,23 @@ "audienceVerdict": "perfect", "auditStatus": "pass" }, + { + "id": "CAP-016-feed-reorder", + "capabilityId": "CAP-016", + "testId": "workflows/feed-app.demo.spec.ts :: feed-app product surface :: CAP-016 a signal write reorders the feed immediately", + "file": "captures/CAP-016-feed-reorder.png", + "expected": "The same query, before and after one like: the liked item moves from last to first, with its like_boost visible", + "businessPurpose": "A signal write changes the order with no ETL in between \u2014 the product thesis, not the deployment", + "personas": [ + "cluster operator", + "application developer" + ], + "width": 1600, + "height": 900, + "contentHash": "sha256:ede7ca3fab758d46cabdf1185f6645bc3055c7e8bbb25e380f8cff509d2136b4", + "audienceVerdict": "perfect", + "auditStatus": "pass" + }, { "id": "CAP-008-network-isolation", "capabilityId": "CAP-008", diff --git a/demo/public/captures/CAP-016-feed-reorder.png b/demo/public/captures/CAP-016-feed-reorder.png new file mode 100644 index 0000000..8f22481 Binary files /dev/null and b/demo/public/captures/CAP-016-feed-reorder.png differ diff --git a/demo/src/scenes.ts b/demo/src/scenes.ts index 8d39aef..8d22141 100644 --- a/demo/src/scenes.ts +++ b/demo/src/scenes.ts @@ -76,6 +76,17 @@ export const scenes: Scene[] = [ caption: 'Same path, three credentials. Then a write a majority of nodes acknowledged — the one claim a health endpoint cannot fake.', }, + { + kind: 'proof', + id: 'B3a-feed-reorder', + seconds: 8, + rung: 'need', + capabilityId: 'CAP-016', + captureId: 'CAP-016-feed-reorder', + heading: 'And the product it exists to be', + caption: + 'Everything before this proves the deployment answers. This is what it is for: one like, the same query again, and the item is first. No ETL between the write and the read — a local node on fixture data, so the claim is the mechanism, not the corpus.', + }, { kind: 'proof', id: 'B4-isolation', diff --git a/demo/storyboard.md b/demo/storyboard.md index 9f21093..236ff5f 100644 --- a/demo/storyboard.md +++ b/demo/storyboard.md @@ -14,6 +14,7 @@ the rung it serves. A beat serving no rung is cut. | B1 opening | — | need | operator | Orientation: what this is and what it will prove | Title card | — | — | "tidalDB deploy verification. Three voters on `orchard9-k3sf`, one public endpoint, and a runbook an operator can walk. Every number that follows came from a command that ran against the live cluster." | 9 | | B2 convergence | CAP-002 | need | operator | Quorum with fault tolerance actually exists, rather than being inferred from pod readiness | Per-node `lag=0`, no reseed, agreed leader per shard group | `01-cluster-convergence.spec.ts :: every node reports zero lag…` | `CAP-002-convergence` | "Each node asked for its own view. A pod can be Ready while its replication is stalled — that is what the reseed livelock exploited." | 7 | | B3 boundary + write | CAP-005 CAP-006 | need | operator | The strongest single proof: full stack works AND the data plane is closed | 401, 401, 200, then 201 quorum-acked | `03-auth-boundary.spec.ts :: a quorum-acked write is committed…` | `CAP-006-quorum-write` | "Same path, three credentials. Then a write a majority of nodes acknowledged — the one claim a health endpoint cannot fake." | 8 | +| B3a feed reorder | CAP-016 | need | developer, operator | The product thesis: a signal write changes the order of the next query | Same 5-item feed before and after one `like`; the liked item goes 5 -> 1 with `like_boost 2.000` | `feed-app.demo.spec.ts :: CAP-016 a signal write reorders the feed immediately` | `CAP-016-feed-reorder` | "Everything before this proves the deployment answers. This is what it is for: one like, the same query again, and the item is first. No ETL between the write and the read — a local node on fixture data, so the claim is the mechanism, not the corpus." | 8 | | B4 isolation | CAP-008 | need | operator | Least-privilege network access without blinding monitoring | Refused from a foreign namespace; 332 series to the scraper | `04-network-isolation.spec.ts :: a pod in an unrelated namespace is refused…` | `CAP-008-network-isolation` | "The refusal is the point. A policy that blocks everything is an outage; one that blocks nothing is theatre." | 7 | | B5 authority | CAP-014 | want | operator | Blast radius of a leaked application key stops at data, not cluster topology | Data bearer 403, admin bearer authorised | `09-operator-authority.spec.ts :: the data-plane credential is refused…` | `CAP-014-authority` | "403, not 401 — the key is valid, it just cannot remove a cluster member." | 6 | | B6 dashboard | CAP-010 | want | operator | The first surface opened during an incident actually shows the cluster | Cluster health OK, reseed none, 33.3K vectors, populated latency charts | `05-metrics-dashboard.spec.ts :: an operator opening the dashboard sees populated charts` | `CAP-010-dashboard` | "The board an operator opens at 3am. Health, reseed state, corpus size, and per-node latency — scoped to the cluster." | 8 | @@ -23,7 +24,7 @@ the rung it serves. A beat serving no rung is cut. | **B9 the drift catch** | CAP-014 | **dream** | operator | Verification that audits its own documentation instead of drifting from it | The committed doc text, the actually-running image, and the live 403/200 probe, side by side | `deploy-verification.demo.spec.ts :: the harness corrected its own runbook` | `CAP-014-drift` | "The runbook said the credential split was 'not active yet — requires an image roll'. On its first run the harness read the live image and the live secret, and proved the gate was already enforcing. The document was wrong. The harness said so, before anyone noticed." | 10 | | B10 recap | — | need | operator | Restate the needs as met, including the gaps | Closing card | — | — | "32 checks green against the live deployment. Two committed features are absent from the running image, and the suite asserts that absence deliberately — so the day it changes, it says so." | 7 | -**Total: 82 s** across 11 beats. +**Total: 90 s** across 12 beats. ## Ladder coverage @@ -31,9 +32,10 @@ Two-way mapping, per the audience protocol. | Brief entry | Served by | | --- | --- | -| Need — evidence is real, commands visible | B1, B2, B3, B4 (every capture shows the command that produced it) | +| Need — evidence is real, commands visible | B1, B2, B3, B3a, B4 (every capture shows the command or the surface that produced it) | | Need — write proven committed by quorum | B3 | | Need — every boundary shows its denial | B3 (401×2), B4 (refused), B5 (403) | +| Need — the database does the thing it exists to do | B3a (one signal write reorders the feed, immediately) | | Need — what is not verified is stated | B9a (evidence on screen), B10 (recap) | | Want — one command a different person can run | B1, B10 | | Want — operator/data authority visibly enforced | B5 | @@ -51,4 +53,11 @@ pass were fixed at the capture layer and re-audited. where the bisect evidence can be read, not in a 75-second walkthrough where a red frame would read as a deploy failure. - **Clicking, scrolling, or navigating.** The dashboard beat shows the board, not - the act of opening it. + the act of opening it. B3a is the one place a click happens, and even there the + frame shows the two resulting states rather than the gesture. +- **A `skip` beat.** The suite proves `skip` is durably accepted and then ignored + by all 27 built-in ranking profiles (no profile populates `penalties`), so there + is no order change to show. That finding belongs in the written record, not in a + frame that would imply a working demotion. +- **A search or vector-search beat.** Both work, but neither is the thesis, and a + 75-second walkthrough that shows four surfaces shows none of them. diff --git a/demo/visual-audit.md b/demo/visual-audit.md index 4726896..139b996 100644 --- a/demo/visual-audit.md +++ b/demo/visual-audit.md @@ -6,22 +6,23 @@ written in the **actual decision-maker's** voice (Jordan Washburn). - Build revision: see `capture-manifest.json.buildRevision` - Verified image: `registry.threesix.ai/tidal/server:m12-admin-gate-20260823@sha256:6e220060…` -- Render: `demo/out/deploy-verification.mp4` — 82.05 s, 1920×1080, 30 fps, h264, 2460 frames -- Regression suite at time of capture: **32 passed** -- Demo capture suite: **9 passed** (each asserts before it photographs) +- Render: `demo/out/deploy-verification.mp4` — 90.05 s, 1920×1080, 30 fps, h264, 2700 frames +- Regression suite at time of capture: **34 passed** (32 deployment + 2 cluster ranking tripwires) +- Hermetic ranking-semantics suite: **5 passed** (`npm run test:e2e:semantics`, no cluster required) +- Demo capture suite: **10 passed** (each asserts before it photographs) ## 1. Programmatic preflight | Check | Result | | --- | --- | -| Every manifest file exists and decodes | pass — 9/9 | -| Filesystem inventory equals ledger count | pass — 9 promoted, 9 rows, 0 unclassified | -| Dimensions match declared viewport or documented crop | pass — 8 × 1600×900, 1 × 1600×502 (documented crop) | +| Every manifest file exists and decodes | pass — 10/10 | +| Filesystem inventory equals ledger count | pass — 10 promoted, 10 rows, 0 unclassified | +| Dimensions match declared viewport or documented crop | pass — 9 × 1600×900, 1 × 1600×502 (documented crop) | | Files non-empty, not near-uniform blanks | pass — 64 KB–187 KB | | Capture IDs and filenames unique | pass | | No unexpected duplicates across distinct proof states | pass | -| Source test green | pass — 9 demo capture tests green; each asserts before it photographs | -| Secrets / tokens / local paths absent | pass — `redact()` masks every known secret before render; spot-checked all 9 | +| Source test green | pass — 10 demo capture tests green; each asserts before it photographs | +| Secrets / tokens / local paths absent | pass — `redact()` masks every known secret before render; spot-checked all 10. CAP-016 additionally carries no credential by construction: the page reaches the node only through the harness proxy, which injects auth server-side | ## 2. Per-image review @@ -40,10 +41,11 @@ that was silently overwritten is not an audit. | `CAP-014-authority` | correct | complete | ok (was: dead space) | 403 then 200 | clean | none | **perfect** (was `slop` weak-design) | BUG-011 | | `CAP-014-drift` | correct | complete | ok | git text + live image + 403/200 | clean | none | **perfect** | BUG-013 | | `CAP-015-inert` | correct | complete | ok (was: dead space) | 332 baseline / 0 http | clean | none | **perfect** (was `slop` weak-design) | BUG-011 | +| `CAP-016-feed-reorder` | correct | complete | ok (was: footer clipped, hover artefact) | 5-item feed before/after, 5→1 with `like_boost 2.000` | clean — no bearer reaches the page by construction | none | **perfect** (was `slop` twice: cut-off honesty footer, then a stray `:hover` ring) | BUG-016, BUG-017 | ### Non-promoted images -None. Nine images were produced by the capture run and nine were promoted; no +None. Ten images were produced by the capture runs and ten were promoted; no failure or debug images were generated because every capture test passed on the run that produced the promoted set. @@ -58,16 +60,17 @@ frames extracted from the **encoded MP4** (not re-renders) at | f120 opening | purpose, personas, scope readable | pass | | f375 B2, f600 B3, f825 B4, f1020 B5 | proof legible, entrance settled | pass | | f1230 B6 dashboard | legible at delivery resolution | pass after BUG-010 | +| f840 B3a feed reorder | before/after both legible, capture at scale 1.0, honesty footer visible | pass after BUG-016 | | f1440 B7, f1635 B8 | proof legible | pass | | f1890 B9 dream | image agrees with caption | pass after BUG-013 | | f2145 B10 recap | copy matches what was shown | pass | -| All 10 scene boundaries | no black/white/empty frame | pass after BUG-014 — 0 empty frames | -| Mean luma at all 10 boundaries | 17.49–23.21, smooth progression | pass (was flat 13.00 = bare background) | -| Contact sheet, 30 samples across 75 s | order matches storyboard, no repeats or stale content | pass | +| All 11 scene boundaries | no black/white/empty frame | pass after BUG-014 — 0 empty frames | +| Mean luma across all 11 boundary triplets (f-1, f, f+1) | continuous, no isolated dip | pass — every triplet varies by ≤1 (e.g. 5/4/4 at f960); the BUG-014 signature was an isolated collapse to bare background | +| Contact sheet, 30 samples across 90 s | order matches storyboard, no repeats or stale content | pass | -Captures are presented at scale ≥ 1.0 — the 1600×900 terminal panels render -1:1 and the 1600×502 dashboard crop renders at 1.10 — so no evidence is -downscaled. +Captures are presented at scale ≥ 1.0 — the 1600×900 panels render 1:1 (CAP-016 +measured at 1597/1600 px in-frame) and the 1600×502 dashboard crop renders at +1.10 — so no evidence is downscaled. ## 4. Walk the render as Jordan Washburn @@ -78,14 +81,15 @@ One row per distinct screen a viewer reads. | B1 opening | 0–9 s | "Right, it names the cluster and the image up front. And it says every number came from a real command — that's the claim I actually care about." | neutral-orientation | | B2 convergence | 9–16 s | "Per-node, not the aggregate. Good — the aggregate is exactly what lied to me last week. lag=0 on all nine group-replicas and they agree on the leaders." | earns-interest | | B3 boundary + write | 16–24 s | "401, 401, 200, then a 201 quorum ack. That single 201 is worth more than the rest of the page — it means DNS, TLS, the gateway, auth and Raft all worked in one request." | earns-interest | -| B4 isolation | 24–31 s | "Connection refused from gitea, 332 series to the scraper. That's the pair I'd want — it proves the policy is real without blinding monitoring." | earns-interest | -| B5 authority | 31–37 s | "403 not 401. So a leaked app key can't remove a member. That's the exposure I was worried about and it's closed." | earns-interest | -| B6 dashboard | 37–45 s | "That's the board I'd actually open. Health OK, reseed none, 33.3K vectors, and the legends finally say which node is which." | earns-interest | -| B7 recovery | 45–51 s | "3708/3708 and 48 of 48 volumes, selected by the schedule label. Good — picking the newest backup would have grabbed a canary." | earns-interest | -| B8 blind spot | 51–58 s | "It says NO REPORT instead of inventing 13 million events of lag. And it admits the exit code makes the deploy gate unusable. I trust a tool that tells me that." | earns-interest | -| B9a inert | 58–65 s | "It shows me what it cannot check yet, with the scrape count proving the scrape actually ran. That is the opposite of a green wall." | earns-interest | -| B9 dream | 65–75 s | "Wait — the doc I wrote said that was pending an image roll, and the harness proved it was already live. It caught my own documentation being wrong before I did. I want this running after every deploy." | earns-interest | -| B10 recap | 75–82 s | "32 green, and it names what it does not verify. That's the version I'd hand to someone else." | neutral-orientation | +| B3a feed reorder | 24–32 s | "So that’s what it is actually for. One like, same query, and it goes from fifth to first with the boost shown next to it. And it says local node and fixture catalog, so it isn’t pretending that’s production data." | earns-interest | +| B4 isolation | 32–39 s | "Connection refused from gitea, 332 series to the scraper. That's the pair I'd want — it proves the policy is real without blinding monitoring." | earns-interest | +| B5 authority | 39–45 s | "403 not 401. So a leaked app key can't remove a member. That's the exposure I was worried about and it's closed." | earns-interest | +| B6 dashboard | 45–53 s | "That's the board I'd actually open. Health OK, reseed none, 33.3K vectors, and the legends finally say which node is which." | earns-interest | +| B7 recovery | 53–59 s | "3708/3708 and 48 of 48 volumes, selected by the schedule label. Good — picking the newest backup would have grabbed a canary." | earns-interest | +| B8 blind spot | 59–66 s | "It says NO REPORT instead of inventing 13 million events of lag. And it admits the exit code makes the deploy gate unusable. I trust a tool that tells me that." | earns-interest | +| B9a inert | 66–73 s | "It shows me what it cannot check yet, with the scrape count proving the scrape actually ran. That is the opposite of a green wall." | earns-interest | +| B9 dream | 73–83 s | "Wait — the doc I wrote said that was pending an image roll, and the harness proved it was already live. It caught my own documentation being wrong before I did. I want this running after every deploy." | earns-interest | +| B10 recap | 83–90 s | "32 green, and it names what it does not verify. That's the version I'd hand to someone else." | neutral-orientation | No `fails` rows. The dream beat's thought shows genuine surprise and desire, so it earns its rung. @@ -100,6 +104,7 @@ un-muted passes are identical and captions carry the whole narrative. | --- | --- | | Opening purpose and personas readable | pass — 9 s hold | | Each proof state legible long enough | pass — 6–10 s, dense screens get the longer holds | +| Two-state screen readable as one comparison | pass — B3a stacks before/after at native scale rather than downscaling both to sit side by side | | Dense screens get more time | pass — dream 10 s, write 8 s, dashboard 8 s vs authority 6 s | | Captions agree with the visible outcome | pass after BUG-013 | | Transitions smooth, sections clear | pass after BUG-014 | @@ -119,6 +124,8 @@ Product/test defects found during verification are in | BUG-012 | `CAP-012-tidalctl` | medium | Colour does not imply a meaning the content lacks | "Exit code 2" rendered green, i.e. as a success, when it is the finding | Verdict line defaulted to the positive colour role | Marked the block `negative: true` (amber) and rewrote the text to name the consequence | Re-captured and re-inspected | verified | | BUG-013 | scene B9 (dream) | **blocker** | Scene title, claim, and image agree | The dream caption described the runbook's stale claim and the probe that disproved it, while the image showed the unrelated inert-features panel | Beat reused an existing capture instead of one built for the claim | New `CAP-014-drift` capture showing the committed doc text from `git show`, the actually-running image, and the live 403/200 probe side by side | Re-captured, inspected, frame f1890 re-audited, walk-the-render row rewritten | verified | | BUG-014 | every scene boundary | **blocker** | No empty frame between scenes | The video blinked to bare background for one frame at all 9 boundaries | Remotion Sequences do not overlap; each scene faded out over its last 8 frames while the next faded in from its own frame 0, so both sat at opacity 0 on the boundary | Removed the fade-out and started each Sequence `OVERLAP=10` frames early running long, producing a true 333 ms cross-dissolve. End times unchanged, so every storyboard hold is preserved | Re-rendered; boundary luma went from flat 13.00 (bare background) to 17.5–23.2 with smooth progression; boundary frames re-inspected | verified | +| BUG-016 | `CAP-016-feed-reorder` | **blocker** | The frame states its environment | Two six-row lists plus labels totalled ~922 px in a 900 px frame, pushing the “local standalone node · 60-item fixture catalog” footer off the bottom and clipping the last row | Composition height was never checked against the frame; the footer is `margin-top:auto` so it was the first thing squeezed out | Reduced each half from six rows to five. Explicitly NOT solved by scaling the lists down — that is the BUG-010 mistake, and an unlabelled fixture screenshot reads as production | Re-captured, re-inspected: footer present, both lists complete, lists still 1:1 | verified | +| BUG-017 | `CAP-016-feed-reorder` | low | No control looks meaningful unless it is | A blue-ringed `Like` button appeared on row 5 — an unrelated row — in the after state | The mouse physically stays where it clicked; after the re-render a DIFFERENT row’s button occupies that pixel and picks up `:hover`. First fix attempt blurred focus, which changed nothing because it was hover, not focus | `page.mouse.move(0, 0)` before the screenshot, plus a blur for the focus case | Re-captured and re-inspected: no ring on any row | verified | | BUG-015 | proof scenes | low | No duplicated headings | Remotion drew a scene heading above a capture that already carried its own title, producing a card-in-card with two competing titles | Composition and capture both owned a heading | Removed the heading from the proof scene; Remotion now owns only the capability badge, rung, and audience caption. Reclaimed 66 px for the evidence | Re-rendered and re-inspected f600, f1890 | verified | ## Approval @@ -126,7 +133,7 @@ Product/test defects found during verification are in - Every promoted image, card, and clip has an individual review record with an audience verdict of `perfect`. No `acceptable-with-note` verdicts exist, so no screen was promoted on a soft pass. -- All blocker/high defects (BUG-010, BUG-013, BUG-014) are fixed and verified. +- All blocker/high defects (BUG-010, BUG-013, BUG-014, BUG-016) are fixed and verified. - Root causes are evidence-backed; no unresolved evidence gaps. - Every scene proof and transition sample approved. - Walk-the-render ledger complete for every hold and card, no `fails` rows, and diff --git a/docs/runbooks/deploy-verification.md b/docs/runbooks/deploy-verification.md index 2135b65..f6a191e 100644 --- a/docs/runbooks/deploy-verification.md +++ b/docs/runbooks/deploy-verification.md @@ -23,11 +23,36 @@ cargo build -p tidalctl # section 7 needs the binary npm run test:e2e:list # discovery: syntax, imports, registration npm run test:e2e:smoke # cluster plane + public plane + a quorum write -npm run test:e2e # the whole runbook, ~40 s +npm run test:e2e # the whole runbook, 34 checks, ~55 s ``` Credentials are read from the cluster by `globalSetup`, so nothing is pasted into a shell. A missing prerequisite fails loudly rather than skipping a check. + +### The product, not just the deployment + +Everything in this runbook verifies that the deployment *answers*. None of it +verifies that the database does the thing it exists to do, so there is a second, +separate suite for ranking semantics: + +```bash +npm run test:e2e:semantics # 5 checks, ~14 s, NO cluster required +npm run app:dev # open the same app by hand and click Like +``` + +It boots a throwaway standalone node, seeds a 60-item fixture catalog with +deterministic vectors from `tidal-stress`'s own generator, and asserts that a +signal write reorders the next query, that decay is applied at the declared +half-life, and that ANN results equal brute-force cosine. It has its own config +(`playwright.semantics.config.ts`) precisely because it must run with no cluster +and no credentials — `npm run test:all` runs both suites. + +It deliberately never touches the deployed cluster: `skip` is declared +`permanent: true`, so seeding signals into the live corpus would be +irreversible. Two of its five checks report a product gap rather than a success +(`skip` is inert under all 27 built-in profiles; see `demo/capability-inventory.md` +BUG-018), and one cluster-targeted tripwire pins the duplicated `rank` +(BUG-020) with its root cause in `scatter_merge`. The HTML report at `playwright-report/` carries the transcript of every command the suite ran, which is the evidence trail this document used to describe in prose. diff --git a/package.json b/package.json index ae2f31e..0234f72 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { "name": "tidaldb-deploy-verification", "private": true, + "type": "module", "version": "0.0.0", "description": "Playwright evidence harness for docs/runbooks/deploy-verification.md", "scripts": { @@ -8,6 +9,9 @@ "test:e2e:smoke": "playwright test --config playwright.config.ts tests/e2e/smoke.spec.ts", "test:e2e:list": "playwright test --config playwright.config.ts --reporter=list --list", "test:e2e:ui": "playwright test --config playwright.config.ts --ui", + "test:e2e:semantics": "playwright test --config playwright.semantics.config.ts", + "test:all": "npm run test:e2e:semantics && npm run test:e2e", + "app:dev": "node --experimental-strip-types tests/e2e/app/dev.ts", "test:demo": "playwright test --config playwright.demo.config.ts", "test:demo:list": "playwright test --config playwright.demo.config.ts --reporter=list --list", "demo:preflight": "node --experimental-strip-types demo/preflight.ts", diff --git a/playwright.config.ts b/playwright.config.ts index f7fe713..33b1c00 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -25,7 +25,10 @@ import { defineConfig, devices } from '@playwright/test'; export default defineConfig({ testDir: './tests/e2e', - testIgnore: ['**/demo/**'], + // The demo suite has its own config; the ranking-semantics suite has its own + // too (playwright.semantics.config.ts) because it is hermetic and must not be + // gated on this config's cluster prerequisites. + testIgnore: ['**/demo/**', '**/features/10-ranking-semantics.spec.ts'], globalSetup: './tests/e2e/support/env-bootstrap.ts', timeout: 120_000, expect: { timeout: 20_000 }, diff --git a/playwright.semantics.config.ts b/playwright.semantics.config.ts new file mode 100644 index 0000000..696e9cd --- /dev/null +++ b/playwright.semantics.config.ts @@ -0,0 +1,69 @@ +import { defineConfig, devices } from '@playwright/test'; + +/** + * Hermetic config for the ranking-semantics suite. + * + * These checks prove tidalDB's product thesis — that writing a signal changes + * the order of a query, immediately — against a throwaway standalone node this + * suite boots and seeds itself. Nothing here touches the deployed cluster, and + * nothing here needs a credential, a kubeconfig, or a network. + * + * **Why a separate config rather than a project inside `playwright.config.ts`.** + * That config's `globalSetup` (`tests/e2e/support/env-bootstrap.ts`) hard-fails + * when kubectl cannot reach the cluster, and it must: sourcing credentials from + * the cluster is what stops a check silently skipping. Two things rule out + * gating it per-project: + * + * 1. `FullConfig.projects` handed to `globalSetup` is NOT filtered by + * `--project` (measured: selecting one of two projects still reports both), + * so setup cannot tell whether a cluster-targeted test was even selected. + * 2. `globalSetup` publishes credentials into `process.env` of the MAIN process + * so forked workers inherit them. A Playwright setup *project* runs inside a + * worker, where that propagation does not happen. + * + * Separate configs keep each suite's prerequisites honest: this one requires + * nothing, and the regression config keeps failing loudly when the cluster is + * unreachable. Same split the demo suite already uses. + */ + +export default defineConfig({ + testDir: './tests/e2e/features', + testMatch: ['10-ranking-semantics.spec.ts'], + timeout: 180_000, + expect: { timeout: 20_000 }, + forbidOnly: !!process.env.CI, + + // Each test boots its own node, so a retry would hide a genuine ordering bug + // behind a second roll of the dice — the same reasoning as the regression + // config. If a ranking assertion is unstable, the assertion is wrong. + retries: 0, + workers: 1, + fullyParallel: false, + + reporter: process.env.CI + ? [ + ['github'], + ['html', { open: 'never', outputFolder: 'playwright-report-semantics' }], + ['junit', { outputFile: 'test-results/playwright-semantics-junit.xml' }], + ] + : [['list'], ['html', { open: 'never', outputFolder: 'playwright-report-semantics' }]], + + outputDir: 'test-results/playwright-semantics', + + use: { + // No baseURL: the app's origin is allocated per test by the harness, so a + // config-level default would be a lie. + actionTimeout: 20_000, + navigationTimeout: 45_000, + trace: 'retain-on-failure', + screenshot: 'only-on-failure', + video: 'retain-on-failure', + }, + + projects: [ + { + name: 'semantics', + use: { ...devices['Desktop Chrome'] }, + }, + ], +}); diff --git a/tests/e2e/app/dev.ts b/tests/e2e/app/dev.ts new file mode 100644 index 0000000..4618a35 --- /dev/null +++ b/tests/e2e/app/dev.ts @@ -0,0 +1,38 @@ +/** + * `npm run app:dev` — boot the content-feed app and leave it running. + * + * The whole point of this app is that a human clicks Like and watches the order + * change, so there has to be a one-command way to open it. It drives the exact + * same `startApp()` lifecycle the assertions use, so what you click is what the + * spec ran against. + */ + +import { rmSync } from 'node:fs'; +import { startApp } from './harness.ts'; + +const app = await startApp({ verbose: true }); + +/** + * Synchronous last resort. The graceful path below removes the data dir, but an + * `exit` handler is the only thing that still runs if that path throws part-way + * — and a stale multi-megabyte index dir in $TMPDIR is exactly the kind of litter + * nobody goes looking for. (Nothing survives SIGKILL; that is the OS's problem.) + */ +process.on('exit', () => rmSync(app.dataDir, { recursive: true, force: true })); + +console.log(''); +console.log(` content feed ${app.url}`); +console.log(` tidalDB node ${app.nodeUrl}`); +console.log(` catalog ${Object.keys(app.groundTruth.neighboursByItem).length} items seeded`); +console.log(''); +console.log(' Ctrl-C to tear down the node and remove its data dir.'); +console.log(''); + +const shutdown = async (signal: string) => { + console.log(`\n[app] ${signal} — closing`); + await app.close(); + process.exit(0); +}; + +process.once('SIGINT', () => void shutdown('SIGINT')); +process.once('SIGTERM', () => void shutdown('SIGTERM')); diff --git a/tests/e2e/app/fixture-contract.ts b/tests/e2e/app/fixture-contract.ts new file mode 100644 index 0000000..9cb0f4a --- /dev/null +++ b/tests/e2e/app/fixture-contract.ts @@ -0,0 +1,186 @@ +/** + * The content-feed app's fixture contract — the single home for the catalog and + * for the shape of the ground truth the Rust fixture binary emits. + * + * Everything else imports this: the harness (to seed and to serve + * `/catalog.json`), the page (via that endpoint), and the specs (to pick probe + * items and to read the oracle). Without one contract, five consumers each + * decide independently what an item is and which ids exist. + * + * Data and types only. No I/O, no HTTP, and — deliberately — no ranking + * arithmetic: `CODING_GUIDELINES.md:88` puts scoring in a named ranking profile + * inside the database, and this app exists to demonstrate exactly that. + */ + +/** The embedding width the schema declares (`k8s/cluster/schema-configmap.yaml:51`). */ +export const EMBEDDING_DIM = 1536; + +/** + * Items per category. 15 keeps a whole category inside one 100-id embedding + * cluster; crossing 100 would spill into the next cluster and silently destroy + * the ground-truth separation the oracle depends on. + */ +export const ITEMS_PER_CATEGORY = 15; + +/** + * Each category owns ONE embedding cluster. + * + * `embedding_for` assigns cluster = id / 100 (`tidal-stress/src/recall.rs:104`), + * so a category's ids must share a 100-id band. These bands were probed against + * the live corpus and are unoccupied: the nearest existing vector sits at + * distance ~2.04 while intra-cluster neighbours sit at ~0.435. That 4.7x + * separation is what makes a brute-force top-k over just these 60 items the + * *global* top-k — no 6 GB oracle required. + * + * The `999_000_0xx` band is deliberately avoided: earlier verification probes + * already wrote items there, so its cluster is polluted. + */ +export const CATEGORIES = [ + { name: 'Field recordings', idBase: 900_000_000 }, + { name: 'Analog synthesis', idBase: 900_000_100 }, + { name: 'Choral & sacred', idBase: 900_000_200 }, + { name: 'Free jazz', idBase: 900_000_300 }, +] as const; + +export type CatalogItem = { + entityId: number; + title: string; + category: string; +}; + +/** + * Titles per category, in id order. These are the only prose a viewer reads on + * screen, so they are real and human rather than `post-1` — a bare id would make + * the recording read as a developer artifact rather than a product. + */ +const TITLES: Record = { + 'Field recordings': [ + 'Harbour Ice at Thaw', + 'Nightjars, Dungeness Shingle', + 'Tram Depot, 04:40', + 'Rain on a Zinc Roof', + 'Cicadas Before the Storm', + 'Understory, Monsoon Week', + 'Fog Signal, Outer Channel', + 'Beehive Interior, Midsummer', + 'Grain Elevator, Idling', + 'Salt Flats, Wind Only', + 'Cathedral Steps at Dusk', + 'Snowmelt Under Rock', + 'Ferry Wake, North Passage', + 'Sparrows in a Bus Shelter', + 'Powerlines, Dry Heat', + ], + 'Analog synthesis': [ + 'Ladder Filter Sketch No. 4', + 'Two Oscillators, Slight Detune', + 'Ring Modulator Study', + 'Tape Delay, Self-Oscillating', + 'Sample and Hold Lullaby', + 'Patchbay at Low Voltage', + 'Sawtooth Descending', + 'Envelope Follower Duet', + 'Noise Source, Filtered Slowly', + 'Sequencer Drift', + 'Bucket Brigade Chorus', + 'Sync Lead, Held Open', + 'Resonance at the Edge', + 'Pulse Width Breathing', + 'Cold Start, Warm Bias', + ], + 'Choral & sacred': [ + 'Vespers for a Small Room', + 'Antiphon in Two Voices', + 'Kyrie, Winter Setting', + 'Plainchant, Reconstructed', + 'Nunc Dimittis at Compline', + 'Motet for Eight Parts', + 'Requiem Fragment, Anonymous', + 'Magnificat in the Old Style', + 'Litany with Drone', + 'Alleluia, Second Mode', + 'Lament for Holy Saturday', + 'Te Deum, Village Choir', + 'Canticle of the Three', + 'Hymn at the Lighting of Lamps', + 'Psalm 130, Unaccompanied', + ], + 'Free jazz': [ + 'Ashfall Quartet, Take 3', + 'Blindfold Duet', + 'Circular Breathing Suite', + 'Downtown Loft, Second Set', + 'Extended Technique No. 9', + 'Fractured Standard', + 'Glass Reeds', + 'Horns Against a Wall', + 'Inside the Piano', + 'Junk Percussion Trio', + 'Kinetic Sculpture Session', + 'Long Tones, No Meter', + 'Multiphonic Conversation', + 'Nine Bells and a Bass', + 'Overblown Ballad', + ], +}; + +/** + * The 60-item catalog: 4 categories x 15 items, ids inside each category's band. + * + * Insertion order is category-grouped, and the feed's initial order is + * id-ascending, so the first page shows one category. That is not a layout bug: + * a freshly seeded corpus has no signals, every item ties on score, and + * `for_you`'s tie-break is the entity id. Interleaving was tried and reverted — + * the `for_you` candidate scan declares `sort_field: "created_at"` + * (`tidal/src/ranking/builtins.rs:49`) but ignores a `created_at` metadata value + * entirely (measured: an order matching neither id-ascending nor + * created_at-descending came back strictly id-ascending), so insertion order and + * timestamps have no observable effect. The page stops looking uniform the + * instant a signal is written, which is the point. + */ +export const CATALOG: readonly CatalogItem[] = CATEGORIES.flatMap(({ name, idBase }) => + (TITLES[name] ?? []).map((title, offset) => ({ + entityId: idBase + offset, + title, + category: name, + })), +); + +/** Id -> item, for the id→title join the page performs (`/feed` returns no metadata). */ +export const CATALOG_BY_ID: Record = Object.fromEntries( + CATALOG.map((item) => [String(item.entityId), item]), +); + +/** + * Items the spec probes for the ANN assertion — one per category, so every + * cluster is covered rather than just the first. + */ +export const PROBE_IDS = [900_000_007, 900_000_107, 900_000_207, 900_000_307] as const; + +/** + * What `tidal-stress`'s `feed-fixture` binary writes. A generated artifact, never + * committed: a checked-in oracle drifts from its generator the first time either + * one changes, and the drift is silent. + */ +export type FixtureGroundTruth = { + dim: number; + /** entityId -> its true cosine-ranked neighbour ids, nearest first. */ + neighboursByItem: Record; + /** + * entityId -> its raw 1536-float vector, for the handful of items the spec + * probes. The ANN assertion has to POST a query vector, and the vectors + * otherwise exist only inside the Rust binary. Emitting a few probes rather + * than all 60 keeps this artifact ~100 KB instead of ~1 MB. + */ + probeVectors: Record; + /** + * How far an item's own vector may land from itself and still count as "zero". + * Non-zero purely because the vector round-trips through f32 JSON and the + * engine re-normalises on write. + */ + selfDistanceTolerance: number; +}; + +/** The three signal names the schema declares. Nothing else exists. */ +export const SIGNALS = ['view', 'like', 'skip'] as const; +export type SignalName = (typeof SIGNALS)[number]; diff --git a/tests/e2e/app/harness.ts b/tests/e2e/app/harness.ts new file mode 100644 index 0000000..b4ae4aa --- /dev/null +++ b/tests/e2e/app/harness.ts @@ -0,0 +1,516 @@ +/** + * The content-feed app's lifecycle, in one place. + * + * `startApp()` boots a throwaway standalone tidalDB, seeds the fixture catalog, + * serves the page plus a token-injecting `/api` proxy, and hands back a handle + * whose `close()` unwinds all of it. The semantic spec, the demo capture and + * `npm run app:dev` all drive the identical lifecycle, so what a human sees by + * hand is what the assertions ran against. + * + * Two rules shape the design: + * + * - **Relative `/api` only.** `.sdlc/guidance.md:165` forbids a hardcoded + * `http://localhost:PORT` in frontend code and prescribes a dev-server proxy. + * Independently, a bearer must never reach a browser — a page holding the key + * leaks it to anyone who opens devtools, and the demo capture would photograph + * it. The proxy satisfies both: the page calls `/api/feed`, the server adds + * `Authorization`, the key stays server-side. + * - **Poll, never sleep.** A fixed sleep either wastes time or produces the + * empty-body false negative that reads exactly like a dead node. This is the + * same discipline as `tests/e2e/support/cluster.ts:154` `portForward`. + * + * It targets a LOCAL standalone node, never the deployed cluster: `skip` is + * declared `permanent: true`, so seeding signals into production would be an + * irreversible mutation of the live corpus. + */ + +import { spawn, type ChildProcess } from 'node:child_process'; +import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http'; +import { createConnection } from 'node:net'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { extname, join, resolve, sep } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; + +import { run } from '../support/cluster.ts'; +import { redact } from '../support/env.ts'; +import { + CATALOG, + EMBEDDING_DIM, + PROBE_IDS, + type FixtureGroundTruth, +} from './fixture-contract.ts'; + +const REPO_ROOT = resolve(import.meta.dirname, '../../..'); +const PUBLIC_DIR = join(import.meta.dirname, 'public'); +const SCHEMA_CONFIGMAP = join(REPO_ROOT, 'k8s/cluster/schema-configmap.yaml'); +const SERVER_BIN = join(REPO_ROOT, 'target/debug/tidal-server'); +const FIXTURE_BIN = join(REPO_ROOT, 'target/debug/feed-fixture'); + +const BOOT_TIMEOUT_MS = 60_000; +const BUILD_TIMEOUT_MS = 900_000; +const SEED_TIMEOUT_MS = 120_000; +const KILL_GRACE_MS = 3_000; + +/** + * Ask the OS for a free port rather than computing one from a base. + * + * A fixed base collides with whatever is already holding it — a leftover node, a + * second checkout, or `npm run app:dev` running in another terminal while the + * suite runs. Letting the kernel choose removes that whole class of failure. The + * handover window between close and the child's bind is microseconds, and a lost + * race surfaces as the child's own "address in use" stderr rather than a hang. + */ +async function freePort(): Promise { + const probe = createServer(); + const { promise, resolve: settle, reject } = Promise.withResolvers(); + probe.once('error', reject); + probe.listen(0, '127.0.0.1', () => { + const address = probe.address(); + const port = typeof address === 'object' && address !== null ? address.port : 0; + probe.close(() => settle(port)); + }); + return promise; +} + +export type RunningApp = { + /** The app origin: serves the page and proxies `/api/*`. */ + url: string; + /** The tidalDB node itself, for probes that bypass the app. */ + nodeUrl: string; + /** The oracle the fixture binary emitted for this exact corpus. */ + groundTruth: FixtureGroundTruth; + /** Decay as declared by the schema THIS node loaded, not a copy of it. */ + declaredDecay: DeclaredDecay; + /** + * The throwaway data dir. Exposed so a caller can say where a run's state went, + * and so a long-lived host can install a synchronous last-resort cleanup — an + * async `close()` that throws would otherwise leave the dir behind. + */ + dataDir: string; + close: () => Promise; +}; + +export type StartAppOptions = { + /** Bearer to inject into proxied requests. A local standalone needs none. */ + apiKey?: string; + /** Log lifecycle progress to stdout — for `npm run app:dev`, quiet in tests. */ + verbose?: boolean; +}; + +/** + * Extract the schema the DEPLOYED cluster loads, from its ConfigMap wrapper. + * + * Not interchangeable with `tidal-server/config/default-schema.yaml`: that file is + * otherwise identical but declares `dimensions: 128`, while the cluster declares + * 1536. Seeding 1536-wide vectors against a 128-wide slot is rejected with a 422 + * that reads like a malformed body, so the dimension is asserted here rather than + * discovered three layers downstream. + */ +async function extractSchema(): Promise { + const raw = await readFile(SCHEMA_CONFIGMAP, 'utf8'); + const marker = ' schema.yaml: |\n'; + const start = raw.indexOf(marker); + if (start < 0) { + throw new Error(`${SCHEMA_CONFIGMAP} has no ' schema.yaml: |' block scalar to extract`); + } + const body = raw.slice(start + marker.length); + const schema = body + .split('\n') + .map((line) => (line.startsWith(' ') ? line.slice(4) : line)) + .join('\n'); + + if (!schema.startsWith('signals:')) { + throw new Error(`extracted schema does not start with 'signals:'; got: ${schema.slice(0, 80)}`); + } + if (!new RegExp(`dimensions:\\s*${EMBEDDING_DIM}\\b`).test(schema)) { + throw new Error( + `extracted schema does not declare dimensions: ${EMBEDDING_DIM}. The fixture ` + + `contract and the schema must agree or every embedding write is rejected as 422.`, + ); + } + return schema; +} + +/** + * The decay each signal declares, read out of the schema the node actually + * loaded. + * + * The decay assertion compares an OBSERVED decay rate against a DECLARED + * half-life. Hardcoding the declared value in the test would make it a + * tautology the moment the schema changed, so it is parsed from the same string + * that was handed to `--schema`. + * + * A deliberately small parser rather than a YAML dependency: the shape it reads + * is four lines of a file this repo owns, and it fails loudly (returns nothing + * for a signal) rather than guessing. + */ +export type DeclaredDecay = Record; + +function parseDeclaredDecay(schema: string): DeclaredDecay { + const declared: DeclaredDecay = {}; + let current: string | undefined; + for (const line of schema.split('\n')) { + const name = /^\s*-\s*name:\s*(\S+)/.exec(line); + if (name?.[1]) { + current = name[1]; + continue; + } + if (current === undefined) continue; + const halfLife = /^\s*half_life_seconds:\s*(\d+)/.exec(line); + if (halfLife?.[1]) { + declared[current] = Number(halfLife[1]); + continue; + } + if (/^\s*permanent:\s*true/.test(line)) declared[current] = 'permanent'; + } + return declared; +} + +/** True once something accepts a TCP connection on the port. */ +async function portAccepts(port: number): Promise { + const { promise, resolve: settle } = Promise.withResolvers(); + const socket = createConnection({ port, host: '127.0.0.1' }); + const done = (ok: boolean) => { + socket.destroy(); + settle(ok); + }; + socket.setTimeout(1_000); + socket.once('connect', () => done(true)); + socket.once('timeout', () => done(false)); + socket.once('error', () => done(false)); + return promise; +} + +/** True once the node answers `/health` with 2xx. */ +async function healthOk(nodeUrl: string): Promise { + try { + const response = await fetch(`${nodeUrl}/health`, { + signal: AbortSignal.timeout(2_000), + }); + return response.ok; + } catch { + return false; + } +} + +const buildsInFlight = new Map>(); + +/** + * Build a cargo binary if it is not already present, once per process. + * + * `feed-fixture` is new, so no existing tree has it; failing with "binary + * missing, go run cargo" would make the hermetic suite un-runnable from a clean + * checkout for no reason. Dependencies are already compiled, so this is seconds, + * not minutes — but the timeout allows for a cold cache. + */ +function ensureBinary(binPath: string, cargoArgs: string[], verbose: boolean): Promise { + if (existsSync(binPath)) return Promise.resolve(); + const existing = buildsInFlight.get(binPath); + if (existing) return existing; + + const build = (async () => { + if (verbose) console.log(`[app] building ${binPath} (missing)`); + const result = await run('cargo', cargoArgs, { timeoutMs: BUILD_TIMEOUT_MS }); + if (result.code !== 0 || !existsSync(binPath)) { + throw new Error( + `could not build ${binPath}\n$ ${result.command}\nexit ${result.code}\n${result.stderr}`, + ); + } + })(); + buildsInFlight.set(binPath, build); + return build; +} + +/** MIME types for the handful of things the page is made of. */ +const CONTENT_TYPES: Record = { + '.html': 'text/html; charset=utf-8', + '.js': 'text/javascript; charset=utf-8', + '.css': 'text/css; charset=utf-8', + '.json': 'application/json; charset=utf-8', + '.svg': 'image/svg+xml', +}; + +/** + * Read the whole request body as text. + * + * Text rather than bytes because the only thing the page ever sends is one small + * JSON object, and `string` is unambiguously a `BodyInit` — TS's `BufferSource` + * is `ArrayBufferView`, which a pooled Node `Buffer` does not + * satisfy. If this ever needs to forward binary, that is a real change, not a cast. + */ +async function readBody(req: IncomingMessage): Promise { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(chunk as Buffer); + return Buffer.concat(chunks).toString('utf8'); +} + +/** + * Forward one `/api/*` request to the node, adding the bearer. + * + * A byte pipe: it must never reorder, re-score or filter a response, because the + * ordering IS the thing under test (`CODING_GUIDELINES.md:88`). + */ +async function proxy( + req: IncomingMessage, + res: ServerResponse, + nodeUrl: string, + apiKey: string | undefined, +): Promise { + const path = (req.url ?? '/').slice('/api'.length); + const headers: Record = { accept: 'application/json' }; + const contentType = req.headers['content-type']; + if (contentType) headers['content-type'] = contentType; + if (apiKey) headers.authorization = `Bearer ${apiKey}`; + + const method = req.method ?? 'GET'; + const hasBody = method !== 'GET' && method !== 'HEAD'; + const body = hasBody ? await readBody(req) : undefined; + + try { + const upstream = await fetch(`${nodeUrl}${path}`, { + method, + headers, + body, + signal: AbortSignal.timeout(30_000), + }); + const payload = Buffer.from(await upstream.arrayBuffer()); + res.writeHead(upstream.status, { + 'content-type': upstream.headers.get('content-type') ?? 'application/json', + 'content-length': String(payload.byteLength), + }); + res.end(payload); + } catch (error) { + // Surface the upstream fault as a real status rather than hanging the page. + const message = redact(error instanceof Error ? error.message : String(error)); + const payload = Buffer.from(JSON.stringify({ error: `proxy: ${message}` })); + res.writeHead(502, { + 'content-type': 'application/json', + 'content-length': String(payload.byteLength), + }); + res.end(payload); + } +} + +/** Serve one file from the public dir, refusing anything outside it. */ +async function serveStatic(req: IncomingMessage, res: ServerResponse): Promise { + const requested = (req.url ?? '/').split('?')[0] ?? '/'; + const relative = requested === '/' ? 'index.html' : requested.replace(/^\/+/, ''); + const target = resolve(PUBLIC_DIR, relative); + if (target !== PUBLIC_DIR && !target.startsWith(PUBLIC_DIR + sep)) { + res.writeHead(403, { 'content-type': 'text/plain' }); + res.end('forbidden'); + return; + } + try { + const body = await readFile(target); + res.writeHead(200, { + 'content-type': CONTENT_TYPES[extname(target)] ?? 'application/octet-stream', + 'content-length': String(body.byteLength), + }); + res.end(body); + } catch { + res.writeHead(404, { 'content-type': 'text/plain' }); + res.end('not found'); + } +} + +/** + * Boot a standalone node, seed the fixture catalog, and serve the app against it. + * + * Every stage that can fail reports why: an early child exit surfaces the node's + * stderr, and a non-zero seed exit surfaces the seeder's transcript, because a + * partially seeded corpus would make every downstream assertion meaningless. + */ +export async function startApp(options: StartAppOptions = {}): Promise { + const verbose = options.verbose ?? false; + const log = (message: string) => { + if (verbose) console.log(`[app] ${message}`); + }; + + await ensureBinary(SERVER_BIN, ['build', '-p', 'tidal-server', '--bin', 'tidal-server'], verbose); + await ensureBinary( + FIXTURE_BIN, + ['build', '-p', 'tidal-stress', '--bin', 'feed-fixture'], + verbose, + ); + + const dataDir = await mkdtemp(join(tmpdir(), 'tidaldb-feed-app-')); + const schemaPath = join(dataDir, 'schema.yaml'); + const catalogPath = join(dataDir, 'catalog.json'); + const truthPath = join(dataDir, 'truth.json'); + const schema = await extractSchema(); + const declaredDecay = parseDeclaredDecay(schema); + await writeFile(schemaPath, schema, 'utf8'); + await writeFile(catalogPath, JSON.stringify(CATALOG), 'utf8'); + // `--data-dir` must already exist: the server treats a missing directory as a + // config error rather than creating it (verified against its own stderr). + await mkdir(join(dataDir, 'data'), { recursive: true }); + + const nodePort = await freePort(); + const nodeUrl = `http://127.0.0.1:${nodePort}`; + + // `--listen` also reads the PORT env var (tidal-server/src/main.rs:99). An + // explicit flag wins in clap, but dropping PORT removes the ambiguity entirely. + const childEnv = { ...process.env }; + delete childEnv.PORT; + delete childEnv.TIDAL_CONFIG; + + log(`booting node on ${nodeUrl} (data ${dataDir})`); + const child: ChildProcess = spawn( + SERVER_BIN, + [ + 'standalone', + '--listen', + `127.0.0.1:${nodePort}`, + '--schema', + schemaPath, + '--data-dir', + join(dataDir, 'data'), + ], + { cwd: REPO_ROOT, env: childEnv, stdio: ['ignore', 'pipe', 'pipe'] }, + ); + + let nodeStderr = ''; + let exitInfo: string | undefined; + child.stderr?.on('data', (chunk: Buffer) => { + nodeStderr += chunk.toString(); + }); + child.stdout?.on('data', () => { + /* the node's own log; kept off the test transcript unless it fails */ + }); + child.once('exit', (code, signal) => { + exitInfo = `tidal-server exited early (code=${code} signal=${signal})`; + }); + + const stopNode = async () => { + if (child.exitCode !== null || child.signalCode !== null) return; + const { promise, resolve: settle } = Promise.withResolvers(); + const hardKill = setTimeout(() => { + child.kill('SIGKILL'); + settle(); + }, KILL_GRACE_MS); + child.once('exit', () => { + clearTimeout(hardKill); + settle(); + }); + child.kill('SIGTERM'); + await promise; + }; + + const cleanup = async (server?: Server) => { + if (server) { + const { promise, resolve: settle } = Promise.withResolvers(); + server.close(() => settle()); + server.closeAllConnections?.(); + await promise; + } + await stopNode(); + await rm(dataDir, { recursive: true, force: true }); + }; + + try { + const deadline = Date.now() + BOOT_TIMEOUT_MS; + let ready = false; + while (Date.now() < deadline) { + if (exitInfo) { + throw new Error(`${exitInfo}\n--- node stderr ---\n${redact(nodeStderr.trim())}`); + } + if (await healthOk(nodeUrl)) { + ready = true; + break; + } + await sleep(150); + } + if (!ready) { + throw new Error( + `tidal-server never answered ${nodeUrl}/health within ${BOOT_TIMEOUT_MS}ms\n` + + `--- node stderr ---\n${redact(nodeStderr.trim())}`, + ); + } + log(`node healthy; seeding ${CATALOG.length} items`); + + const seed = await run( + FIXTURE_BIN, + [ + '--base-url', + nodeUrl, + '--catalog', + catalogPath, + '--out', + truthPath, + '--dim', + String(EMBEDDING_DIM), + ...PROBE_IDS.flatMap((id) => ['--probe', String(id)]), + ], + { timeoutMs: SEED_TIMEOUT_MS }, + ); + if (seed.code !== 0) { + throw new Error( + `fixture seeding failed — a partial corpus makes every assertion ` + + `meaningless, so this is fatal.\n$ ${seed.command}\nexit ${seed.code}\n` + + `${seed.stdout}\n${seed.stderr}`, + ); + } + const groundTruth = JSON.parse(await readFile(truthPath, 'utf8')) as FixtureGroundTruth; + + const route = async (req: IncomingMessage, res: ServerResponse): Promise => { + if (req.url?.startsWith('/api/') === true) { + return proxy(req, res, nodeUrl, options.apiKey); + } + if ((req.url ?? '/').split('?')[0] === '/catalog.json') { + // The page joins ranked ids to titles, so it needs the catalog — served + // from the same contract the seeder used, never a second copy. + const payload = Buffer.from(JSON.stringify(CATALOG)); + res.writeHead(200, { + 'content-type': 'application/json; charset=utf-8', + 'content-length': String(payload.byteLength), + }); + res.end(payload); + return; + } + return serveStatic(req, res); + }; + + const server = createServer((req, res) => { + route(req, res).catch((error: unknown) => { + if (res.headersSent) { + res.end(); + return; + } + res.writeHead(500, { 'content-type': 'text/plain' }); + res.end(redact(error instanceof Error ? error.message : String(error))); + }); + }); + + // Bind port 0 and read back what the kernel gave us: no probe, no race. + const listening = Promise.withResolvers(); + server.once('error', (error) => listening.reject(error)); + server.listen(0, '127.0.0.1', () => { + const address = server.address(); + listening.resolve(typeof address === 'object' && address !== null ? address.port : 0); + }); + const appPort = await listening.promise; + + const url = `http://127.0.0.1:${appPort}`; + // Prove the app origin is actually reachable before handing it to a caller, + // so a bind that silently failed cannot look like a page-render bug later. + if (!(await portAccepts(appPort))) { + throw new Error(`app server bound ${url} but the port does not accept connections`); + } + log(`app ready at ${url}`); + + return { + url, + nodeUrl, + groundTruth, + declaredDecay, + dataDir, + close: () => cleanup(server), + }; + } catch (error) { + await cleanup(); + throw error; + } +} diff --git a/tests/e2e/app/public/index.html b/tests/e2e/app/public/index.html new file mode 100644 index 0000000..efffbfb --- /dev/null +++ b/tests/e2e/app/public/index.html @@ -0,0 +1,204 @@ + + + + + +Content feed — ranked by tidalDB + + + + +
+
+

Content feed

+

+ Ranked by tidalDB. This page holds the catalog; the database holds the + ranking. Position is whatever order the query returned — nothing here + sorts. +

+
+ +
    + +
    + local standalone node · 60-item fixture catalog · profile for_you +
    +
    + + + + diff --git a/tests/e2e/demo/workflows/feed-app.demo.spec.ts b/tests/e2e/demo/workflows/feed-app.demo.spec.ts new file mode 100644 index 0000000..00cb698 --- /dev/null +++ b/tests/e2e/demo/workflows/feed-app.demo.spec.ts @@ -0,0 +1,164 @@ +/** + * Demo capture — the product working, not just the deployment answering. + * + * The walkthrough's other nine captures prove the cluster is real: it converges, + * it refuses bad credentials, it commits a quorum write, it can be observed and + * restored. None of them shows what tidalDB is FOR. This one does, and the claim + * is deliberately narrow: a signal write changes the order of the next query, + * with nothing in between. + * + * It is one image holding both states. Two separate stills of a list would force + * a viewer to diff two frames from memory; stacked before/after makes the + * movement self-evident. The lists are composed at their native resolution rather + * than scaled to fill the frame — a downscaled list is the BUG-010 mistake. + * + * This runs against a LOCAL standalone node with fixture data, and the frame says + * so. `demo/audience-brief.md` forbids implying a dataset that is not what it + * appears to be, and `skip` being `permanent: true` is exactly why the fixture + * never touches production. + */ + +import { expect, test } from '@playwright/test'; +import { + CAPTURE_HEIGHT, + CAPTURE_WIDTH, + recordScreenshot, + writeManifestFragment, + type CaptureRecord, +} from '../support/proof-panel.ts'; +import { startApp } from '../../app/harness.ts'; + +const records: CaptureRecord[] = []; + +/** + * Rows shown in each half. + * + * Five, not six: two six-row lists plus their labels came to ~922px inside a + * 900px frame, which pushed the environment footer off the bottom and clipped the + * last row. The footer is what stops a fixture screenshot reading as production, + * so it is not optional — and the fix is fewer rows, never a downscaled list + * (that was the BUG-010 mistake). + */ +const ROWS = 5; + +/** Viewport that makes the page's centred column fill the frame exactly. */ +const APP_VIEWPORT = { width: 1240, height: 900 }; + +type Row = { position: string; title: string }; + +function composition(before: string, after: string, movedTitle: string, rows: Row[]): string { + const label = (text: string) => `
    ${text}
    `; + return ` + + + ${label(`before — ${rows.length} of 60 items, no signals written yet. Everything ties, so the order is the tie-break.`)} + + ${label(`after — one like on “${movedTitle}”, then the same query again. It is now first.`)} + +
    local standalone node · 60-item fixture catalog · profile for_you · no ETL, no reindex, no second system
    +`; +} + +test.describe('feed-app product surface', () => { + test('CAP-016 a signal write reorders the feed immediately', async ({ page }, testInfo) => { + const app = await startApp(); + try { + await page.setViewportSize(APP_VIEWPORT); + await page.goto(`${app.url}/?limit=${ROWS}`, { waitUntil: 'networkidle' }); + await page.waitForSelector('#feed li'); + await page.evaluate(() => document.fonts.ready); + + const list = page.locator('#feed'); + const readRows = (): Promise => + page.$$eval('#feed li', (nodes) => + nodes.map((li) => ({ + position: li.querySelector('.pos')!.textContent!.trim().replace(/\s+/g, ' '), + title: li.querySelector('.title')!.textContent!.trim().replace(/\s+/g, ' '), + })), + ); + + const before = await readRows(); + expect(before.length, `the page must render ${ROWS} rows to compose from`).toBe(ROWS); + const beforeShot = await list.screenshot(); + + // The last visible row, so the movement spans the whole frame. + const targetTitle = before[before.length - 1]!.title; + await page.locator('#feed li').last().locator('button[data-signal="like"]').click(); + + // Wait for the re-read to land by watching the thing under test change, + // rather than sleeping a guessed interval. `startsWith`, not equality: once + // a row moves, its title element also carries the delta badge ("\u25b25"). + await page.waitForFunction( + (title) => + document.querySelector('#feed li .title')?.textContent?.trim().startsWith(title) === true, + targetTitle, + { timeout: 15_000 }, + ); + + const after = await readRows(); + const afterIndex = after.findIndex((row) => row.title.startsWith(targetTitle)); + + // Assert BEFORE photographing. A capture is a photograph of an + // already-proven state; if the order did not change there is nothing + // honest to show. + expect( + afterIndex, + `the liked item must have moved to the top; "${targetTitle}" is at index ${afterIndex}`, + ).toBe(0); + + // Move the pointer off the list and drop focus before photographing. + // The cursor physically stays where it clicked, so after the re-render a + // DIFFERENT row's button sits under it and picks up `:hover` — a highlighted + // control on an unrelated row, which reads as meaningful when it is not. + await page.mouse.move(0, 0); + await page.evaluate(() => (document.activeElement as HTMLElement | null)?.blur()); + const afterShot = await list.screenshot(); + const dataUri = (buffer: Buffer): string => + `data:image/png;base64,${buffer.toString('base64')}`; + + await page.setViewportSize({ width: CAPTURE_WIDTH, height: CAPTURE_HEIGHT }); + await page.setContent( + composition(dataUri(beforeShot), dataUri(afterShot), targetTitle, before), + { waitUntil: 'load' }, + ); + await page.evaluate(() => document.fonts.ready); + + records.push( + await recordScreenshot(await page.screenshot(), testInfo, { + captureId: 'CAP-016-feed-reorder', + capabilityId: 'CAP-016', + expected: + 'The same query, before and after one like: the liked item moves from last to first, with its like_boost visible', + businessPurpose: + 'A signal write changes the order with no ETL in between — the product thesis, not the deployment', + personas: ['cluster operator', 'application developer'], + width: CAPTURE_WIDTH, + height: CAPTURE_HEIGHT, + }), + ); + } finally { + await app.close(); + } + }); + + test.afterAll(async () => { + await writeManifestFragment('feed-app', records); + }); +}); diff --git a/tests/e2e/features/10-ranking-semantics.spec.ts b/tests/e2e/features/10-ranking-semantics.spec.ts new file mode 100644 index 0000000..36cdf9c --- /dev/null +++ b/tests/e2e/features/10-ranking-semantics.spec.ts @@ -0,0 +1,415 @@ +/** + * Section 10 — ranking semantics. + * + * The other ten spec files prove the deployment answers: TLS, auth, quorum + * commit, convergence, isolation, dashboards, backups. Not one of them writes a + * signal and observes an order change, so tidalDB's actual thesis — `VISION.md:17` + * "Ranking is not a feature. It is a primitive." — was unverified. + * + * These five checks verify it against a throwaway standalone node this suite + * boots and seeds itself (60 items, 4 categories, deterministic vectors from + * `tidal-stress`'s own generator). Hermetic: no cluster, no credential, no + * network. Run with `npm run test:e2e:semantics`. + * + * Two of the five report a product gap rather than a success, because that is + * what the measurement said. They are written as tripwires in the honest + * direction — the same shape as the inert-feature checks in + * `09-operator-authority.spec.ts` — so the day the gap closes, the test fails + * with an instruction instead of the gap persisting silently. + */ + +import { expect, test, type APIRequestContext } from '@playwright/test'; +import { recordJson } from '../support/evidence.ts'; +import { PROBE_IDS } from '../app/fixture-contract.ts'; +import { startApp, type RunningApp } from '../app/harness.ts'; + +/** One row of a ranked response (`tidal-server/src/dto.rs:269`). */ +type FeedItem = { + entity_id: number; + score: number; + rank: number; + /** Absent, not `[]`, when a profile reports nothing (`dto.rs:373`). */ + signals?: { name: string; value: number }[]; +}; + +/** The whole catalog in one read, so "last" means globally worst, not page-worst. */ +const FULL_CORPUS = 60; + +/** + * `for_you`, never `trending`. Measured on this schema, `trending` reports + * `view_velocity` and `share_velocity` — and `share` is not a declared signal + * here, so that term is permanently 0. An unknown profile name is answered with + * 500 rather than 400, so profile names are pinned to ones probed as live. + */ +const PROFILE = 'for_you'; + +async function readFeed( + request: APIRequestContext, + app: RunningApp, + limit = FULL_CORPUS, +): Promise { + const response = await request.get(`${app.url}/api/feed?profile=${PROFILE}&limit=${limit}`); + expect(response.status(), 'the feed must answer before anything is concluded from it').toBe(200); + const body = (await response.json()) as { items?: FeedItem[] }; + const items = body.items ?? []; + expect(items.length, 'an empty feed cannot support any ranking assertion').toBeGreaterThan(0); + return items; +} + +async function writeSignal( + request: APIRequestContext, + app: RunningApp, + entityId: number, + signal: string, +): Promise { + const response = await request.post(`${app.url}/api/signals`, { + data: { entity_id: entityId, signal, weight: 1.0 }, + }); + expect(response.status(), `POST /signals ${signal} on ${entityId} must be accepted`).toBe(204); +} + +const positionOf = (items: FeedItem[], entityId: number): number => + items.findIndex((item) => item.entity_id === entityId); + +const signalValue = (item: FeedItem | undefined, name: string): number | undefined => + item?.signals?.find((signal) => signal.name === name)?.value; + +test.describe('section 10 — ranking semantics', () => { + test('a like moves the item up, immediately — no ETL in between', async ({ + request, + }, testInfo) => { + const app = await startApp(); + try { + const before = await readFeed(request, app); + const target = before[before.length - 1]!.entity_id; + const beforeIndex = positionOf(before, target); + + await writeSignal(request, app, target, 'like'); + + // No sleep and no retry between the write and the read. VISION.md:166 + // claims the next query reflects the write with "no Kafka consumer to lag, + // no feature store sync to schedule". Polling here would be testing our + // patience rather than that claim. + const after = await readFeed(request, app); + const afterIndex = positionOf(after, target); + + await recordJson(testInfo, 'like-moves-up', { + entityId: target, + beforeIndex, + afterIndex, + likeBoost: signalValue(after[afterIndex], 'like_boost'), + profile: PROFILE, + }); + + // Position, not score. Scores are floats a profile owns and re-normalises + // across the candidate set (measured: unliked items go 0.5 -> 0.0 once one + // item is boosted). Position is the contract a user actually experiences. + expect( + afterIndex, + `a like must improve position immediately; ${target} went ${beforeIndex} -> ${afterIndex}`, + ).toBeLessThan(beforeIndex); + expect(afterIndex, 'a single like on the worst item should reach the top').toBe(0); + } finally { + await app.close(); + } + }); + + test('a skip is durably accepted and then ignored by every built-in profile', async ({ + request, + }, testInfo) => { + const app = await startApp(); + try { + // Every profile this deployment exposes and which resolves (`top` returns + // 500 despite existing in engine source, so it is not probed). + const profiles = [ + 'for_you', + 'trending', + 'hot', + 'new', + 'shuffle', + 'hidden_gems', + 'controversial', + ]; + const baseline = await readFeed(request, app); + const target = baseline[20]!.entity_id; + + const positionsBefore: Record = {}; + for (const profile of profiles) { + const response = await request.get( + `${app.url}/api/feed?profile=${profile}&limit=${FULL_CORPUS}`, + ); + const body = (await response.json()) as { items?: FeedItem[] }; + positionsBefore[profile] = positionOf(body.items ?? [], target); + } + + // Five skips, not one: a single write could plausibly fall below a rounding + // threshold. Five cannot. + for (let n = 0; n < 5; n += 1) await writeSignal(request, app, target, 'skip'); + + const positionsAfter: Record = {}; + const reported: Record = {}; + for (const profile of profiles) { + const response = await request.get( + `${app.url}/api/feed?profile=${profile}&limit=${FULL_CORPUS}`, + ); + const body = (await response.json()) as { items?: FeedItem[] }; + const items = body.items ?? []; + positionsAfter[profile] = positionOf(items, target); + reported[profile] = (items[positionsAfter[profile]]?.signals ?? []).map((s) => s.name); + } + + await recordJson(testInfo, 'skip-is-inert', { + entityId: target, + skipsWritten: 5, + declaredDecay: app.declaredDecay, + positionsBefore, + positionsAfter, + signalNamesReported: reported, + }); + + // The schema declares `skip` (permanent), and the write is accepted, so the + // event is durably recorded. But `Penalty` — the mechanism that would let it + // demote anything (`tidal/src/ranking/profile.rs:227`, applied at + // `tidal/src/ranking/executor/signal_values.rs:183`, labelled `{signal}_penalty` + // at `tidal/src/ranking/executor/mod.rs:65`) — is never populated: every + // built-in profile is built from `skeleton()`, which sets + // `penalties: vec![]` (`tidal/src/ranking/builtins.rs:62`), and no built-in + // overrides it. So `VISION.md:187` "Negative signals are equal citizens" does + // not hold for any shipped profile: a skip is stored and query-time inert. + expect(app.declaredDecay.skip, 'the schema must still declare skip').toBe('permanent'); + for (const profile of profiles) { + expect( + positionsAfter[profile], + `${profile} moved ${target} after 5 skips (${positionsBefore[profile]} -> ` + + `${positionsAfter[profile]}). Good news: a profile now applies a skip ` + + `penalty. Rewrite this test as a demotion assertion and close the ` + + `"negative signals are inert" finding.`, + ).toBe(positionsBefore[profile]); + expect( + reported[profile], + `${profile} now reports a skip term. Same good news as above.`, + ).not.toContain('skip_penalty'); + } + } finally { + await app.close(); + } + }); + + test('decay is applied continuously at query time, at each signal\u2019s declared half-life', async ({ + request, + }, testInfo) => { + const app = await startApp(); + try { + const viewItem = 900_000_300; + const likeItem = 900_000_301; + + // Written concurrently so both carry the same elapsed time, which makes the + // ratio of their decay rates depend only on their declared half-lives. + await Promise.all([ + writeSignal(request, app, viewItem, 'view'), + writeSignal(request, app, likeItem, 'like'), + ]); + + const first = await readFeed(request, app); + const readOne = Date.now(); + + // A real interval, deliberately. This is not a sleep to make an assertion + // pass — elapsed time IS the independent variable of the property under + // test. `CODING_GUIDELINES.md:88` says "decay is a type, not a formula you + // call"; the observable consequence is that the SAME stored event reports a + // smaller value on a later read, with no writes in between. + await new Promise((resolve) => setTimeout(resolve, 4_000)); + + const second = await readFeed(request, app); + const readTwo = Date.now(); + const elapsedSeconds = (readTwo - readOne) / 1_000; + + const viewFirst = signalValue(first[positionOf(first, viewItem)], 'view_boost'); + const viewSecond = signalValue(second[positionOf(second, viewItem)], 'view_boost'); + const likeFirst = signalValue(first[positionOf(first, likeItem)], 'like_boost'); + const likeSecond = signalValue(second[positionOf(second, likeItem)], 'like_boost'); + + // Guard the measurement before concluding anything from it: an absent value + // would otherwise read as "no decay". + for (const [label, value] of Object.entries({ viewFirst, viewSecond, likeFirst, likeSecond })) { + expect(value, `${label} must be reported before decay can be measured`).toBeDefined(); + } + + // Recover each signal's half-life from the two readings: + // value(t) = value(0) * 2^(-t/H) => H = t * ln2 / -ln(v2/v1) + const impliedHalfLife = (v1: number, v2: number): number => + (elapsedSeconds * Math.LN2) / -Math.log(v2 / v1); + const viewHalfLife = impliedHalfLife(viewFirst!, viewSecond!); + const likeHalfLife = impliedHalfLife(likeFirst!, likeSecond!); + + const declaredView = app.declaredDecay.view; + const declaredLike = app.declaredDecay.like; + expect(typeof declaredView, 'view must declare an exponential half-life').toBe('number'); + expect(typeof declaredLike, 'like must declare an exponential half-life').toBe('number'); + + await recordJson(testInfo, 'decay-recovers-declared-half-life', { + elapsedSeconds, + declaredDecay: app.declaredDecay, + view: { first: viewFirst, second: viewSecond, impliedHalfLifeSeconds: viewHalfLife }, + like: { first: likeFirst, second: likeSecond, impliedHalfLifeSeconds: likeHalfLife }, + impliedDays: { view: viewHalfLife / 86_400, like: likeHalfLife / 86_400 }, + }); + + expect(viewSecond!, 'the same stored view must report lower on a later read').toBeLessThan( + viewFirst!, + ); + expect(likeSecond!, 'the same stored like must report lower on a later read').toBeLessThan( + likeFirst!, + ); + + // +/-20% is generous against a 4-second observation window and still + // separates the two decisively: 7d x 1.2 = 8.4d sits well below 14d x 0.8 = 11.2d. + const tolerance = 0.2; + expect( + Math.abs(viewHalfLife - (declaredView as number)) / (declaredView as number), + `view decayed as if its half-life were ${(viewHalfLife / 86_400).toFixed(2)}d, but the ` + + `loaded schema declares ${((declaredView as number) / 86_400).toFixed(2)}d`, + ).toBeLessThan(tolerance); + expect( + Math.abs(likeHalfLife - (declaredLike as number)) / (declaredLike as number), + `like decayed as if its half-life were ${(likeHalfLife / 86_400).toFixed(2)}d, but the ` + + `loaded schema declares ${((declaredLike as number) / 86_400).toFixed(2)}d`, + ).toBeLessThan(tolerance); + + // Note on scope: `skip` is declared permanent, and permanence is the one + // decay claim this surface cannot show — no built-in profile reports a skip + // term at all (see the inertness check above), so there is no value to watch + // hold still. Recorded rather than faked. + expect(app.declaredDecay.skip, 'skip remains declared permanent').toBe('permanent'); + } finally { + await app.close(); + } + }); + + test('vector search returns the true nearest neighbours, not an approximation', async ({ + request, + }, testInfo) => { + const app = await startApp(); + try { + const observations: Record[] = []; + + for (const probe of PROBE_IDS) { + const vector = app.groundTruth.probeVectors[String(probe)]; + expect(vector, `ground truth carries no query vector for probe ${probe}`).toBeDefined(); + expect(vector!.length, 'the query vector must match the declared width').toBe( + app.groundTruth.dim, + ); + + // Field is `vector`, not `values` — a wrong name is answered 422, which + // reads like a malformed body. Response field is `items`, not `matches`. + const response = await request.post(`${app.url}/api/vector_search`, { + data: { vector, k: 10 }, + }); + expect(response.status(), `vector_search for ${probe} must answer`).toBe(200); + const body = (await response.json()) as { + items?: { entity_id: number; distance: number }[]; + }; + const items = body.items ?? []; + expect( + items.length, + 'an empty result cannot distinguish a perfect index from a broken one', + ).toBeGreaterThan(0); + + const truth = app.groundTruth.neighboursByItem[String(probe)] ?? []; + const returned = items.map((item) => item.entity_id); + const compared = Math.min(truth.length, returned.length); + + observations.push({ + probe, + selfDistance: items[0]?.distance, + returned: returned.slice(0, compared), + truth: truth.slice(0, compared), + }); + + // An item's own vector must find the item itself, at ~zero distance. + expect(items[0]?.entity_id, `${probe}'s own vector must rank ${probe} first`).toBe(probe); + expect( + items[0]!.distance, + `${probe} found itself at distance ${items[0]!.distance}, beyond the f32 round-trip ` + + `tolerance of ${app.groundTruth.selfDistanceTolerance}`, + ).toBeLessThan(app.groundTruth.selfDistanceTolerance); + + // And the whole top-k must equal brute-force cosine over the same corpus, + // computed by `tidal-stress`'s own oracle from the same generator that + // produced the indexed vectors. + expect( + returned.slice(0, compared), + `ANN top-${compared} for ${probe} diverged from brute-force cosine`, + ).toEqual(truth.slice(0, compared)); + } + + await recordJson(testInfo, 'ann-matches-brute-force', { + dim: app.groundTruth.dim, + tolerance: app.groundTruth.selfDistanceTolerance, + observations, + }); + } finally { + await app.close(); + } + }); + + test('reported signal values reconcile with what was written, and rank is sequential', async ({ + request, + }, testInfo) => { + const app = await startApp(); + try { + const likedTwice = 900_000_200; + const viewedOnce = 900_000_201; + + await writeSignal(request, app, likedTwice, 'like'); + await writeSignal(request, app, likedTwice, 'like'); + await writeSignal(request, app, viewedOnce, 'view'); + + const items = await readFeed(request, app); + const liked = items[positionOf(items, likedTwice)]; + const viewed = items[positionOf(items, viewedOnce)]; + + const likeBoost = signalValue(liked, 'like_boost'); + const viewCount = signalValue(viewed, 'view'); + const viewBoost = signalValue(viewed, 'view_boost'); + const ranks = items.map((item) => item.rank); + const expectedRanks = items.map((_unused, index) => index + 1); + + await recordJson(testInfo, 'explainability-reconciles', { + likedTwice: { entityId: likedTwice, likeBoost }, + viewedOnce: { entityId: viewedOnce, viewCount, viewBoost }, + ranks, + }); + + // Two unit-weight likes report a boost of 2 per like. Decay over the few + // milliseconds since the write makes this marginally short of 4, so the + // comparison is a tolerance rather than an equality. + expect(likeBoost, 'two unit likes must be reported, not silently dropped').toBeDefined(); + expect( + Math.abs(likeBoost! - 4), + `two unit-weight likes reported like_boost ${likeBoost}, expected ~4 (2 per like)`, + ).toBeLessThan(0.01); + + // The raw `view` term is a count, and it must equal the number of writes. + expect(viewCount, 'the raw view count must be reported').toBe(1); + expect( + Math.abs(viewBoost! - 1), + `one unit-weight view reported view_boost ${viewBoost}, expected ~1`, + ).toBeLessThan(0.01); + + // Rank must be a dense 1..n sequence. This is the assertion that caught the + // duplicate-rank defect on the deployed cluster, where `scatter_merge` + // returns a merged slice without re-stamping rank + // (`tidal-server/src/cluster/node.rs:7542`). Standalone numbers ranks at + // `tidal/src/query/executor/pipeline.rs:611` and is unaffected — the pair of + // results is what localises the defect to the merge. + // See 11-ranking-integrity.spec.ts for the cluster half. + expect( + ranks, + 'rank must be a dense 1..n sequence; duplicates mean a merge did not re-stamp it', + ).toEqual(expectedRanks); + } finally { + await app.close(); + } + }); +}); diff --git a/tests/e2e/features/11-ranking-integrity.spec.ts b/tests/e2e/features/11-ranking-integrity.spec.ts new file mode 100644 index 0000000..b9b0534 --- /dev/null +++ b/tests/e2e/features/11-ranking-integrity.spec.ts @@ -0,0 +1,144 @@ +/** + * Section 11 — ranking integrity on the deployed cluster. + * + * A tripwire pair, in the honest direction. `rank` is currently WRONG on every + * corpus-wide ranked response from the cluster, and these checks pin that with + * its root cause so the fix announces itself instead of the defect persisting + * silently. Same shape as the inert-feature checks in + * `09-operator-authority.spec.ts`. + * + * Root cause, localised: `scatter_merge` + * (`tidal-server/src/cluster/node.rs:7471`) concatenates each shard group's + * locally-ranked slice, sorts by score, truncates, and returns at `:7542` + * WITHOUT re-stamping rank. Its sibling `merge_cross_shard` (`:7583`) does + * re-stamp, at `:7609`, with a comment naming this exact hazard — but `:7621` + * documents that full placement short-circuits past it, and this cluster is + * full-placement RF3, so the guarded path never runs. + * + * The evidence that it is the MERGE and not the engine is two-sided: + * - standalone numbers ranks densely (`10-ranking-semantics.spec.ts`, measured + * 1..60 with no gaps) via `tidal/src/query/executor/pipeline.rs:611`; + * - the cluster returns per-group ranks concatenated, while SCORES remain + * correctly ordered — so the merge's sort is fine and only the stamp is absent. + * + * Read-only. Nothing here writes to the deployed cluster. + */ + +import { expect, test, type APIRequestContext } from '@playwright/test'; +import { recordJson } from '../support/evidence.ts'; +import { PUBLIC_BASE_URL, apiKey } from '../support/env.ts'; + +type RankedItem = { entity_id: number; score: number; rank: number }; + +/** + * Enough rows that at least two shard groups must both contribute. With three + * groups a limit of 1 or 2 can be answered from one group and would show no + * duplicate at all. + */ +const LIMIT = 12; + +/** What a correctly stamped response looks like: dense, ascending, from 1. */ +const denseRanks = (count: number): number[] => + Array.from({ length: count }, (_unused, index) => index + 1); + +async function ranked( + request: APIRequestContext, + path: string, +): Promise<{ items: RankedItem[]; ranks: number[]; scores: number[] }> { + const response = await request.get(`${PUBLIC_BASE_URL}${path}`, { + headers: { authorization: `Bearer ${apiKey()}` }, + }); + expect(response.status(), `${path} must answer before any conclusion is drawn`).toBe(200); + const body = (await response.json()) as { items?: RankedItem[] }; + const items = body.items ?? []; + expect( + items.length, + 'an empty result cannot distinguish correct ranking from broken ranking', + ).toBeGreaterThan(1); + return { + items, + ranks: items.map((item) => item.rank), + scores: items.map((item) => item.score), + }; +} + +/** The order is right even though the stamp is wrong — the precise localisation. */ +function expectScoresOrdered(scores: number[], surface: string): void { + for (let index = 1; index < scores.length; index += 1) { + expect( + scores[index]!, + `${surface} returned scores out of order at position ${index} ` + + `(${scores[index - 1]} then ${scores[index]}). That is a DIFFERENT and worse ` + + `defect than the rank stamp: it would mean the merge's sort is broken too.`, + ).toBeLessThanOrEqual(scores[index - 1]!); + } +} + +const FIX_INSTRUCTION = + 'Good news: scatter_merge appears to be fixed. Delete this tripwire, keep the ' + + 'dense-rank assertion in 10-ranking-semantics.spec.ts, and close the defect.'; + +test.describe('section 11 — ranking integrity (cluster tripwires)', () => { + test('cluster /feed rank is duplicated — pinned defect in scatter_merge', async ({ + request, + }, testInfo) => { + const { items, ranks, scores } = await ranked( + request, + `/feed?profile=for_you&limit=${LIMIT}`, + ); + const duplicates = ranks.length - new Set(ranks).size; + + await recordJson(testInfo, 'cluster-feed-ranks', { + surface: `/feed?profile=for_you&limit=${LIMIT}`, + ranks, + expectedIfFixed: denseRanks(ranks.length), + duplicateCount: duplicates, + scores, + entityIds: items.map((item) => item.entity_id), + rootCause: 'tidal-server/src/cluster/node.rs:7542 (scatter_merge returns without set_rank)', + }); + + expect(ranks, FIX_INSTRUCTION).not.toEqual(denseRanks(ranks.length)); + expect( + duplicates, + 'expected duplicate ranks — the signature of per-group slices merged without ' + + `a re-stamp. ${FIX_INSTRUCTION}`, + ).toBeGreaterThan(0); + + // Ordering is correct; only the stamp is missing. If this ever fails, the + // defect has become materially worse. + expectScoresOrdered(scores, '/feed'); + }); + + test('cluster /search rank is duplicated the same way, from the same merge', async ({ + request, + }, testInfo) => { + // A term known to be indexed on this corpus. The production corpus has no + // titles for its 33k items, so an arbitrary word returns zero candidates and + // would make this check vacuous. + const { items, ranks, scores } = await ranked( + request, + `/search?query=verification&limit=${LIMIT}`, + ); + const duplicates = ranks.length - new Set(ranks).size; + + await recordJson(testInfo, 'cluster-search-ranks', { + surface: `/search?query=verification&limit=${LIMIT}`, + ranks, + expectedIfFixed: denseRanks(ranks.length), + duplicateCount: duplicates, + scores, + entityIds: items.map((item) => item.entity_id), + }); + + // `/search` shares the same gateway merge, so the same defect surfaces here. + // Asserting it on both surfaces is what shows the fault is in the merge + // rather than in one query pipeline. + expect(ranks, FIX_INSTRUCTION).not.toEqual(denseRanks(ranks.length)); + expect( + duplicates, + `expected duplicate ranks on /search too. ${FIX_INSTRUCTION}`, + ).toBeGreaterThan(0); + expectScoresOrdered(scores, '/search'); + }); +}); diff --git a/tests/e2e/support/cluster.ts b/tests/e2e/support/cluster.ts index 6f30687..0054a26 100644 --- a/tests/e2e/support/cluster.ts +++ b/tests/e2e/support/cluster.ts @@ -15,7 +15,11 @@ import { execFile, spawn, type ChildProcess } from 'node:child_process'; import { createConnection } from 'node:net'; import { setTimeout as sleep } from 'node:timers/promises'; import { promisify } from 'node:util'; -import { KUBECONFIG, TIDALCTL_BIN, redact } from './env'; +// Explicit `.ts` extension: this module is in the transitive closure of +// `tests/e2e/app/harness.ts`, which `npm run app:dev` loads with bare Node's +// type stripping — and Node's ESM resolver does not guess extensions. Playwright +// resolves either form, so specs elsewhere keep the extensionless style. +import { KUBECONFIG, TIDALCTL_BIN, redact } from './env.ts'; const execFileAsync = promisify(execFile); diff --git a/tidal-stress/src/bin/feed-fixture.rs b/tidal-stress/src/bin/feed-fixture.rs new file mode 100644 index 0000000..24df8b2 --- /dev/null +++ b/tidal-stress/src/bin/feed-fixture.rs @@ -0,0 +1,247 @@ +//! `feed-fixture` — seed a small NAMED catalog into a running tidalDB and emit +//! its brute-force ground truth. +//! +//! The content-feed verification app needs three things that must agree exactly: +//! the items the database indexed, the vectors it indexed for them, and an oracle +//! saying which of those vectors are truly nearest to which. This binary produces +//! all three from one generator, so they cannot disagree. +//! +//! Vectors come from [`tidal_stress::recall::embedding_for`] — the SAME generator +//! the recall harness and the engine-side tests use. There is already a second +//! copy of that scheme in `tidal/src/db/items.rs`; a third (in the TypeScript +//! harness, say) would mean the oracle could silently stop matching what was +//! indexed. Reuse is the whole point of putting this binary in this crate. +//! +//! It deliberately writes NO signals. Signals are what the assertions +//! manipulate, so seeding any here would pre-bias every run. +//! +//! Usage: +//! ```text +//! feed-fixture --base-url http://127.0.0.1:9400 \ +//! --catalog catalog.json --out truth.json \ +//! --probe 900000007 --probe 900000107 +//! ``` + +use std::collections::HashMap; +use std::path::PathBuf; +use std::process::ExitCode; + +use clap::Parser; +use serde::{Deserialize, Serialize}; +use tidal_stress::recall::{GroundTruth, embedding_for}; + +/// How far an item's own vector may land from itself and still count as "zero" +/// distance. Non-zero only because the vector round-trips through f32 JSON and +/// the engine re-normalises on write; measured self-distance on the live cluster +/// was 0.013, so this is ~4x headroom over observed rounding. +const SELF_DISTANCE_TOLERANCE: f32 = 0.05; + +#[derive(Parser)] +#[command( + version, + about = "Seed a named fixture catalog + emit its brute-force ground truth" +)] +struct Cli { + /// Base URL of the target node, e.g. `http://127.0.0.1:9400`. + #[arg(long)] + base_url: String, + + /// JSON array of `{entityId, title, category}` — the catalog to seed. + #[arg(long)] + catalog: PathBuf, + + /// Where to write the ground-truth JSON. A generated artifact; never commit it. + #[arg(long)] + out: PathBuf, + + /// Embedding width. Must match the schema's declared dimension. + #[arg(long, default_value_t = 1536)] + dim: usize, + + /// Neighbours to record per item. + #[arg(long, default_value_t = 10)] + k: usize, + + /// Ids whose raw query vector is emitted, for the ANN assertion's input. + /// Repeatable. + #[arg(long = "probe")] + probes: Vec, + + /// Bearer token, when the target requires auth. A local standalone does not + /// (`tidal-server/tests/standalone.rs:32`). + #[arg(long, env = "TIDAL_API_KEY")] + api_key: Option, +} + +/// One catalog entry, as the TypeScript fixture contract serialises it. +#[derive(Debug, Deserialize)] +struct CatalogItem { + #[serde(rename = "entityId")] + entity_id: u64, + title: String, + category: String, +} + +/// `POST /items` body (mirrors `tidal-server/src/dto.rs:27` `ItemRequest`). +#[derive(Serialize)] +struct ItemBody<'a> { + entity_id: u64, + metadata: HashMap<&'a str, &'a str>, +} + +/// `POST /embeddings` body (mirrors `dto.rs:37` `EmbeddingRequest`). +#[derive(Serialize)] +struct EmbeddingBody<'a> { + entity_id: u64, + values: &'a [f32], +} + +/// What the harness and the specs read back. +#[derive(Serialize)] +struct Truth { + dim: usize, + #[serde(rename = "neighboursByItem")] + neighbours_by_item: HashMap>, + #[serde(rename = "probeVectors")] + probe_vectors: HashMap>, + #[serde(rename = "selfDistanceTolerance")] + self_distance_tolerance: f32, +} + +fn main() -> ExitCode { + let cli = Cli::parse(); + match tokio::runtime::Runtime::new() { + Ok(rt) => match rt.block_on(run(cli)) { + Ok(()) => ExitCode::SUCCESS, + Err(err) => { + eprintln!("error: {err}"); + ExitCode::FAILURE + } + }, + Err(err) => { + eprintln!("error: could not start runtime: {err}"); + ExitCode::FAILURE + } + } +} + +async fn run(cli: Cli) -> Result<(), String> { + let raw = std::fs::read_to_string(&cli.catalog) + .map_err(|e| format!("could not read --catalog {}: {e}", cli.catalog.display()))?; + let catalog: Vec = + serde_json::from_str(&raw).map_err(|e| format!("malformed catalog JSON: {e}"))?; + if catalog.is_empty() { + return Err("catalog is empty — nothing to seed".to_owned()); + } + + let client = reqwest::Client::builder() + .build() + .map_err(|e| format!("http client build failed: {e}"))?; + let base = cli.base_url.trim_end_matches('/'); + + // Generate every vector ONCE, up front, so the bytes posted to the engine and + // the bytes fed to the oracle are literally the same values. + let vectors: Vec<(u64, Vec)> = catalog + .iter() + .map(|item| (item.entity_id, embedding_for(item.entity_id, cli.dim))) + .collect(); + + for item in &catalog { + let metadata = HashMap::from([ + ("title", item.title.as_str()), + ("category", item.category.as_str()), + ]); + post( + &client, + &format!("{base}/items"), + cli.api_key.as_deref(), + &ItemBody { + entity_id: item.entity_id, + metadata, + }, + ) + .await?; + } + + for (id, values) in &vectors { + post( + &client, + &format!("{base}/embeddings"), + cli.api_key.as_deref(), + &EmbeddingBody { + entity_id: *id, + values, + }, + ) + .await?; + } + + // The oracle covers ONLY the catalog ids. That is sound because each category + // owns an otherwise-unoccupied embedding cluster: intra-cluster distance + // ~0.435 against ~2.04 for the nearest foreign vector, so the local top-k IS + // the global top-k. Anything else would need a 900-million-vector oracle. + let truth_oracle = GroundTruth::from_ids(vectors.iter().map(|(id, _)| *id).collect(), cli.dim); + let neighbours_by_item: HashMap> = vectors + .iter() + .map(|(id, values)| (id.to_string(), truth_oracle.top_k(values, cli.k))) + .collect(); + + let mut probe_vectors: HashMap> = HashMap::new(); + for probe in &cli.probes { + let found = vectors.iter().find(|(id, _)| id == probe); + let (_, values) = found.ok_or_else(|| { + format!("--probe {probe} is not in the catalog; it would have no ground truth") + })?; + probe_vectors.insert(probe.to_string(), values.clone()); + } + + let truth = Truth { + dim: cli.dim, + neighbours_by_item, + probe_vectors, + self_distance_tolerance: SELF_DISTANCE_TOLERANCE, + }; + let encoded = + serde_json::to_string(&truth).map_err(|e| format!("could not encode ground truth: {e}"))?; + std::fs::write(&cli.out, encoded) + .map_err(|e| format!("could not write --out {}: {e}", cli.out.display()))?; + + println!( + "seeded {} items + {} embeddings into {base}; ground truth -> {} ({} probes)", + catalog.len(), + vectors.len(), + cli.out.display(), + cli.probes.len() + ); + Ok(()) +} + +/// POST one JSON body, treating any non-2xx as fatal. +/// +/// A partially seeded corpus makes every downstream assertion meaningless — the +/// ANN top-k would be wrong for reasons that have nothing to do with the index — +/// so there is no "skip and continue" path here by design. +async fn post( + client: &reqwest::Client, + url: &str, + api_key: Option<&str>, + body: &B, +) -> Result<(), String> { + let mut request = client.post(url).json(body); + if let Some(key) = api_key { + request = request.bearer_auth(key); + } + let response = request + .send() + .await + .map_err(|e| format!("POST {url} failed to send: {e}"))?; + let status = response.status(); + if status.is_success() { + return Ok(()); + } + let detail = response + .text() + .await + .unwrap_or_else(|e| format!("")); + Err(format!("POST {url} returned {status}: {detail}")) +} diff --git a/tidal-stress/src/recall.rs b/tidal-stress/src/recall.rs index 5d31481..d6d6206 100644 --- a/tidal-stress/src/recall.rs +++ b/tidal-stress/src/recall.rs @@ -129,33 +129,51 @@ pub fn embedding_for(id: u64, dim: usize) -> Vec { pub struct GroundTruth { flat: Vec, norms: Vec, + /// The corpus ids, positionally aligned with `flat`/`norms`. + /// + /// Stored rather than implied by `1..=n` so the oracle also serves a SPARSE + /// id set. The fixture catalog's ids live in deliberately unoccupied 100-id + /// clusters (`900_000_0xx`), so a dense oracle would have to materialise 900 + /// million vectors to answer a question about 60 of them. + ids: Vec, dim: usize, - n: u64, } impl GroundTruth { /// Build the deterministic corpus for ids `1..=n` at `dim` dimensions. #[must_use] pub fn build(n: u64, dim: usize) -> Self { - let mut flat = Vec::with_capacity((n as usize) * dim); - let mut norms = Vec::with_capacity(n as usize); - for id in 1..=n { + // The collected ids ARE the stored ids — no transient copy even at 1M. + Self::from_ids((1..=n).collect(), dim) + } + + /// Build the deterministic corpus for an arbitrary, possibly sparse id set. + /// + /// Same generator and same cosine oracle as [`Self::build`]; the only + /// difference is that the ids need not be contiguous. This is what lets a + /// small named catalog get an *exact* oracle: when every id sits in an + /// otherwise-unoccupied cluster, brute force over just those ids is the + /// global answer. + #[must_use] + pub fn from_ids(ids: Vec, dim: usize) -> Self { + let mut flat = Vec::with_capacity(ids.len() * dim); + let mut norms = Vec::with_capacity(ids.len()); + for &id in &ids { let v = embedding_for(id, dim); - let norm = v.iter().map(|x| x * x).sum::().sqrt(); - norms.push(norm); + norms.push(v.iter().map(|x| x * x).sum::().sqrt()); flat.extend_from_slice(&v); } Self { flat, norms, + ids, dim, - n, } } #[must_use] - pub const fn n(&self) -> u64 { - self.n + pub fn n(&self) -> u64 { + self.ids.len() as u64 } #[must_use] @@ -163,10 +181,11 @@ impl GroundTruth { self.dim } - /// The raw stored vector for `id` (1-based). + /// The raw stored vector at positional `index` (NOT keyed by id — ids may be + /// sparse, so position is the only stable key into `flat`). #[must_use] - fn vector(&self, id: u64) -> &[f32] { - let start = ((id - 1) as usize) * self.dim; + fn vector_at(&self, index: usize) -> &[f32] { + let start = index * self.dim; &self.flat[start..start + self.dim] } @@ -181,12 +200,16 @@ impl GroundTruth { // A bounded top-k kept as a min-by-score Vec (k is tiny, ~10): cheaper and // allocation-lighter than a full sort of n scored pairs per query. let mut top: Vec<(f32, u64)> = Vec::with_capacity(k + 1); - for id in 1..=self.n { - let xn = self.norms[(id - 1) as usize]; + for (index, &id) in self.ids.iter().enumerate() { + let xn = self.norms[index]; if xn == 0.0 { continue; } - let dot: f32 = query.iter().zip(self.vector(id)).map(|(a, b)| a * b).sum(); + let dot: f32 = query + .iter() + .zip(self.vector_at(index)) + .map(|(a, b)| a * b) + .sum(); let score = dot / (q_norm * xn); // cosine; higher = nearer if top.len() < k { top.push((score, id)); @@ -236,8 +259,8 @@ impl QueryPool { // is not clustered on the low ids. let queries: Vec> = (0..pool_size) .map(|i| { - let base = ((i as u64).wrapping_mul(2_654_435_761) % n) + 1; - let mut q = gt.vector(base).to_vec(); + let index = ((i as u64).wrapping_mul(2_654_435_761) % n) as usize; + let mut q = gt.vector_at(index).to_vec(); let mut state = (i as u64) .wrapping_mul(0x100_0000_01B3) .wrapping_add(0xABCD); @@ -505,6 +528,36 @@ mod tests { assert_eq!(top[0], 42, "an item's own vector is its nearest neighbor"); } + #[test] + fn ground_truth_serves_sparse_ids_and_preserves_cluster_structure() { + // The fixture catalog's shape: two 15-item bands in DIFFERENT clusters, + // ids nowhere near 1..=n. A dense oracle could not answer this at all. + let band_a: Vec = (900_000_000..900_000_015).collect(); + let band_b: Vec = (900_000_100..900_000_115).collect(); + let ids: Vec = band_a.iter().chain(&band_b).copied().collect(); + let gt = GroundTruth::from_ids(ids, 64); + assert_eq!(gt.n(), 30); + + // An item's own vector still ranks itself first. + let q = embedding_for(900_000_107, 64); + let top = gt.top_k(&q, 15); + assert_eq!( + top[0], 900_000_107, + "own vector is its own nearest neighbor" + ); + + // And every one of the 15 nearest is its OWN cluster-mate: the bands are + // separable, which is exactly the property that makes a 60-item oracle + // the global oracle for the fixture catalog. + for id in &top { + assert!( + band_b.contains(id), + "id {id} from a foreign cluster outranked a cluster-mate; the \ + ground-truth separation the fixture depends on is broken" + ); + } + } + #[test] fn recall_at_k_counts_overlap_over_truth_size() { let truth = vec![1, 2, 3, 4, 5]; diff --git a/tsconfig.json b/tsconfig.json index c1d27bc..b1b75dc 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -10,7 +10,10 @@ "skipLibCheck": true, "resolveJsonModule": true, "noEmit": true, - "types": ["node"] + "types": ["node"], + // Explicit `.ts` import specifiers: tests/e2e/app/* must be loadable by bare + // Node for `npm run app:dev`, and Node's ESM resolver does not guess extensions. + "allowImportingTsExtensions": true }, "include": [ "playwright.config.ts",