diff --git a/AGENTS.md b/AGENTS.md index a7ac6ad..011f117 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,6 +2,23 @@ Agent instructions for tidalDB. +## Deploy Verification — mandatory + +**No deploy is done until `npm run test:e2e` is green against the live cluster.** 34 +checks, ~50s. Then `npm run test:e2e:semantics` (5, hermetic). + +Video: `npm run test:demo && npm run demo:promote && npm run demo:preflight && npm run demo:render`. +`demo:promote` is NOT optional — it re-stamps the manifest from the live StatefulSet. +Skipping it renders stale frames that preflight will still call "audited perfect". + +**Maintaining the suite is part of the change, not follow-up.** It pins current reality +including defects, so a correct improvement WILL turn it red — that is the harness +working. When a test says "good news — invert this assertion", invert it and fix the +runbook section it names. Never add `retries`, never loosen a threshold to make red go +away, and calibrate every threshold against a measurement rather than a guess. + +Full contract: `CLAUDE.md §Deploy Verification` · detail: `docs/runbooks/deploy-verification.md`. + ## Team | Agent | Identity | Model | Invoke when | diff --git a/CLAUDE.md b/CLAUDE.md index 2b6acc7..62bdfa3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -93,6 +93,47 @@ Dev servers use port range **59520–59529** (e.g. `site/` on 59520). - **Single-node first:** Embeddable. Scales vertically before horizontally. - **Language:** Rust. - **Docs have two canonical homes:** top-level `*.md` and `docs/`. Edit the canonical file — never a per-crate mirror. `.sdlc/` is live SDLC tooling and `applications/*/` docs belong to those consumer products; neither is part of the database doc set. +- **Every deploy MUST be verified through the Playwright harness, and the harness MUST be maintained.** See `## Deploy Verification`. + +## Deploy Verification + +**Non-negotiable: no deploy is done until this suite is green. Run it after every roll.** + +```bash +export KUBECONFIG=~/.kube/orchard9-k3sf.yaml +npm install && npx playwright install chromium # first time +cargo build -p tidalctl # section 7 needs it +npm run test:e2e # 34 checks, ~50s, LIVE cluster +npm run test:e2e:semantics # 5 checks, hermetic, no cluster +``` + +Video walkthrough, in this order (`demo:promote` is NOT optional — it is the step +that re-stamps the manifest from the live StatefulSet): + +```bash +npm run test:demo && npm run demo:promote && npm run demo:preflight && npm run demo:render +npm run demo:promote -- --audited "" # after opening the frames +``` + +Full detail: `docs/runbooks/deploy-verification.md`. + +### Maintaining it is part of the change, not follow-up + +This suite deliberately pins **current** reality, including defects and gaps. So a +correct improvement WILL turn it red. That failure is the harness working. + +- A test fails claiming "good news — invert this assertion" → **invert it**, and fix the + runbook section it names. Do not skip, retry, or loosen it. +- Never add `retries` to `playwright.config.ts`. `retries: 0` is deliberate: a live check + that only passes on attempt two has told you something true. +- A flake means the assertion is wrong, not that the suite is unreliable. Readiness is + polled (`waitForPodsReady`) because `reseed_self_restart: true` makes a bounded + exit(0)/reinstall cycle designed behavior. +- Calibrate thresholds against a measurement, never a guess. A reseed restart produces + ~1200-2000 consecutive ship failures; a count ceiling cannot separate that from a stuck + shipper. Assert whether the burst **recovered**. +- Stale evidence is worse than none: the promoted captures and rendered video sat two + image rolls behind while preflight reported "audited perfect". ## Repository Structure diff --git a/demo/capture-manifest.json b/demo/capture-manifest.json index 16652df..086c4ef 100644 --- a/demo/capture-manifest.json +++ b/demo/capture-manifest.json @@ -1,7 +1,7 @@ { "schemaVersion": 1, - "buildRevision": "d21a202", - "verifiedImage": "registry.threesix.ai/tidal/server:m12-admin-gate-20260823@sha256:6e220060a342658b734d258245b20f6233d96e26415b3a44956b1c3bceebe48c", + "buildRevision": "59d7dad", + "verifiedImage": "registry.threesix.ai/tidal/server:m12-vsc-20260830@sha256:5c18d2b10f71d7ed63f776e45a7d4dba2889a52087b2eb11a6509afe0df0f1cf", "capturedEnvironment": "orchard9-k3sf / namespace tidaldb-cluster (live)", "viewport": { "width": 1600, @@ -11,7 +11,7 @@ { "id": "CAP-002-convergence", "capabilityId": "CAP-002", - "testId": "workflows/deploy-verification.demo.spec.ts :: demo capture \u2014 deploy verification :: CAP-002 every node is converged", + "testId": "workflows/deploy-verification.demo.spec.ts :: demo capture — deploy verification :: CAP-002 every node is converged", "file": "captures/CAP-002-convergence.png", "expected": "Three nodes, zero lag on every shard group, no reseed pending", "businessPurpose": "Quorum with one-node fault tolerance actually exists, rather than being assumed from pod readiness", @@ -20,14 +20,16 @@ ], "width": 1600, "height": 900, - "contentHash": "sha256:69da2d8601585268be468d049afc4747aa2e6ae4c7c1f46b927e0de7ab5ca1b9", + "contentHash": "sha256:afd700c9860e46452721d49c3ab754e54c54cc6f62f0f63c2d47234c09a49b9a", "audienceVerdict": "perfect", - "auditStatus": "pass" + "auditStatus": "pass", + "auditedAt": "2026-08-30T21:26:24.971Z", + "auditNote": "Re-audited 2026-08-30 after the final gate re-shot all 10 captures. Verified content current for m12-vsc-20260830, no secrets rendered, all legible at full resolution; CAP-015 correctly shows HTTP metrics live with unstructured logs as the remaining gap." }, { "id": "CAP-006-quorum-write", "capabilityId": "CAP-006", - "testId": "workflows/deploy-verification.demo.spec.ts :: demo capture \u2014 deploy verification :: CAP-005 CAP-006 the boundary refuses, then a quorum write commits", + "testId": "workflows/deploy-verification.demo.spec.ts :: demo capture — deploy verification :: CAP-005 CAP-006 the boundary refuses, then a quorum write commits", "file": "captures/CAP-006-quorum-write.png", "expected": "401, 401, 200, then 201 for a quorum-acked write", "businessPurpose": "The single strongest available proof: the full stack works and the data plane is closed to strangers", @@ -38,29 +40,14 @@ "height": 900, "contentHash": "sha256:f1fe76f19643382349006988d5bbddb68a501c1bf3037720c07e98a5a27ddfea", "audienceVerdict": "perfect", - "auditStatus": "pass" - }, - { - "id": "CAP-016-feed-reorder", - "capabilityId": "CAP-016", - "testId": "workflows/feed-app.demo.spec.ts :: feed-app product surface :: CAP-016 a signal write reorders the feed immediately", - "file": "captures/CAP-016-feed-reorder.png", - "expected": "The same query, before and after one like: the liked item moves from last to first, with its like_boost visible", - "businessPurpose": "A signal write changes the order with no ETL in between \u2014 the product thesis, not the deployment", - "personas": [ - "cluster operator", - "application developer" - ], - "width": 1600, - "height": 900, - "contentHash": "sha256:ede7ca3fab758d46cabdf1185f6645bc3055c7e8bbb25e380f8cff509d2136b4", - "audienceVerdict": "perfect", - "auditStatus": "pass" + "auditStatus": "pass", + "auditedAt": "2026-08-30T21:26:24.971Z", + "auditNote": "Re-audited 2026-08-30 after the final gate re-shot all 10 captures. Verified content current for m12-vsc-20260830, no secrets rendered, all legible at full resolution; CAP-015 correctly shows HTTP metrics live with unstructured logs as the remaining gap." }, { "id": "CAP-008-network-isolation", "capabilityId": "CAP-008", - "testId": "workflows/deploy-verification.demo.spec.ts :: demo capture \u2014 deploy verification :: CAP-008 the metrics port is closed to foreign pods but open to the scraper", + "testId": "workflows/deploy-verification.demo.spec.ts :: demo capture — deploy verification :: CAP-008 the metrics port is closed to foreign pods but open to the scraper", "file": "captures/CAP-008-network-isolation.png", "expected": "Connection refused from a foreign namespace; hundreds of series to the scraper", "businessPurpose": "Least-privilege network access without blinding the monitoring stack", @@ -69,30 +56,70 @@ ], "width": 1600, "height": 900, - "contentHash": "sha256:247fce3324c519319176a20095a1c7a7b2b5eb5ad6745ff9046e8b6886271c6a", + "contentHash": "sha256:adc14b987a4f47525f3710f31ebf5fbff0db7976053ff5ea758ce3e1f27699eb", "audienceVerdict": "perfect", - "auditStatus": "pass" + "auditStatus": "pass", + "auditedAt": "2026-08-30T21:26:24.971Z", + "auditNote": "Re-audited 2026-08-30 after the final gate re-shot all 10 captures. Verified content current for m12-vsc-20260830, no secrets rendered, all legible at full resolution; CAP-015 correctly shows HTTP metrics live with unstructured logs as the remaining gap." }, { "id": "CAP-010-dashboard", "capabilityId": "CAP-010", - "testId": "workflows/deploy-verification.demo.spec.ts :: demo capture \u2014 deploy verification :: CAP-010 the operator dashboard renders live data", + "testId": "workflows/deploy-verification.demo.spec.ts :: demo capture — deploy verification :: CAP-010 the operator dashboard renders live data", "file": "captures/CAP-010-dashboard.png", - "expected": "Cluster health OK, reseed none, corpus size, and per-node latency charts \u2014 legible at delivery resolution", + "expected": "Cluster health OK, reseed none, corpus size, and per-node latency charts — legible at delivery resolution", "businessPurpose": "The first surface an operator opens during an incident actually shows the cluster", "personas": [ "cluster operator" ], "width": 1600, "height": 502, - "contentHash": "sha256:296570addb43426d4fbb1f8bb69b8a1fac9c691d9cdccf2e927a5db81e4407fe", + "contentHash": "sha256:6a1b5773470e4a3b48b9e35395195dacb691c35ba3c2701cff58ce00b7826215", "audienceVerdict": "perfect", - "auditStatus": "pass" + "auditStatus": "pass", + "auditedAt": "2026-08-30T21:26:24.971Z", + "auditNote": "Re-audited 2026-08-30 after the final gate re-shot all 10 captures. Verified content current for m12-vsc-20260830, no secrets rendered, all legible at full resolution; CAP-015 correctly shows HTTP metrics live with unstructured logs as the remaining gap." + }, + { + "id": "CAP-012-tidalctl", + "capabilityId": "CAP-012", + "testId": "workflows/deploy-verification.demo.spec.ts :: demo capture — deploy verification :: CAP-012 tidalctl gives an operator a live view and an exit code", + "file": "captures/CAP-012-tidalctl.png", + "expected": "Leader, region table with NO REPORT markers, shard table, exit 2", + "businessPurpose": "An operator can interrogate the cluster without hand-rolling curl, and is told what the tool cannot see", + "personas": [ + "cluster operator" + ], + "width": 1600, + "height": 900, + "contentHash": "sha256:6ac6962b86e65242599b103e684c39df53fa827dc6b9c325cd84128bc3348434", + "audienceVerdict": "perfect", + "auditStatus": "pass", + "auditedAt": "2026-08-30T21:26:24.971Z", + "auditNote": "Re-audited 2026-08-30 after the final gate re-shot all 10 captures. Verified content current for m12-vsc-20260830, no secrets rendered, all legible at full resolution; CAP-015 correctly shows HTTP metrics live with unstructured logs as the remaining gap." + }, + { + "id": "CAP-013-backup", + "capabilityId": "CAP-013", + "testId": "workflows/deploy-verification.demo.spec.ts :: demo capture — deploy verification :: CAP-013 the fleet backup captured every volume", + "file": "captures/CAP-013-backup.png", + "expected": "Completed, all items, every PodVolumeBackup Completed", + "businessPurpose": "The cluster can actually be restored, and the alert is trustworthy", + "personas": [ + "cluster operator" + ], + "width": 1600, + "height": 900, + "contentHash": "sha256:9ba62f24fac0777556b178fa10eb173f880a8893bfe0f7a0c6e2c834c18ff177", + "audienceVerdict": "perfect", + "auditStatus": "pass", + "auditedAt": "2026-08-30T21:26:24.971Z", + "auditNote": "Re-audited 2026-08-30 after the final gate re-shot all 10 captures. Verified content current for m12-vsc-20260830, no secrets rendered, all legible at full resolution; CAP-015 correctly shows HTTP metrics live with unstructured logs as the remaining gap." }, { "id": "CAP-014-authority", "capabilityId": "CAP-014", - "testId": "workflows/deploy-verification.demo.spec.ts :: demo capture \u2014 deploy verification :: CAP-014 operator authority is separate from data access", + "testId": "workflows/deploy-verification.demo.spec.ts :: demo capture — deploy verification :: CAP-014 operator authority is separate from data access", "file": "captures/CAP-014-authority.png", "expected": "403 for the data credential, not-403 for the admin credential", "businessPurpose": "Blast radius of a leaked application key is bounded to data, not cluster topology", @@ -103,12 +130,14 @@ "height": 900, "contentHash": "sha256:9897670f83c5c408b57a9c88c0097d3fcc43f983ad463be7bfbcabc49e3522b1", "audienceVerdict": "perfect", - "auditStatus": "pass" + "auditStatus": "pass", + "auditedAt": "2026-08-30T21:26:24.971Z", + "auditNote": "Re-audited 2026-08-30 after the final gate re-shot all 10 captures. Verified content current for m12-vsc-20260830, no secrets rendered, all legible at full resolution; CAP-015 correctly shows HTTP metrics live with unstructured logs as the remaining gap." }, { "id": "CAP-014-drift", "capabilityId": "CAP-014", - "testId": "workflows/deploy-verification.demo.spec.ts :: demo capture \u2014 deploy verification :: CAP-014 CAP-015 the harness corrected its own runbook", + "testId": "workflows/deploy-verification.demo.spec.ts :: demo capture — deploy verification :: CAP-014 CAP-015 the harness corrected its own runbook", "file": "captures/CAP-014-drift.png", "expected": "The superseded claim beside the live probe that contradicts it", "businessPurpose": "Verification that audits its own documentation instead of drifting away from it", @@ -117,57 +146,48 @@ ], "width": 1600, "height": 900, - "contentHash": "sha256:7901e70f8df834a91f1399d1ad8fcf4398546bef14fa46ad23ec605e9d12a344", + "contentHash": "sha256:545a9b619e5db931bdd25b29706ab9d5c4f19c6c4b420c24749140342218e52b", "audienceVerdict": "perfect", - "auditStatus": "pass" - }, - { - "id": "CAP-013-backup", - "capabilityId": "CAP-013", - "testId": "workflows/deploy-verification.demo.spec.ts :: demo capture \u2014 deploy verification :: CAP-013 the fleet backup captured every volume", - "file": "captures/CAP-013-backup.png", - "expected": "Completed, all items, every PodVolumeBackup Completed", - "businessPurpose": "The cluster can actually be restored, and the alert is trustworthy", - "personas": [ - "cluster operator" - ], - "width": 1600, - "height": 900, - "contentHash": "sha256:e570917de51ac352a18949117dbf6c9e990fc75127b1bc6c71d0473e2f21371f", - "audienceVerdict": "perfect", - "auditStatus": "pass" + "auditStatus": "pass", + "auditedAt": "2026-08-30T21:26:24.971Z", + "auditNote": "Re-audited 2026-08-30 after the final gate re-shot all 10 captures. Verified content current for m12-vsc-20260830, no secrets rendered, all legible at full resolution; CAP-015 correctly shows HTTP metrics live with unstructured logs as the remaining gap." }, { "id": "CAP-015-inert", "capabilityId": "CAP-015", - "testId": "workflows/deploy-verification.demo.spec.ts :: demo capture \u2014 deploy verification :: CAP-015 what is NOT verified is stated", + "testId": "workflows/deploy-verification.demo.spec.ts :: demo capture — deploy verification :: CAP-015 what is NOT verified is stated", "file": "captures/CAP-015-inert.png", - "expected": "Zero tidaldb_http_* families while baseline tidaldb_* families are present", + "expected": "HTTP metric families present, and no JSON_LOGS on the StatefulSet", "businessPurpose": "A verification that hides its gaps cannot be trusted about the parts it claims", "personas": [ "cluster operator" ], "width": 1600, "height": 900, - "contentHash": "sha256:671ce44f8e3a629f88cab8f807c9389e068660f51a00e68232c9f92c87b55a76", + "contentHash": "sha256:f853a0d6163bc6cf3fa58f1dd36e9924dee6814bf2f9b2dfb1660e677d81fcc3", "audienceVerdict": "perfect", - "auditStatus": "pass" + "auditStatus": "pass", + "auditedAt": "2026-08-30T21:26:24.971Z", + "auditNote": "Re-audited 2026-08-30 after the final gate re-shot all 10 captures. Verified content current for m12-vsc-20260830, no secrets rendered, all legible at full resolution; CAP-015 correctly shows HTTP metrics live with unstructured logs as the remaining gap." }, { - "id": "CAP-012-tidalctl", - "capabilityId": "CAP-012", - "testId": "workflows/deploy-verification.demo.spec.ts :: demo capture \u2014 deploy verification :: CAP-012 tidalctl gives an operator a live view and an exit code", - "file": "captures/CAP-012-tidalctl.png", - "expected": "Leader, region table with NO REPORT markers, shard table, exit 2", - "businessPurpose": "An operator can interrogate the cluster without hand-rolling curl, and is told what the tool cannot see", + "id": "CAP-016-feed-reorder", + "capabilityId": "CAP-016", + "testId": "workflows/feed-app.demo.spec.ts :: feed-app product surface :: CAP-016 a signal write reorders the feed immediately", + "file": "captures/CAP-016-feed-reorder.png", + "expected": "The same query, before and after one like: the liked item moves from last to first, with its like_boost visible", + "businessPurpose": "A signal write changes the order with no ETL in between — the product thesis, not the deployment", "personas": [ - "cluster operator" + "cluster operator", + "application developer" ], "width": 1600, "height": 900, - "contentHash": "sha256:e13e0bfecc766d02f839334487cf0364472c43bf0f1fbe1a11159e6312a58aa6", + "contentHash": "sha256:ede7ca3fab758d46cabdf1185f6645bc3055c7e8bbb25e380f8cff509d2136b4", "audienceVerdict": "perfect", - "auditStatus": "pass" + "auditStatus": "pass", + "auditedAt": "2026-08-30T21:26:24.971Z", + "auditNote": "Re-audited 2026-08-30 after the final gate re-shot all 10 captures. Verified content current for m12-vsc-20260830, no secrets rendered, all legible at full resolution; CAP-015 correctly shows HTTP metrics live with unstructured logs as the remaining gap." } ] } diff --git a/demo/preflight.ts b/demo/preflight.ts index fad5ccf..ba28925 100644 --- a/demo/preflight.ts +++ b/demo/preflight.ts @@ -88,6 +88,33 @@ async function main(): Promise { process.stdout.write(` note: promoted but unused by any scene: ${orphans.join(', ')}\n`); } + // The title card must not contradict the manifest. It hardcoded the image tag + // and check count until 2026-08-30 and drifted two image rolls behind, so the + // rendered walkthrough described a deployment that was no longer running. + // scenes.ts now derives both; this asserts the derivation is actually in place, + // because a future edit could paste a literal back in. + const verifiedTag = + manifest.verifiedImage.split('@')[0]?.split(':').pop() ?? manifest.verifiedImage; + // Strip comments before scanning: this file's own explanation of the rule cites + // the stale literal as an example, and matching that would fail forever. + const scenesCode = scenes + .replace(/\/\*[\s\S]*?\*\//g, '') + .split('\n') + .filter((line) => !line.trim().startsWith('//') && !line.trim().startsWith('*')) + .join('\n'); + if (!scenesCode.includes('DEPLOYMENT_FOOTER')) { + problems.push( + 'scenes.ts no longer uses DEPLOYMENT_FOOTER — the title card has been hardcoded again ' + + 'and will drift from the verified image on the next roll', + ); + } + if (/image m12-[a-z0-9-]+ ·/.test(scenesCode)) { + problems.push( + `scenes.ts contains a hardcoded image tag; it must derive from manifest.verifiedImage ` + + `(currently ${verifiedTag})`, + ); + } + if (problems.length > 0) { process.stderr.write(`\npreflight FAILED:\n${problems.map((p) => ` ✗ ${p}`).join('\n')}\n`); process.exit(1); diff --git a/demo/promote.ts b/demo/promote.ts new file mode 100644 index 0000000..a73b0cb --- /dev/null +++ b/demo/promote.ts @@ -0,0 +1,166 @@ +/** + * Promote a capture run into the Remotion asset set. + * + * The capture suite writes PNGs and manifest fragments to + * `test-results/demo-captures/` — deliberately NOT into `demo/public/captures/`, + * because a capture becomes a published frame only after someone has looked at + * it. `demo/preflight.ts` enforces that gate: it refuses any capture whose + * `audienceVerdict` is not `perfect`. + * + * Until now the copy-and-merge step between those two directories did not exist + * — it was done by hand on 2026-08-23, which is exactly why the promoted set and + * its manifest still pointed at `m12-admin-gate-20260823` a week and two image + * rolls later, while `preflight` reported "audited perfect" about stale frames. + * A verification artifact that silently describes an old deployment is worse than + * no artifact. + * + * Usage: + * node --experimental-strip-types demo/promote.ts + * Copy + merge + stamp the live revision. Verdicts land as `pending`, so + * `preflight` FAILS until the frames are audited. That failure is correct. + * + * node --experimental-strip-types demo/promote.ts --audited "" + * Mark every promoted capture audited, recording the note and a timestamp. + * Only pass this after actually opening the images at full resolution + * against `demo/audience-brief.md`. + */ + +import { createHash } from 'node:crypto'; +import { execFile } from 'node:child_process'; +import { copyFile, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +const SOURCE_DIR = join('test-results', 'demo-captures'); +const TARGET_DIR = join('demo', 'public', 'captures'); +const MANIFEST = join('demo', 'capture-manifest.json'); + +type CaptureRow = { + id: string; + file: string; + contentHash: string; + audienceVerdict: string; + auditStatus: string; + auditedAt?: string; + auditNote?: string; + [key: string]: unknown; +}; + +type Manifest = { + schemaVersion: string; + buildRevision: string; + verifiedImage: string; + capturedEnvironment: string; + viewport: { width: number; height: number }; + captures: CaptureRow[]; +}; + +const auditedIndex = process.argv.indexOf('--audited'); +const audited = auditedIndex !== -1; +const auditNote = audited ? (process.argv[auditedIndex + 1] ?? '').trim() : ''; + +if (audited && auditNote === '') { + process.stderr.write( + 'promote: --audited requires a note describing what was checked.\n' + + 'An audit with no statement of what was looked at is not an audit.\n', + ); + process.exit(2); +} + +const fragments = (await readdir(SOURCE_DIR)).filter( + (name) => name.startsWith('manifest-') && name.endsWith('.json'), +); +if (fragments.length === 0) { + process.stderr.write( + `promote: no manifest fragments in ${SOURCE_DIR}.\n` + + `Run 'npm run test:demo' first — promotion cannot invent a capture run.\n`, + ); + process.exit(2); +} + +const rows: CaptureRow[] = []; +for (const fragment of fragments) { + const parsed = JSON.parse(await readFile(join(SOURCE_DIR, fragment), 'utf8')) as CaptureRow[]; + rows.push(...parsed); +} +rows.sort((left, right) => left.id.localeCompare(right.id)); + +const duplicates = rows.map((row) => row.id).filter((id, index, all) => all.indexOf(id) !== index); +if (duplicates.length > 0) { + process.stderr.write(`promote: duplicate capture ids across fragments: ${duplicates.join(', ')}\n`); + process.exit(2); +} + +// Stamp what was ACTUALLY verified, read from the cluster and the repo rather +// than carried over from the previous manifest. A stale stamp is the whole defect +// this script exists to close. +const { stdout: revisionOut } = await execFileAsync('git', ['rev-parse', '--short', 'HEAD']); +const { stdout: imageOut } = await execFileAsync('kubectl', [ + '-n', + process.env.E2E_NAMESPACE ?? 'tidaldb-cluster', + 'get', + 'statefulset', + 'tidaldb', + '-o', + 'jsonpath={.spec.template.spec.containers[0].image}', +]); + +const previous = JSON.parse(await readFile(MANIFEST, 'utf8')) as Manifest; + +await mkdir(TARGET_DIR, { recursive: true }); + +// Remove any promoted file that this run did not produce. Preflight already +// refuses an orphan, but leaving one here would make a re-run of THIS script the +// thing that finally reports it, one step further from the cause. +const promotedIds = new Set(rows.map((row) => row.id)); +for (const existing of await readdir(TARGET_DIR)) { + if (!existing.endsWith('.png')) continue; + if (!promotedIds.has(existing.replace(/\.png$/, ''))) { + process.stdout.write(` removing orphan ${existing} (no row in this capture run)\n`); + await rm(join(TARGET_DIR, existing)); + } +} + +const stampedAt = new Date().toISOString(); +for (const row of rows) { + const name = `${row.id}.png`; + const bytes = await readFile(join(SOURCE_DIR, name)); + const hash = `sha256:${createHash('sha256').update(bytes).digest('hex')}`; + + if (hash !== row.contentHash) { + process.stderr.write( + `promote: ${row.id} hash mismatch — the fragment says ${row.contentHash} but the PNG ` + + `hashes to ${hash}. The capture run and its manifest disagree; re-run the suite.\n`, + ); + process.exit(2); + } + + await copyFile(join(SOURCE_DIR, name), join(TARGET_DIR, name)); + + if (audited) { + row.auditStatus = 'pass'; + row.audienceVerdict = 'perfect'; + row.auditedAt = stampedAt; + row.auditNote = auditNote; + } +} + +const manifest: Manifest = { + schemaVersion: previous.schemaVersion, + buildRevision: revisionOut.trim(), + verifiedImage: imageOut.trim(), + capturedEnvironment: previous.capturedEnvironment, + viewport: previous.viewport, + captures: rows, +}; + +await writeFile(MANIFEST, `${JSON.stringify(manifest, null, 2)}\n`); + +process.stdout.write( + `\npromoted ${rows.length} captures\n` + + ` revision: ${manifest.buildRevision}\n` + + ` image: ${manifest.verifiedImage}\n` + + ` verdicts: ${audited ? `audited (${auditNote})` : 'PENDING — preflight will fail until audited'}\n\n`, +); diff --git a/demo/public/captures/CAP-002-convergence.png b/demo/public/captures/CAP-002-convergence.png index 4be3bf6..802d17c 100644 Binary files a/demo/public/captures/CAP-002-convergence.png and b/demo/public/captures/CAP-002-convergence.png differ diff --git a/demo/public/captures/CAP-008-network-isolation.png b/demo/public/captures/CAP-008-network-isolation.png index 389d147..ccc756b 100644 Binary files a/demo/public/captures/CAP-008-network-isolation.png and b/demo/public/captures/CAP-008-network-isolation.png differ diff --git a/demo/public/captures/CAP-010-dashboard.png b/demo/public/captures/CAP-010-dashboard.png index b175d1f..16d4888 100644 Binary files a/demo/public/captures/CAP-010-dashboard.png and b/demo/public/captures/CAP-010-dashboard.png differ diff --git a/demo/public/captures/CAP-012-tidalctl.png b/demo/public/captures/CAP-012-tidalctl.png index a470b43..922bb90 100644 Binary files a/demo/public/captures/CAP-012-tidalctl.png and b/demo/public/captures/CAP-012-tidalctl.png differ diff --git a/demo/public/captures/CAP-013-backup.png b/demo/public/captures/CAP-013-backup.png index 69917a4..3e50c15 100644 Binary files a/demo/public/captures/CAP-013-backup.png and b/demo/public/captures/CAP-013-backup.png differ diff --git a/demo/public/captures/CAP-014-drift.png b/demo/public/captures/CAP-014-drift.png index 42a39e5..395f5e5 100644 Binary files a/demo/public/captures/CAP-014-drift.png and b/demo/public/captures/CAP-014-drift.png differ diff --git a/demo/public/captures/CAP-015-inert.png b/demo/public/captures/CAP-015-inert.png index 57b7ca8..f4ddd0e 100644 Binary files a/demo/public/captures/CAP-015-inert.png and b/demo/public/captures/CAP-015-inert.png differ diff --git a/demo/src/scenes.ts b/demo/src/scenes.ts index 8d22141..1909ec0 100644 --- a/demo/src/scenes.ts +++ b/demo/src/scenes.ts @@ -5,6 +5,29 @@ * a defect the preflight rejects. */ +import manifest from '../capture-manifest.json' with { type: 'json' }; + +/** + * Deployment facts, DERIVED from the capture manifest rather than retyped here. + * + * These were hardcoded until 2026-08-30, and went stale exactly as you would + * expect: the title card still read `image m12-admin-gate-20260823 · 32 checks + * green` after two image rolls and four new tests, so the rendered walkthrough + * confidently described a deployment that had not been running for a week. The + * manifest is stamped by `demo/promote.ts` from the live StatefulSet, so reading + * it here means the frame cannot disagree with what was captured. + * + * `CHECK_COUNT` still has to be maintained by hand — Playwright's total is not in + * the manifest — but it now sits beside the value it must agree with, and + * `demo/preflight.ts` asserts the title card matches the manifest image. + */ +const VERIFIED_IMAGE_TAG = + manifest.verifiedImage.split('@')[0]?.split(':').pop() ?? manifest.verifiedImage; +const CHECK_COUNT = 34; +export const DEPLOYMENT_FOOTER = + `namespace tidaldb-cluster · image ${VERIFIED_IMAGE_TAG} · ${CHECK_COUNT} checks green`; +export const CHECKS_LINE = `${CHECK_COUNT} checks green against the live deployment.`; + export const FPS = 30; export const WIDTH = 1920; export const HEIGHT = 1080; @@ -52,7 +75,7 @@ export const scenes: Scene[] = [ 'Three voters on orchard9-k3sf. One public endpoint. A runbook an operator can walk.', 'Every number that follows came from a command that ran against the live cluster.', ], - footer: 'namespace tidaldb-cluster · image m12-admin-gate-20260823 · 32 checks green', + footer: DEPLOYMENT_FOOTER, }, { kind: 'proof', @@ -171,8 +194,8 @@ export const scenes: Scene[] = [ rung: 'need', heading: 'Verified, including the gaps', lines: [ - '32 checks green against the live deployment.', - 'Two committed features are absent from the running image — and the suite asserts that absence deliberately.', + CHECKS_LINE, + 'One gap remains — unstructured logs — and the suite asserts that absence deliberately.', 'The day it changes, the suite fails and says so.', ], footer: 'docs/runbooks/deploy-verification.md · npm run test:e2e', diff --git a/demo/visual-audit.md b/demo/visual-audit.md index 139b996..07959e9 100644 --- a/demo/visual-audit.md +++ b/demo/visual-audit.md @@ -4,10 +4,20 @@ Audited against `audience-brief.md`. Per-screen verdicts are judged at the **quality-bar judge's** standard (Kyle Kingsbury); the walk-the-render ledger is written in the **actual decision-maker's** voice (Jordan Washburn). -- Build revision: see `capture-manifest.json.buildRevision` -- Verified image: `registry.threesix.ai/tidal/server:m12-admin-gate-20260823@sha256:6e220060…` +- Build revision: see `capture-manifest.json.buildRevision` (do not retype it here) +- Verified image: see `capture-manifest.json.verifiedImage` — stamped from the live + StatefulSet by `demo:promote`. It read `m12-admin-gate-20260823` here for a week + after two image rolls, which is why this file no longer carries its own copy. - Render: `demo/out/deploy-verification.mp4` — 90.05 s, 1920×1080, 30 fps, h264, 2700 frames - Regression suite at time of capture: **34 passed** (32 deployment + 2 cluster ranking tripwires) + +> **Re-audited 2026-08-30** for `m12-vsc-20260830` (`8aa1fbb`). All 10 captures +> re-shot and re-audited at full resolution; per-capture verdicts and the audit note +> now live in `capture-manifest.json` (`auditedAt`, `auditNote`) rather than only in +> this prose. Two panels changed meaning with the roll: `CAP-015-inert` now reports +> HTTP metrics LIVE with unstructured logs as the remaining gap, and the title card +> derives its image tag and check count from the manifest instead of hardcoding them. +> The tables below are the 2026-08-23 audit, retained as the prior record. - Hermetic ranking-semantics suite: **5 passed** (`npm run test:e2e:semantics`, no cluster required) - Demo capture suite: **10 passed** (each asserts before it photographs) diff --git a/docs/runbooks/deploy-verification.md b/docs/runbooks/deploy-verification.md index f6a191e..a754a67 100644 --- a/docs/runbooks/deploy-verification.md +++ b/docs/runbooks/deploy-verification.md @@ -47,6 +47,42 @@ half-life, and that ANN results equal brute-force cosine. It has its own config (`playwright.semantics.config.ts`) precisely because it must run with no cluster and no credentials — `npm run test:all` runs both suites. +### Capturing the walkthrough (evidence a human can watch) + +The regression suite answers "is the deployment correct?". The capture suite +answers "can someone watch the proof?" — it photographs states the regression +suite has *already asserted*, then Remotion assembles them into one video. + +```bash +npm run test:demo # 10 captures, each asserts before it photographs +npm run demo:promote # copy captures in + stamp the LIVE image/revision +npm run demo:preflight # gate: every capture present, hash-stable, audited +npm run demo:render # -> demo/out/deploy-verification.mp4 (90 s, 1920x1080) +``` + +**Run these in order, and never skip `demo:promote`.** The capture suite writes to +`test-results/demo-captures/`, not into `demo/public/captures/`, because a capture +becomes a published frame only after someone looks at it. Promotion is the step +that copies frames across and re-stamps the manifest from the live StatefulSet. + +That step did not exist until 2026-08-30 — it was done by hand once — and the +consequence is the exact failure this runbook exists to prevent: the promoted +frames and the rendered video still described `m12-admin-gate-20260823` after two +image rolls, while `demo:preflight` cheerfully reported "audited perfect" about +week-old evidence. `demo:render` will happily encode stale PNGs; nothing else +notices. + +`demo:promote` leaves every verdict `pending`, so `demo:preflight` **fails** until +the frames are audited. That failure is correct — clear it by opening the images +at full resolution against `demo/audience-brief.md` and then: + +```bash +npm run demo:promote -- --audited "" +``` + +The note is stored in the manifest beside the verdict. An audit with no statement +of what was looked at is not an audit. + It deliberately never touches the deployed cluster: `skip` is declared `permanent: true`, so seeding signals into the live corpus would be irreversible. Two of its five checks report a product gap rather than a success @@ -440,11 +476,27 @@ metadata: ## 9. Operator authority, and what is still inert The cluster runs -`registry.threesix.ai/tidal/server:m12-admin-gate-20260823@sha256:6e220060…` -(pods started 2026-08-23 05:32–05:41 UTC). That image carries the -operator/data credential split but **predates** the observability commit, so -§9.1 and §9.3 below are still inert. They are listed so their absence is not -mistaken for a regression. +`registry.threesix.ai/tidal/server:m12-vsc-20260830@sha256:5c18d2b1…` +(rolled 2026-08-30, built from `8aa1fbb`). That image carries the operator/data +credential split (§9.2), the HTTP request metrics (§9.1), and the +blob-replication ledger. + +**§9.1 went LIVE with this roll.** It was inert for the previous two images, and +this document said so — until the harness contradicted it. Both +`09-operator-authority.spec.ts` and the `CAP-015` capture asserted +`tidaldb_http_* = 0`; the roll made that false and they failed, which is how the +drift surfaced within minutes instead of rotting here. Both assertions are now +inverted, so a rollback that removes the HTTP metrics fails them again. + +Live reading after the roll: `tidaldb_http_* = 185` against a baseline of +`tidaldb_* = 552` — the baseline is what proves the scrape worked, so "present" +is distinguishable from "unscraped". + +**§9.3 remains inert.** The StatefulSet still carries no `JSON_LOGS`, so logs are +unstructured plain text and VictoriaLogs `level:error` cannot match them; filter +at the source. Container logs no longer carry ANSI escapes (BUG-006 resolved on +this image), and `06-logs.spec.ts` now pins that in the other direction — a +regression to coloured output fails there. ### 9.2 Operator/data credential split — LIVE, verify it stays that way diff --git a/package.json b/package.json index 0234f72..b235859 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,8 @@ "demo:preflight": "node --experimental-strip-types demo/preflight.ts", "demo:studio": "remotion studio demo/src/index.ts", "demo:still": "remotion still demo/src/index.ts DeployVerification demo/out/still.png", - "demo:render": "remotion render demo/src/index.ts DeployVerification demo/out/deploy-verification.mp4" + "demo:render": "remotion render demo/src/index.ts DeployVerification demo/out/deploy-verification.mp4", + "demo:promote": "node --experimental-strip-types demo/promote.ts" }, "devDependencies": { "@playwright/test": "^1.56.1", diff --git a/tests/e2e/demo/workflows/deploy-verification.demo.spec.ts b/tests/e2e/demo/workflows/deploy-verification.demo.spec.ts index d3b584d..ca9d3fc 100644 --- a/tests/e2e/demo/workflows/deploy-verification.demo.spec.ts +++ b/tests/e2e/demo/workflows/deploy-verification.demo.spec.ts @@ -645,7 +645,29 @@ test.describe('demo capture — deploy verification', () => { 'jsonpath={.spec.template.spec.containers[0].image}', ]); - expect(counts.stdout).toMatch(/tidaldb_http_\* = 0/); + // Structured logging is the gap that REMAINS. Read it from the StatefulSet's + // own env rather than inferring it from log shape, so the panel shows the + // cause and not a symptom. + const logEnv = await kubectl([ + '-n', + NAMESPACE, + 'get', + 'statefulset', + 'tidaldb', + '-o', + 'jsonpath={range .spec.template.spec.containers[0].env[*]}{.name}={.value}{"\\n"}{end}', + ]); + const jsonLogsConfigured = /JSON_LOGS=(1|true)/i.test(logEnv.stdout); + + // Inverted 2026-08-30. This asserted `tidaldb_http_* = 0` and was correct for + // the image running when it was written; rolling m12-vsc-20260830 made it + // false, and the test failing is how the drift surfaced. The panel's PURPOSE + // is unchanged — state the gaps — so it now names the gap that is still real. + expect(counts.stdout).not.toMatch(/tidaldb_http_\* = 0$/m); + expect( + jsonLogsConfigured, + 'JSON logging became enabled — this panel must stop calling it a gap', + ).toBe(false); records.push( await captureProofPanel( @@ -656,29 +678,41 @@ test.describe('demo capture — deploy verification', () => { capabilityId: 'CAP-015', title: 'What this deployment does not yet do', subtitle: - 'Two committed features are absent from the running image. Stating that is part of ' + - 'the verification, not a footnote to it.', + 'One gap closed with this roll; one remains. Stating both is part of the ' + + 'verification, not a footnote to it.', blocks: [ { command: 'kubectl get statefulset tidaldb -o jsonpath={..image}', output: image.stdout.trim(), - verdict: 'This image carries the credential split but predates the observability commit.', + verdict: + 'This image carries the credential split, the HTTP request metrics, and the ' + + 'blob-replication ledger.', }, { command: 'scrape :9091 and count metric families', output: counts.stdout.trim(), verdict: - 'Zero HTTP metrics — and the baseline count proves the scrape WORKED, so "absent" ' + - 'is distinguishable from "unscraped". Five dashboard panels are legitimately empty.', + 'HTTP metrics are now LIVE — a gap this suite asserted as absent until it ' + + 'failed on this roll and forced the update. The baseline count proves the ' + + 'scrape worked, so the number means what it says.', + }, + { + command: 'kubectl get statefulset tidaldb -o jsonpath={..env}', + output: logEnv.stdout.trim() || '(no JSON_LOGS entry)', + verdict: + 'Still unstructured: no JSON_LOGS. VictoriaLogs level:error cannot match, so ' + + 'log filtering stays at the source. This gap is real and unclosed.', negative: true, }, ], footnote: - 'The suite asserts this absence deliberately: the day the observability image is ' + - 'rolled, these tests fail and say so — instead of the runbook silently rotting.', + 'The suite asserts each gap deliberately, in whichever direction is currently ' + + 'true: the day one closes, these tests fail and say so — instead of the runbook ' + + 'silently rotting. That is exactly what happened to the HTTP-metrics claim above.', }, { - expected: 'Zero tidaldb_http_* families while baseline tidaldb_* families are present', + expected: + 'HTTP metric families present, and no JSON_LOGS on the StatefulSet', businessPurpose: 'A verification that hides its gaps cannot be trusted about the parts it claims', personas: OPERATOR, diff --git a/tests/e2e/features/01-cluster-convergence.spec.ts b/tests/e2e/features/01-cluster-convergence.spec.ts index b71c08f..5ff81d9 100644 --- a/tests/e2e/features/01-cluster-convergence.spec.ts +++ b/tests/e2e/features/01-cluster-convergence.spec.ts @@ -13,7 +13,7 @@ */ import { expect, test } from '@playwright/test'; -import { kubectl, withPortForward } from '../support/cluster'; +import { kubectl, waitForPodsReady, withPortForward } from '../support/cluster'; import { observed, recordJson } from '../support/evidence'; import { NAMESPACE, POD_NAMES, PORT_CLIENT, apiKey } from '../support/env'; @@ -85,8 +85,17 @@ test.describe('section 1 — cluster convergence', () => { for (const pod of pods) { expect(pod.phase, `${pod.name} phase`).toBe('Running'); - expect(pod.ready, `${pod.name} must be Ready`).toBe(true); } + + // Readiness is polled, not sampled: `reseed_self_restart: true` makes a + // bounded exit(0)/reinstall cycle DESIGNED behavior, so one unlucky sample + // would report a converging cluster as a broken one. + const readiness = await waitForPodsReady(NAMESPACE, POD_NAMES); + await recordJson(testInfo, 'pod-readiness-timeline', readiness); + expect( + readiness.converged, + `pods did not all reach Ready within the budget; final=${JSON.stringify(readiness.final)}`, + ).toBe(true); }); test('every node reports zero lag, no reseed, and agrees on one leader', async ({ diff --git a/tests/e2e/features/06-logs.spec.ts b/tests/e2e/features/06-logs.spec.ts index 97720c6..2ce27d1 100644 --- a/tests/e2e/features/06-logs.spec.ts +++ b/tests/e2e/features/06-logs.spec.ts @@ -71,12 +71,44 @@ const BENIGN_WARNINGS: { match: RegExp; why: string }[] = [ }, ]; -/** Replication distress that must be absent in the health window. */ +/** Replication distress that must not be ONGOING in the health window. */ const REPLICATION_DISTRESS = /batch ship failing|transport channel closed|quarantin/i; +/** The leader's own "it is fine again" line, which closes a distress burst. */ +const SHIP_RECOVERED = /peer recovered; shipping resumed/i; + +/* + * There is deliberately NO consecutive-failure ceiling here. + * + * A first version of this test capped `consecutive_failures` at 500, on the guess + * that a restart burst was "~100" and the 2026-08-30 episode's 2697 was + * qualitatively different. Measurement killed that idea: this cluster sets + * `reseed_self_restart: true`, so a node latching a reseed marker exits(0), + * reinstalls a snapshot, and returns — a ~2 minute absence. At the shipper's + * 100ms retry cadence that is ~1200-2000 consecutive failures, entirely normal. + * Observed the same day: tidaldb-2 self-restarted for a shard-1 reseed and + * tidaldb-0 logged 1950 failures, then `peer recovered … failed_attempts=1950`, + * with `peer_acked_seqno` back at the leader's frontier. + * + * So the COUNT cannot separate "a peer restarted" from "shipping is broken" — + * both produce large bursts, and the count only encodes how long the peer was + * away. What separates them is whether the burst CLOSED. That is what this test + * asserts, and it is strictly stronger: a genuinely stuck shipper never logs + * `peer recovered`, so it fails here no matter how high or low its count is. + */ + test.describe('section 6 — logs', () => { - test('replication is quiet right now on every pod', async ({}, testInfo) => { - const perPod: Record = {}; + test('replication distress, if any, is a bounded burst that has already recovered', async ({}, testInfo) => { + const perPod: Record< + string, + { + sampled: number; + distressLines: string[]; + peakConsecutiveFailures: number; + recovered: boolean; + lastDistressIsAfterLastRecovery: boolean; + } + > = {}; for (const pod of POD_NAMES) { const result = await observed(testInfo, `recent logs ${pod}`, () => @@ -89,23 +121,62 @@ test.describe('section 6 — logs', () => { const lines = decolour(result.stdout) .split('\n') .filter((line) => line.trim() !== ''); + + const distressIndexes = lines + .map((line, index) => (REPLICATION_DISTRESS.test(line) ? index : -1)) + .filter((index) => index >= 0); + const recoveryIndexes = lines + .map((line, index) => (SHIP_RECOVERED.test(line) ? index : -1)) + .filter((index) => index >= 0); + + let peak = 0; + for (const index of distressIndexes) { + const match = /consecutive_failures=(\d+)/.exec(lines[index]!); + if (match) peak = Math.max(peak, Number.parseInt(match[1]!, 10)); + } + + const lastDistress = distressIndexes.at(-1) ?? -1; + const lastRecovery = recoveryIndexes.at(-1) ?? -1; + perPod[pod] = { sampled: lines.length, - distressLines: lines.filter((line) => REPLICATION_DISTRESS.test(line)).slice(0, 5), + distressLines: distressIndexes.slice(-5).map((index) => lines[index]!), + peakConsecutiveFailures: peak, + recovered: recoveryIndexes.length > 0, + // The load-bearing question: is the newest distress line NEWER than the + // newest recovery line? If so the burst never closed and shipping is + // still broken right now. + lastDistressIsAfterLastRecovery: lastDistress > lastRecovery, }; } - await recordJson(testInfo, 'replication-health-window', { window: HEALTH_WINDOW, perPod }); + await recordJson(testInfo, 'replication-health-window', { + window: HEALTH_WINDOW, + perPod, + rationale: + 'A burst that has RECOVERED is designed behavior around a reseed_self_restart ' + + '(a ~2min absence is ~1200-2000 retries at 100ms). An UNRECOVERED burst means ' + + 'shipping is still broken right now. peakConsecutiveFailures is recorded as ' + + 'evidence only - it measures how long the peer was away, not whether anything ' + + 'is wrong.', + }); for (const pod of POD_NAMES) { + const state = perPod[pod]!; + if (state.distressLines.length === 0) continue; + // A pod shipping batches into a closed transport channel is not // replicating, even while /cluster/status/local still reports lag=0 - // because the leader has not yet advanced past the stuck position. + // because the leader has not yet advanced past the stuck position. What + // distinguishes "a peer just restarted" from "shipping is broken" is + // whether the burst CLOSED, not whether it happened. expect( - perPod[pod].distressLines, - `${pod} is reporting replication distress in the last ${HEALTH_WINDOW}: ` + - `${perPod[pod].distressLines.join(' | ')}`, - ).toEqual([]); + state.lastDistressIsAfterLastRecovery, + `${pod} replication distress is ONGOING — the newest distress line is newer than ` + + `the newest 'peer recovered' line, so shipping has not resumed: ` + + `${state.distressLines.join(' | ')}`, + ).toBe(false); + } }); @@ -172,7 +243,7 @@ test.describe('section 6 — logs', () => { } }); - test('the deployed image still emits coloured plain text, so level filtering must happen at the source', async ({}, testInfo) => { + test('the deployed image emits uncoloured plain text — level filtering still belongs at the source', async ({}, testInfo) => { const result = await observed(testInfo, 'log format sample', () => kubectl(['-n', NAMESPACE, 'logs', 'tidaldb-0', '--tail=5'], { timeoutMs: 45_000 }), ); @@ -195,23 +266,27 @@ test.describe('section 6 — logs', () => { jsonLines: jsonLines.length, ansiLines: ansiLines.length, conclusion: - 'Coloured plain text. VictoriaLogs level:error cannot match, so filter with ' + - 'kubectl logs | grep until the observability image is rolled (runbook 9.3).', + 'Uncoloured plain text. BUG-006 (ANSI escapes in container logs) is RESOLVED on ' + + 'the deployed image. Logs are still unstructured, so VictoriaLogs `level:error` ' + + 'cannot match and filtering stays at the source (runbook 9.3).', }); - // Two tripwires in the honest direction. Both are CORRECT for the running - // image, and both fail the day the observability image lands — which is - // when the runbook needs updating. That is the drift that made section 9.2 - // stale in the first place. + // BUG-006 resolved 2026-08-30. The previous version of this test asserted + // `ansiLines > 0` — correct for the image running when it was written, and it + // failed the moment a newer image was rolled. That failure is the test doing + // its job: it is how the drift got noticed. Now pinned in the other direction + // so a REGRESSION to coloured output fails here. + expect( + ansiLines.length, + 'ANSI escapes are back in container logs — BUG-006 has regressed. Coloured output ' + + 'breaks log-store level matching and makes every downstream filter guess.', + ).toBe(0); + + // Still-open tripwire, unchanged in direction: logs are NOT yet structured. expect( jsonLines.length, 'logs became structured JSON — roll runbook section 9.3 from pending to live and ' + 'invert this assertion', ).toBe(0); - expect( - ansiLines.length, - 'ANSI escapes are gone — the observability image has been rolled; mark BUG-006 ' + - 'verified and invert this assertion', - ).toBeGreaterThan(0); }); }); diff --git a/tests/e2e/features/09-operator-authority.spec.ts b/tests/e2e/features/09-operator-authority.spec.ts index 9c0f137..9aa5fce 100644 --- a/tests/e2e/features/09-operator-authority.spec.ts +++ b/tests/e2e/features/09-operator-authority.spec.ts @@ -141,7 +141,7 @@ test.describe('section 9 — operator authority', () => { ).toContain('admin-key'); }); - test('HTTP request metrics are absent, and the scrape that proves it actually worked', async ({}, testInfo) => { + test('HTTP request metrics are exported, and the scrape that proves it actually worked', async ({}, testInfo) => { const counts: Record = {}; for (const pod of POD_NAMES) { @@ -191,12 +191,17 @@ test.describe('section 9 — operator authority', () => { `proves nothing`, ).toBeGreaterThan(100); + // Inverted 2026-08-30. This previously asserted `http === 0` and was correct + // for the image running when it was written; it failed the moment a newer + // image was rolled, which is exactly how the drift got noticed. Runbook + // section 9.1 is now LIVE, so a regression to zero HTTP metrics — an image + // rollback, or the route-metrics layer being dropped — fails here. expect( counts[pod].http, - `${pod} now exports tidaldb_http_* metrics — good news: the observability image has ` + - `been rolled. Move runbook section 9.1 from inert to live, confirm the five HTTP ` + - `dashboard panels populate, and invert this assertion.`, - ).toBe(0); + `${pod} exports no tidaldb_http_* metrics. Section 9.1 went live on ` + + `2026-08-30; losing them means an image rollback or the route-metrics layer ` + + `being removed, and the five HTTP dashboard panels are now blank.`, + ).toBeGreaterThan(0); } }); diff --git a/tests/e2e/smoke.spec.ts b/tests/e2e/smoke.spec.ts index 29d07b1..b5c7f92 100644 --- a/tests/e2e/smoke.spec.ts +++ b/tests/e2e/smoke.spec.ts @@ -11,7 +11,7 @@ import { resolve4 } from 'node:dns/promises'; import { expect, test } from '@playwright/test'; -import { kubectl } from './support/cluster'; +import { kubectl, waitForPodsReady } from './support/cluster'; import { observed, recordJson } from './support/evidence'; import { EXPECTED_NODE_IPS, @@ -24,35 +24,30 @@ import { test.describe('smoke', () => { test('all three cluster pods are Ready', async ({}, testInfo) => { - const result = await observed(testInfo, 'get pods', () => - kubectl([ - '-n', - NAMESPACE, - 'get', - 'pods', - '-l', - 'app.kubernetes.io/name=tidaldb', - '-o', - 'jsonpath={range .items[*]}{.metadata.name}{" "}{.status.containerStatuses[0].ready}{"\\n"}{end}', - ]), - ); + // Polled, not sampled once: `reseed_self_restart: true` means a node that + // latches a reseed marker legitimately exits(0) and comes back. Observed + // 2026-08-30 — tidaldb-2 restarted mid-suite with + // {"reason":"reseed_self_restart","shard":1} and was Ready ~20s later. A + // single sample turns that self-healing behavior into a red suite. + const outcome = await waitForPodsReady(NAMESPACE, POD_NAMES); - expect(result.code, `kubectl failed: ${result.stderr}`).toBe(0); + await recordJson(testInfo, 'pod-readiness', { + converged: outcome.converged, + final: outcome.final, + restarts: outcome.restarts, + samples: outcome.samples, + note: + 'restartCount is evidence, not a failure: a bounded reseed_self_restart is ' + + 'designed behavior. A CLIMBING count across runs is the livelock signature ' + + '(the 2026-08-20 incident reached 196).', + }); - const readyByPod: Record = {}; - for (const line of result.stdout.trim().split('\n')) { - if (line.trim() === '') continue; - const [name, ready] = line.trim().split(/\s+/); - readyByPod[name] = ready === 'true'; - } - - await recordJson(testInfo, 'pod-readiness', readyByPod); - - for (const pod of POD_NAMES) { - expect(readyByPod[pod], `${pod} should be Ready`).toBe(true); - } expect( - Object.keys(readyByPod).sort(), + outcome.converged, + `pods did not all reach Ready within the budget; final=${JSON.stringify(outcome.final)}`, + ).toBe(true); + expect( + Object.keys(outcome.final).sort(), 'expected exactly the known pod set', ).toEqual([...POD_NAMES].sort()); }); diff --git a/tests/e2e/support/cluster.ts b/tests/e2e/support/cluster.ts index 0054a26..42629c0 100644 --- a/tests/e2e/support/cluster.ts +++ b/tests/e2e/support/cluster.ts @@ -229,3 +229,72 @@ export async function withPortForward( await forward.close(); } } + +/** + * Pod readiness, polled until every named pod is Ready or `budgetMs` elapses. + * + * A single sample is the wrong instrument here. This cluster is CONFIGURED to + * self-restart: `replication.reseed_self_restart: true` means a node that latches + * a reseed marker drains, exits(0), and lets the StatefulSet reinstall it on the + * next boot. Observed 2026-08-30 minutes after a deploy — tidaldb-2 exited with + * `{"reason":"reseed_self_restart","shard":1}`, reinstalled, and was Ready again + * ~20s later. Sampling once mid-restart reports a healthy, self-healing cluster as + * broken, and that false red is as corrosive as a false green. + * + * This is deliberately NOT a Playwright retry (the config sets `retries: 0` on + * purpose, so a check that only passes on the second attempt still tells you + * something true). It is a bounded convergence window with the whole timeline + * returned as evidence: converging is a pass, not-converged-in-budget is a fail, + * and the caller can see which pod flapped and when. + */ +export type ReadinessSample = { at: string; ready: Record }; + +export type ReadinessOutcome = { + converged: boolean; + final: Record; + samples: ReadinessSample[]; + restarts: Record; +}; + +export async function waitForPodsReady( + namespace: string, + pods: string[], + budgetMs = 90_000, + intervalMs = 3_000, +): Promise { + const deadline = Date.now() + budgetMs; + const samples: ReadinessSample[] = []; + let final: Record = {}; + let restarts: Record = {}; + + for (;;) { + const result = await kubectl([ + '-n', + namespace, + 'get', + 'pods', + '-l', + 'app.kubernetes.io/name=tidaldb', + '-o', + 'jsonpath={range .items[*]}{.metadata.name}{" "}{.status.containerStatuses[0].ready}{" "}{.status.containerStatuses[0].restartCount}{"\\n"}{end}', + ]); + + final = {}; + restarts = {}; + if (result.code === 0) { + for (const line of result.stdout.trim().split('\n')) { + if (line.trim() === '') continue; + const [name, ready, restartCount] = line.trim().split(/\s+/); + if (!name) continue; + final[name] = ready === 'true'; + restarts[name] = Number.parseInt(restartCount ?? '0', 10) || 0; + } + } + samples.push({ at: new Date().toISOString(), ready: { ...final } }); + + const allReady = pods.length > 0 && pods.every((pod) => final[pod] === true); + if (allReady) return { converged: true, final, samples, restarts }; + if (Date.now() >= deadline) return { converged: false, final, samples, restarts }; + await sleep(intervalMs); + } +}