/** * Runbook section 9 — operator authority, and what is still inert. * * CAP-014 operator authority is separated from data-plane access * CAP-015 inert observability features are inert for a known reason * * This spec exists because the runbook was WRONG about its own subject. It * claimed the credential split was "not active yet — requires an image roll". * The harness read the live image and the live secret and found the gate * already enforcing. Documentation drift, caught by the thing it documents. * See BUG-001. * * The inert-feature tests are tripwires in the honest direction: they assert * ABSENCE today, so the day someone rolls the observability image they fail and * say "update the runbook" — instead of the runbook rotting again. */ import { expect, test } from '@playwright/test'; import { kubectl, withPortForward } from '../support/cluster'; import { observed, recordJson } from '../support/evidence'; import { NAMESPACE, OBS_NAMESPACE, PORT_CLIENT, PORT_METRICS, POD_NAMES, adminKey, apiKey, } from '../support/env'; /** Generous: a short scrape timeout returns zero lines for everything (BUG-002). */ const SCRAPE_TIMEOUT_SECONDS = 15; test.describe('section 9 — operator authority', () => { test('the data-plane credential is refused on a destructive operator verb', async ({ playwright, }, testInfo) => { const admin = adminKey(); expect( admin, 'TIDAL_ADMIN_KEY is absent from the secret. Without it the gate degrades to previous ' + 'behaviour by design and any client key can remove a member — add the key before ' + 'treating this deployment as verified.', ).toBeTruthy(); await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => { const base = `https://127.0.0.1:${forward.localPort}`; const body = { region: 'tidaldb-1' }; const dataContext = await playwright.request.newContext({ ignoreHTTPSErrors: true, extraHTTPHeaders: { authorization: `Bearer ${apiKey()}`, 'content-type': 'application/json', }, }); const adminContext = await playwright.request.newContext({ ignoreHTTPSErrors: true, extraHTTPHeaders: { authorization: `Bearer ${admin}`, 'content-type': 'application/json', }, }); try { const dataAttempt = await dataContext.post(`${base}/cluster/promote`, { data: body }); const adminAttempt = await adminContext.post(`${base}/cluster/promote`, { data: body }); await recordJson(testInfo, 'authority-split', { verb: 'POST /cluster/promote', dataCredential: dataAttempt.status(), adminCredential: adminAttempt.status(), }); // 403, not 401: the data bearer IS a valid credential, it simply lacks // operator authority. A 401 here would mean the admin gate rejected it // before authenticating, which would also break peer-callable verbs. expect( dataAttempt.status(), 'the data bearer must be authenticated but NOT authorized (403) on an operator verb', ).toBe(403); // The admin key must clear both gates. It is deliberately a superset // credential — one Authorization header per request means it has to // authenticate as well as authorize, or operators get 401 before the // admin gate ever runs. expect( adminAttempt.status(), 'the admin key must clear both authentication and the admin gate', ).not.toBe(401); expect(adminAttempt.status(), 'the admin key must not be forbidden').not.toBe(403); } finally { await dataContext.dispose(); await adminContext.dispose(); } }); }); test('cluster status requires a credential', async ({ playwright }, testInfo) => { await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => { const anonymous = await playwright.request.newContext({ ignoreHTTPSErrors: true }); try { const response = await anonymous.get( `https://127.0.0.1:${forward.localPort}/cluster/status`, ); await recordJson(testInfo, 'anonymous-cluster-status', { status: response.status() }); // Moved behind auth by 388e445. Unauthenticated access would leak // leader identity, membership, and sequence positions. expect( response.status(), '/cluster/status must require a credential even inside the cluster', ).toBe(401); } finally { await anonymous.dispose(); } }); }); test('the admin key is mounted as a projected secret file', async ({}, testInfo) => { const result = await observed(testInfo, 'list admin-key mount', () => kubectl( ['-n', NAMESPACE, 'exec', 'tidaldb-0', '-c', 'tidaldb', '--', 'ls', '/etc/tidaldb/admin-key/'], { timeoutMs: 45_000 }, ), ); await recordJson(testInfo, 'admin-key-mount', { exitCode: result.code, entries: result.stdout.trim().split('\n').filter(Boolean), }); // The mount is optional:true on purpose — a required mount would prevent // the pod from starting at all when the key is absent. Its presence here // is what let the credential poller hot-load the key without a restart, // which is why the boot log's "not set" WARN is stale (BUG-003). expect(result.code, `admin-key mount unreadable: ${result.stderr}`).toBe(0); expect( result.stdout, 'the projected admin-key file must be present for the poller to load', ).toContain('admin-key'); }); test('HTTP request metrics are exported, and the scrape that proves it actually worked', async ({}, testInfo) => { const counts: Record = {}; for (const pod of POD_NAMES) { const ip = await kubectl([ '-n', NAMESPACE, 'get', 'pod', pod, '-o', 'jsonpath={.status.podIP}', ]); expect(ip.code, ip.stderr).toBe(0); const result = await observed(testInfo, `scrape ${pod}`, () => kubectl( [ '-n', OBS_NAMESPACE, 'exec', 'deploy/vmagent', '--', 'sh', '-c', `wget -qO- --timeout=${SCRAPE_TIMEOUT_SECONDS} http://${ip.stdout.trim()}:${PORT_METRICS}/metrics ` + `| awk '/^tidaldb_http_/{h++} /^tidaldb_/{t++} END{print (t+0)" "(h+0)}'`, ], { timeoutMs: 60_000 }, ), ); expect(result.code, `scrape of ${pod} failed: ${result.stderr}`).toBe(0); const [baseline, http] = result.stdout.trim().split(/\s+/).map(Number); counts[pod] = { baseline, http }; } await recordJson(testInfo, 'metric-family-counts', counts); for (const pod of POD_NAMES) { // Prove the scrape WORKED before concluding a metric is missing. A short // timeout returns zero for everything, which would make the assertion // below pass for entirely the wrong reason — the exact trap that made one // healthy pod look like it had stopped exporting (BUG-002). expect( counts[pod].baseline, `${pod} returned no metrics at all — the scrape failed, so its http-metric count ` + `proves nothing`, ).toBeGreaterThan(100); // Inverted 2026-08-30. This previously asserted `http === 0` and was correct // for the image running when it was written; it failed the moment a newer // image was rolled, which is exactly how the drift got noticed. Runbook // section 9.1 is now LIVE, so a regression to zero HTTP metrics — an image // rollback, or the route-metrics layer being dropped — fails here. expect( counts[pod].http, `${pod} exports no tidaldb_http_* metrics. Section 9.1 went live on ` + `2026-08-30; losing them means an image rollback or the route-metrics layer ` + `being removed, and the five HTTP dashboard panels are now blank.`, ).toBeGreaterThan(0); } }); test('structured logging is not yet enabled on the StatefulSet', async ({}, testInfo) => { const result = await observed(testInfo, 'statefulset env', () => kubectl( [ '-n', NAMESPACE, 'get', 'statefulset', 'tidaldb', '-o', 'jsonpath={range .spec.template.spec.containers[0].env[*]}{.name}={.value}{"\\n"}{end}', ], { timeoutMs: 45_000 }, ), ); expect(result.code, result.stderr).toBe(0); const env = result.stdout .trim() .split('\n') .filter((line) => line.trim() !== ''); await recordJson(testInfo, 'statefulset-env', env); const jsonLogs = env.find((line) => line.startsWith('JSON_LOGS=')); expect( jsonLogs, 'JSON_LOGS is now set — roll runbook section 9.3 from pending to live, verify one JSON ' + 'object per line, and invert this assertion', ).toBeUndefined(); }); });