/** * Runbook section 7 — the live debugging tool. * * CAP-012 tidalctl interrogates a live cluster and its exit codes gate * * The exit-code contract is the part that matters operationally: an operator is * expected to write `tidalctl cluster-status && deploy`, which is only safe if a * non-converged cluster really exits non-zero and a bad credential does not * silently exit 0. */ import { expect, test } from '@playwright/test'; import { tidalctl, withPortForward } from '../support/cluster'; import { observed, recordJson } from '../support/evidence'; import { NAMESPACE, PORT_CLIENT, apiKey } from '../support/env'; test.describe('section 7 — tidalctl live interrogation', () => { test('cluster-status reports leadership, regions, and shards from a live node', async ({}, testInfo) => { await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => { const result = await observed(testInfo, 'tidalctl cluster-status', () => tidalctl( [ 'cluster-status', '--url', `https://127.0.0.1:${forward.localPort}`, '--key', apiKey(), // Required: the client port serves the INTERNAL cluster CA, whose // leaf is issued for in-cluster DNS names, so a localhost tunnel // cannot validate it. '--insecure', ], { timeoutMs: 45_000 }, ), ); // Exit 2, not 0, on a FULLY CONVERGED cluster. This is not a tidalctl // bug in isolation — the aggregated /cluster/status endpoint reports two // of three healthy peers as `region: null, applied_events: 0, // lag_events: 13322235, reachable: false`, while each of those peers' // own /cluster/status/local reports lag=0 and all three agree on the // leader (proven in 01-cluster-convergence.spec.ts). tidalctl correctly // labels the gap `NO REPORT` but still folds it into its degraded // verdict, so the exit code is 2. // // Consequence: `tidalctl cluster-status && deploy` can NEVER pass on this // deployment. The runbook claimed it was a safe gate; that claim was // written from an exit code masked by a shell pipeline. See BUG-005. expect( result.code, 'expected exit 2 — the aggregated-status gap makes a converged cluster report ' + 'degraded. If this is now 0, the engine-side peer reporting was fixed: ' + 'update runbook section 7 and mark BUG-005 verified.', ).toBe(2); // The output must still be correct and complete even though the verdict // is degraded — an operator reads these tables to locate the problem. expect(result.stdout, 'must name the leader').toMatch(/leader:/); expect(result.stdout, 'must list regions').toContain('regions:'); expect(result.stdout, 'must list shards').toContain('shards:'); }); }); test('the aggregated-status gap is reported as NO REPORT, never as fabricated lag', async ({}, testInfo) => { await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => { const result = await observed(testInfo, 'tidalctl cluster-status regions', () => tidalctl( [ 'cluster-status', '--url', `https://127.0.0.1:${forward.localPort}`, '--key', apiKey(), '--insecure', ], { timeoutMs: 45_000 }, ), ); // Degraded verdict for the reason pinned in the previous test (BUG-005); // what matters here is HOW the gap is reported, not the exit code. expect(result.code, result.stderr).toBe(2); // The aggregated endpoint reports a peer it holds no frontier report for // as applied=0 and derives lag against that zero, so a converged peer can // read as the leader's entire history behind. tidalctl must name that // condition rather than repeat it as lag. const regionLines = result.stdout .split('\n') .filter((line) => /applied=/.test(line)); await recordJson(testInfo, 'region-lines', regionLines); expect(regionLines.length, 'expected a line per region').toBeGreaterThan(0); for (const line of regionLines) { if (/applied=0\b/.test(line)) { expect( line, 'a peer with no frontier report must be labelled NO REPORT, not shown as real lag', ).toContain('NO REPORT'); } } }); }); test('watch emits one line per tick and terminates on the requested count', async ({}, testInfo) => { await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => { const result = await observed(testInfo, 'tidalctl watch', () => tidalctl( [ 'watch', '--url', `https://127.0.0.1:${forward.localPort}`, '--key', apiKey(), '--insecure', '--interval', '2', '--count', '3', ], { timeoutMs: 60_000 }, ), ); const ticks = result.stdout.split('\n').filter((line) => line.includes('leader=')); await recordJson(testInfo, 'watch-ticks', ticks); // A bounded --count must terminate on its own. An unbounded watch here // would hang the suite, which is why the flag exists. expect(ticks.length, 'watch must emit exactly the requested number of ticks').toBe(3); for (const tick of ticks) { expect(tick, 'each tick must carry a verdict').toMatch(/\[(ok|DEGRADED)\]/); } }); }); test('exit codes gate correctly for a bad credential and a malformed url', async ({}, testInfo) => { await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => { const badCredential = await observed(testInfo, 'tidalctl with wrong key', () => tidalctl( [ 'cluster-status', '--url', `https://127.0.0.1:${forward.localPort}`, '--key', 'definitely-not-the-key', '--insecure', ], { timeoutMs: 45_000 }, ), ); // Exit 2, not 0. A tool that exits 0 on a rejected credential would make // `tidalctl cluster-status && deploy` deploy against an unverified cluster. expect( badCredential.code, 'a rejected credential must exit 2, never 0', ).toBe(2); }); const malformedUrl = await observed(testInfo, 'tidalctl with schemeless url', () => tidalctl( ['cluster-status', '--url', '127.0.0.1:9500', '--key', apiKey(), '--insecure'], { timeoutMs: 20_000 }, ), ); // Exit 1 = usage error, distinct from exit 2 = reachable but unhealthy. expect(malformedUrl.code, 'a malformed --url must exit 1 (usage)').toBe(1); }); });