/** * Runbook section 7 — the live debugging tool. * * CAP-012 tidalctl interrogates a live cluster and its exit codes gate * * The exit-code contract is the part that matters operationally: an operator is * expected to write `tidalctl cluster-status && deploy`, which is only safe if a * non-converged cluster really exits non-zero and a bad credential does not * silently exit 0. */ import { expect, test } from '@playwright/test'; import { tidalctl, withPortForward } from '../support/cluster'; import { observed, recordJson } from '../support/evidence'; import { NAMESPACE, PORT_CLIENT, apiKey } from '../support/env'; test.describe('section 7 — tidalctl live interrogation', () => { test('cluster-status reports leadership, regions, and shards from a live node', async ({}, testInfo) => { await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => { const result = await observed(testInfo, 'tidalctl cluster-status', () => tidalctl( [ 'cluster-status', '--url', `https://127.0.0.1:${forward.localPort}`, '--key', apiKey(), // Required: the client port serves the INTERNAL cluster CA, whose // leaf is issued for in-cluster DNS names, so a localhost tunnel // cannot validate it. '--insecure', ], { timeoutMs: 45_000 }, ), ); // Exit 2, not 0, on a cluster whose `shards[]` rows are all converged. // // This is no longer a fabrication. `/cluster/status` used to report two of // three healthy peers as `applied_events: 0, lag_events: 13322235` — a // 500ms probe timeout rendered as the leader's entire history as a deficit — // and tidalctl folded that invented lag into its verdict. Both halves are // fixed: the server reports an unknown frontier as `null`, and tidalctl no // longer treats an unknown as a deficit. // // What remains is honest and is the reason the code is still 2: this node // cannot REACH those peers (`reachable: false`, and therefore // `partitioned: true` — `apply_leader_partition_view` unions the unreachable // set into the ship-skip set). A node that cannot see its peers does not // know they are converged, and a deploy gate must not pass on that. The // remaining defect is the peer HTTP probe failing between healthy pods, // which is upstream of tidalctl entirely. expect( result.code, 'expected exit 2 — this node cannot reach its peers, so it cannot vouch for ' + 'them. If this is now 0, the peer HTTP probe between healthy pods was ' + 'fixed: update runbook section 7 and mark BUG-005 verified. If the regions ' + 'table shows a NUMERIC lag instead of `?`, that is a real deficit and a ' + 'different finding.', ).toBe(2); // The output must still be correct and complete even though the verdict // is degraded — an operator reads these tables to locate the problem. expect(result.stdout, 'must name the leader').toMatch(/leader:/); expect(result.stdout, 'must list regions').toContain('regions:'); expect(result.stdout, 'must list shards').toContain('shards:'); }); }); test('an unknown peer frontier is reported as NO REPORT, never as fabricated lag', async ({}, testInfo) => { await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => { const result = await observed(testInfo, 'tidalctl cluster-status regions', () => tidalctl( [ 'cluster-status', '--url', `https://127.0.0.1:${forward.localPort}`, '--key', apiKey(), '--insecure', ], { timeoutMs: 45_000 }, ), ); // Degraded verdict for the reason pinned in the previous test (this node // cannot reach its peers); what matters here is HOW the gap is reported. expect(result.code, result.stderr).toBe(2); // `/cluster/status` reports a peer it holds no frontier report for as // `null`, which tidalctl renders as `?`. Every unknown MUST be labelled // NO REPORT, and — the other direction, which is what stops the marker // being re-inferred — NO REPORT must appear ONLY where the wire said // unknown. A numeric `applied=0` is now a MEASURED zero (the PVC-wipe // shape), and labelling that NO REPORT would hide a genuinely empty // replica: the exact inversion of the truth. const regionLines = result.stdout .split('\n') .filter((line) => /applied=/.test(line)); await recordJson(testInfo, 'region-lines', regionLines); expect(regionLines.length, 'expected a line per region').toBeGreaterThan(0); for (const line of regionLines) { const unknown = /applied=\?|lag=\?/.test(line); if (unknown) { expect( line, 'a peer whose frontier the wire reported as null must be labelled NO REPORT', ).toContain('NO REPORT'); } else { expect( line, 'NO REPORT must be driven off the wire `null`, never inferred from a ' + 'measured number — a replica at a real applied=0 is BEHIND, not unreported', ).not.toContain('NO REPORT'); } } }); }); test('watch emits one line per tick and terminates on the requested count', async ({}, testInfo) => { await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => { const result = await observed(testInfo, 'tidalctl watch', () => tidalctl( [ 'watch', '--url', `https://127.0.0.1:${forward.localPort}`, '--key', apiKey(), '--insecure', '--interval', '2', '--count', '3', ], { timeoutMs: 60_000 }, ), ); const ticks = result.stdout.split('\n').filter((line) => line.includes('leader=')); await recordJson(testInfo, 'watch-ticks', ticks); // A bounded --count must terminate on its own. An unbounded watch here // would hang the suite, which is why the flag exists. expect(ticks.length, 'watch must emit exactly the requested number of ticks').toBe(3); for (const tick of ticks) { expect(tick, 'each tick must carry a verdict').toMatch(/\[(ok|DEGRADED)\]/); } }); }); test('exit codes gate correctly for a bad credential and a malformed url', async ({}, testInfo) => { await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => { const badCredential = await observed(testInfo, 'tidalctl with wrong key', () => tidalctl( [ 'cluster-status', '--url', `https://127.0.0.1:${forward.localPort}`, '--key', 'definitely-not-the-key', '--insecure', ], { timeoutMs: 45_000 }, ), ); // Exit 2, not 0. A tool that exits 0 on a rejected credential would make // `tidalctl cluster-status && deploy` deploy against an unverified cluster. expect( badCredential.code, 'a rejected credential must exit 2, never 0', ).toBe(2); }); const malformedUrl = await observed(testInfo, 'tidalctl with schemeless url', () => tidalctl( ['cluster-status', '--url', '127.0.0.1:9500', '--key', apiKey(), '--insecure'], { timeoutMs: 20_000 }, ), ); // Exit 1 = usage error, distinct from exit 2 = reachable but unhealthy. expect(malformedUrl.code, 'a malformed --url must exit 1 (usage)').toBe(1); }); });