tidaldb/tests/e2e/features/07-tidalctl.spec.ts

/**
 * Runbook section 7 — the live debugging tool.
 *
 * CAP-012 tidalctl interrogates a live cluster and its exit codes gate
 *
 * The exit-code contract is the part that matters operationally: an operator is
 * expected to write `tidalctl cluster-status && deploy`, which is only safe if a
 * non-converged cluster really exits non-zero and a bad credential does not
 * silently exit 0.
 */

import { expect, test } from '@playwright/test';
import { tidalctl, withPortForward } from '../support/cluster';
import { observed, recordJson } from '../support/evidence';
import { NAMESPACE, PORT_CLIENT, apiKey } from '../support/env';

test.describe('section 7 — tidalctl live interrogation', () => {
  test('cluster-status reports leadership, regions, and shards from a live node', async ({}, testInfo) => {
    await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => {
      const result = await observed(testInfo, 'tidalctl cluster-status', () =>
        tidalctl(
          [
            'cluster-status',
            '--url',
            `https://127.0.0.1:${forward.localPort}`,
            '--key',
            apiKey(),
            // Required: the client port serves the INTERNAL cluster CA, whose
            // leaf is issued for in-cluster DNS names, so a localhost tunnel
            // cannot validate it.
            '--insecure',
          ],
          { timeoutMs: 45_000 },
        ),
      );
      // Exit 2, not 0, on a cluster whose `shards[]` rows are all converged.
      //
      // This is no longer a fabrication. `/cluster/status` used to report two of
      // three healthy peers as `applied_events: 0, lag_events: 13322235` — a
      // 500ms probe timeout rendered as the leader's entire history as a deficit —
      // and tidalctl folded that invented lag into its verdict. Both halves are
      // fixed: the server reports an unknown frontier as `null`, and tidalctl no
      // longer treats an unknown as a deficit.
      //
      // What remains is honest and is the reason the code is still 2: this node
      // cannot REACH those peers (`reachable: false`, and therefore
      // `partitioned: true` — `apply_leader_partition_view` unions the unreachable
      // set into the ship-skip set). A node that cannot see its peers does not
      // know they are converged, and a deploy gate must not pass on that. The
      // remaining defect is the peer HTTP probe failing between healthy pods,
      // which is upstream of tidalctl entirely.
      expect(
        result.code,
        'expected exit 2 — this node cannot reach its peers, so it cannot vouch for ' +
          'them. If this is now 0, the peer HTTP probe between healthy pods was ' +
          'fixed: update runbook section 7 and mark BUG-005 verified. If the regions ' +
          'table shows a NUMERIC lag instead of `?`, that is a real deficit and a ' +
          'different finding.',
      ).toBe(2);

      // The output must still be correct and complete even though the verdict
      // is degraded — an operator reads these tables to locate the problem.
      expect(result.stdout, 'must name the leader').toMatch(/leader:/);
      expect(result.stdout, 'must list regions').toContain('regions:');
      expect(result.stdout, 'must list shards').toContain('shards:');
    });
  });

  test('an unknown peer frontier is reported as NO REPORT, never as fabricated lag', async ({}, testInfo) => {
    await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => {
      const result = await observed(testInfo, 'tidalctl cluster-status regions', () =>
        tidalctl(
          [
            'cluster-status',
            '--url',
            `https://127.0.0.1:${forward.localPort}`,
            '--key',
            apiKey(),
            '--insecure',
          ],
          { timeoutMs: 45_000 },
        ),
      );
      // Degraded verdict for the reason pinned in the previous test (this node
      // cannot reach its peers); what matters here is HOW the gap is reported.
      expect(result.code, result.stderr).toBe(2);

      // `/cluster/status` reports a peer it holds no frontier report for as
      // `null`, which tidalctl renders as `?`. Every unknown MUST be labelled
      // NO REPORT, and — the other direction, which is what stops the marker
      // being re-inferred — NO REPORT must appear ONLY where the wire said
      // unknown. A numeric `applied=0` is now a MEASURED zero (the PVC-wipe
      // shape), and labelling that NO REPORT would hide a genuinely empty
      // replica: the exact inversion of the truth.
      const regionLines = result.stdout
        .split('\n')
        .filter((line) => /applied=/.test(line));
      await recordJson(testInfo, 'region-lines', regionLines);

      expect(regionLines.length, 'expected a line per region').toBeGreaterThan(0);
      for (const line of regionLines) {
        const unknown = /applied=\?|lag=\?/.test(line);
        if (unknown) {
          expect(
            line,
            'a peer whose frontier the wire reported as null must be labelled NO REPORT',
          ).toContain('NO REPORT');
        } else {
          expect(
            line,
            'NO REPORT must be driven off the wire `null`, never inferred from a ' +
              'measured number — a replica at a real applied=0 is BEHIND, not unreported',
          ).not.toContain('NO REPORT');
        }
      }
    });
  });

  test('watch emits one line per tick and terminates on the requested count', async ({}, testInfo) => {
    await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => {
      const result = await observed(testInfo, 'tidalctl watch', () =>
        tidalctl(
          [
            'watch',
            '--url',
            `https://127.0.0.1:${forward.localPort}`,
            '--key',
            apiKey(),
            '--insecure',
            '--interval',
            '2',
            '--count',
            '3',
          ],
          { timeoutMs: 60_000 },
        ),
      );

      const ticks = result.stdout.split('\n').filter((line) => line.includes('leader='));
      await recordJson(testInfo, 'watch-ticks', ticks);

      // A bounded --count must terminate on its own. An unbounded watch here
      // would hang the suite, which is why the flag exists.
      expect(ticks.length, 'watch must emit exactly the requested number of ticks').toBe(3);
      for (const tick of ticks) {
        expect(tick, 'each tick must carry a verdict').toMatch(/\[(ok|DEGRADED)\]/);
      }
    });
  });

  test('exit codes gate correctly for a bad credential and a malformed url', async ({}, testInfo) => {
    await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => {
      const badCredential = await observed(testInfo, 'tidalctl with wrong key', () =>
        tidalctl(
          [
            'cluster-status',
            '--url',
            `https://127.0.0.1:${forward.localPort}`,
            '--key',
            'definitely-not-the-key',
            '--insecure',
          ],
          { timeoutMs: 45_000 },
        ),
      );

      // Exit 2, not 0. A tool that exits 0 on a rejected credential would make
      // `tidalctl cluster-status && deploy` deploy against an unverified cluster.
      expect(
        badCredential.code,
        'a rejected credential must exit 2, never 0',
      ).toBe(2);
    });

    const malformedUrl = await observed(testInfo, 'tidalctl with schemeless url', () =>
      tidalctl(
        ['cluster-status', '--url', '127.0.0.1:9500', '--key', apiKey(), '--insecure'],
        { timeoutMs: 20_000 },
      ),
    );
    // Exit 1 = usage error, distinct from exit 2 = reachable but unhealthy.
    expect(malformedUrl.code, 'a malformed --url must exit 1 (usage)').toBe(1);
  });
});