tidaldb/tests/e2e/features/07-tidalctl.spec.ts

/**
 * Runbook section 7 — the live debugging tool.
 *
 * CAP-012 tidalctl interrogates a live cluster and its exit codes gate
 *
 * The exit-code contract is the part that matters operationally: an operator is
 * expected to write `tidalctl cluster-status && deploy`, which is only safe if a
 * non-converged cluster really exits non-zero and a bad credential does not
 * silently exit 0.
 */

import { expect, test } from '@playwright/test';
import { tidalctl, withPortForward } from '../support/cluster';
import { observed, recordJson } from '../support/evidence';
import { NAMESPACE, PORT_CLIENT, apiKey } from '../support/env';

test.describe('section 7 — tidalctl live interrogation', () => {
  test('cluster-status reports leadership, regions, and shards from a live node', async ({}, testInfo) => {
    await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => {
      const result = await observed(testInfo, 'tidalctl cluster-status', () =>
        tidalctl(
          [
            'cluster-status',
            '--url',
            `https://127.0.0.1:${forward.localPort}`,
            '--key',
            apiKey(),
            // Required: the client port serves the INTERNAL cluster CA, whose
            // leaf is issued for in-cluster DNS names, so a localhost tunnel
            // cannot validate it.
            '--insecure',
          ],
          { timeoutMs: 45_000 },
        ),
      );
      // Exit 2, not 0, on a FULLY CONVERGED cluster. This is not a tidalctl
      // bug in isolation — the aggregated /cluster/status endpoint reports two
      // of three healthy peers as `region: null, applied_events: 0,
      // lag_events: 13322235, reachable: false`, while each of those peers'
      // own /cluster/status/local reports lag=0 and all three agree on the
      // leader (proven in 01-cluster-convergence.spec.ts). tidalctl correctly
      // labels the gap `NO REPORT` but still folds it into its degraded
      // verdict, so the exit code is 2.
      //
      // Consequence: `tidalctl cluster-status && deploy` can NEVER pass on this
      // deployment. The runbook claimed it was a safe gate; that claim was
      // written from an exit code masked by a shell pipeline. See BUG-005.
      expect(
        result.code,
        'expected exit 2 — the aggregated-status gap makes a converged cluster report ' +
          'degraded. If this is now 0, the engine-side peer reporting was fixed: ' +
          'update runbook section 7 and mark BUG-005 verified.',
      ).toBe(2);

      // The output must still be correct and complete even though the verdict
      // is degraded — an operator reads these tables to locate the problem.
      expect(result.stdout, 'must name the leader').toMatch(/leader:/);
      expect(result.stdout, 'must list regions').toContain('regions:');
      expect(result.stdout, 'must list shards').toContain('shards:');
    });
  });

  test('the aggregated-status gap is reported as NO REPORT, never as fabricated lag', async ({}, testInfo) => {
    await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => {
      const result = await observed(testInfo, 'tidalctl cluster-status regions', () =>
        tidalctl(
          [
            'cluster-status',
            '--url',
            `https://127.0.0.1:${forward.localPort}`,
            '--key',
            apiKey(),
            '--insecure',
          ],
          { timeoutMs: 45_000 },
        ),
      );
      // Degraded verdict for the reason pinned in the previous test (BUG-005);
      // what matters here is HOW the gap is reported, not the exit code.
      expect(result.code, result.stderr).toBe(2);

      // The aggregated endpoint reports a peer it holds no frontier report for
      // as applied=0 and derives lag against that zero, so a converged peer can
      // read as the leader's entire history behind. tidalctl must name that
      // condition rather than repeat it as lag.
      const regionLines = result.stdout
        .split('\n')
        .filter((line) => /applied=/.test(line));
      await recordJson(testInfo, 'region-lines', regionLines);

      expect(regionLines.length, 'expected a line per region').toBeGreaterThan(0);
      for (const line of regionLines) {
        if (/applied=0\b/.test(line)) {
          expect(
            line,
            'a peer with no frontier report must be labelled NO REPORT, not shown as real lag',
          ).toContain('NO REPORT');
        }
      }
    });
  });

  test('watch emits one line per tick and terminates on the requested count', async ({}, testInfo) => {
    await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => {
      const result = await observed(testInfo, 'tidalctl watch', () =>
        tidalctl(
          [
            'watch',
            '--url',
            `https://127.0.0.1:${forward.localPort}`,
            '--key',
            apiKey(),
            '--insecure',
            '--interval',
            '2',
            '--count',
            '3',
          ],
          { timeoutMs: 60_000 },
        ),
      );

      const ticks = result.stdout.split('\n').filter((line) => line.includes('leader='));
      await recordJson(testInfo, 'watch-ticks', ticks);

      // A bounded --count must terminate on its own. An unbounded watch here
      // would hang the suite, which is why the flag exists.
      expect(ticks.length, 'watch must emit exactly the requested number of ticks').toBe(3);
      for (const tick of ticks) {
        expect(tick, 'each tick must carry a verdict').toMatch(/\[(ok|DEGRADED)\]/);
      }
    });
  });

  test('exit codes gate correctly for a bad credential and a malformed url', async ({}, testInfo) => {
    await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => {
      const badCredential = await observed(testInfo, 'tidalctl with wrong key', () =>
        tidalctl(
          [
            'cluster-status',
            '--url',
            `https://127.0.0.1:${forward.localPort}`,
            '--key',
            'definitely-not-the-key',
            '--insecure',
          ],
          { timeoutMs: 45_000 },
        ),
      );

      // Exit 2, not 0. A tool that exits 0 on a rejected credential would make
      // `tidalctl cluster-status && deploy` deploy against an unverified cluster.
      expect(
        badCredential.code,
        'a rejected credential must exit 2, never 0',
      ).toBe(2);
    });

    const malformedUrl = await observed(testInfo, 'tidalctl with schemeless url', () =>
      tidalctl(
        ['cluster-status', '--url', '127.0.0.1:9500', '--key', apiKey(), '--insecure'],
        { timeoutMs: 20_000 },
      ),
    );
    // Exit 1 = usage error, distinct from exit 2 = reachable but unhealthy.
    expect(malformedUrl.code, 'a malformed --url must exit 1 (usage)').toBe(1);
  });
});