tidaldb/tests/e2e/features/09-operator-authority.spec.ts

/**
 * Runbook section 9 — operator authority, and what is still inert.
 *
 * CAP-014 operator authority is separated from data-plane access
 * CAP-015 inert observability features are inert for a known reason
 *
 * This spec exists because the runbook was WRONG about its own subject. It
 * claimed the credential split was "not active yet — requires an image roll".
 * The harness read the live image and the live secret and found the gate
 * already enforcing. Documentation drift, caught by the thing it documents.
 * See BUG-001.
 *
 * The inert-feature tests are tripwires in the honest direction: they assert
 * ABSENCE today, so the day someone rolls the observability image they fail and
 * say "update the runbook" — instead of the runbook rotting again.
 */

import { expect, test } from '@playwright/test';
import { kubectl, withPortForward } from '../support/cluster';
import { observed, recordJson } from '../support/evidence';
import {
  NAMESPACE,
  OBS_NAMESPACE,
  PORT_CLIENT,
  PORT_METRICS,
  POD_NAMES,
  adminKey,
  apiKey,
} from '../support/env';

/** Generous: a short scrape timeout returns zero lines for everything (BUG-002). */
const SCRAPE_TIMEOUT_SECONDS = 15;

test.describe('section 9 — operator authority', () => {
  test('the data-plane credential is refused on a destructive operator verb', async ({
    playwright,
  }, testInfo) => {
    const admin = adminKey();
    expect(
      admin,
      'TIDAL_ADMIN_KEY is absent from the secret. Without it the gate degrades to previous ' +
        'behaviour by design and any client key can remove a member — add the key before ' +
        'treating this deployment as verified.',
    ).toBeTruthy();

    await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => {
      const base = `https://127.0.0.1:${forward.localPort}`;
      const body = { region: 'tidaldb-1' };

      const dataContext = await playwright.request.newContext({
        ignoreHTTPSErrors: true,
        extraHTTPHeaders: {
          authorization: `Bearer ${apiKey()}`,
          'content-type': 'application/json',
        },
      });
      const adminContext = await playwright.request.newContext({
        ignoreHTTPSErrors: true,
        extraHTTPHeaders: {
          authorization: `Bearer ${admin}`,
          'content-type': 'application/json',
        },
      });

      try {
        const dataAttempt = await dataContext.post(`${base}/cluster/promote`, { data: body });
        const adminAttempt = await adminContext.post(`${base}/cluster/promote`, { data: body });

        await recordJson(testInfo, 'authority-split', {
          verb: 'POST /cluster/promote',
          dataCredential: dataAttempt.status(),
          adminCredential: adminAttempt.status(),
        });

        // 403, not 401: the data bearer IS a valid credential, it simply lacks
        // operator authority. A 401 here would mean the admin gate rejected it
        // before authenticating, which would also break peer-callable verbs.
        expect(
          dataAttempt.status(),
          'the data bearer must be authenticated but NOT authorized (403) on an operator verb',
        ).toBe(403);

        // The admin key must clear both gates. It is deliberately a superset
        // credential — one Authorization header per request means it has to
        // authenticate as well as authorize, or operators get 401 before the
        // admin gate ever runs.
        expect(
          adminAttempt.status(),
          'the admin key must clear both authentication and the admin gate',
        ).not.toBe(401);
        expect(adminAttempt.status(), 'the admin key must not be forbidden').not.toBe(403);
      } finally {
        await dataContext.dispose();
        await adminContext.dispose();
      }
    });
  });

  test('cluster status requires a credential', async ({ playwright }, testInfo) => {
    await withPortForward(NAMESPACE, 'svc/tidaldb', PORT_CLIENT, async (forward) => {
      const anonymous = await playwright.request.newContext({ ignoreHTTPSErrors: true });
      try {
        const response = await anonymous.get(
          `https://127.0.0.1:${forward.localPort}/cluster/status`,
        );
        await recordJson(testInfo, 'anonymous-cluster-status', { status: response.status() });

        // Moved behind auth by 388e445. Unauthenticated access would leak
        // leader identity, membership, and sequence positions.
        expect(
          response.status(),
          '/cluster/status must require a credential even inside the cluster',
        ).toBe(401);
      } finally {
        await anonymous.dispose();
      }
    });
  });

  test('the admin key is mounted as a projected secret file', async ({}, testInfo) => {
    const result = await observed(testInfo, 'list admin-key mount', () =>
      kubectl(
        ['-n', NAMESPACE, 'exec', 'tidaldb-0', '-c', 'tidaldb', '--', 'ls', '/etc/tidaldb/admin-key/'],
        { timeoutMs: 45_000 },
      ),
    );

    await recordJson(testInfo, 'admin-key-mount', {
      exitCode: result.code,
      entries: result.stdout.trim().split('\n').filter(Boolean),
    });

    // The mount is optional:true on purpose — a required mount would prevent
    // the pod from starting at all when the key is absent. Its presence here
    // is what let the credential poller hot-load the key without a restart,
    // which is why the boot log's "not set" WARN is stale (BUG-003).
    expect(result.code, `admin-key mount unreadable: ${result.stderr}`).toBe(0);
    expect(
      result.stdout,
      'the projected admin-key file must be present for the poller to load',
    ).toContain('admin-key');
  });

  test('HTTP request metrics are exported, and the scrape that proves it actually worked', async ({}, testInfo) => {
    const counts: Record<string, { baseline: number; http: number }> = {};

    for (const pod of POD_NAMES) {
      const ip = await kubectl([
        '-n',
        NAMESPACE,
        'get',
        'pod',
        pod,
        '-o',
        'jsonpath={.status.podIP}',
      ]);
      expect(ip.code, ip.stderr).toBe(0);

      const result = await observed(testInfo, `scrape ${pod}`, () =>
        kubectl(
          [
            '-n',
            OBS_NAMESPACE,
            'exec',
            'deploy/vmagent',
            '--',
            'sh',
            '-c',
            `wget -qO- --timeout=${SCRAPE_TIMEOUT_SECONDS} http://${ip.stdout.trim()}:${PORT_METRICS}/metrics ` +
              `| awk '/^tidaldb_http_/{h++} /^tidaldb_/{t++} END{print (t+0)" "(h+0)}'`,
          ],
          { timeoutMs: 60_000 },
        ),
      );
      expect(result.code, `scrape of ${pod} failed: ${result.stderr}`).toBe(0);

      const [baseline, http] = result.stdout.trim().split(/\s+/).map(Number);
      counts[pod] = { baseline, http };
    }

    await recordJson(testInfo, 'metric-family-counts', counts);

    for (const pod of POD_NAMES) {
      // Prove the scrape WORKED before concluding a metric is missing. A short
      // timeout returns zero for everything, which would make the assertion
      // below pass for entirely the wrong reason — the exact trap that made one
      // healthy pod look like it had stopped exporting (BUG-002).
      expect(
        counts[pod].baseline,
        `${pod} returned no metrics at all — the scrape failed, so its http-metric count ` +
          `proves nothing`,
      ).toBeGreaterThan(100);

      // Inverted 2026-08-30. This previously asserted `http === 0` and was correct
      // for the image running when it was written; it failed the moment a newer
      // image was rolled, which is exactly how the drift got noticed. Runbook
      // section 9.1 is now LIVE, so a regression to zero HTTP metrics — an image
      // rollback, or the route-metrics layer being dropped — fails here.
      expect(
        counts[pod].http,
        `${pod} exports no tidaldb_http_* metrics. Section 9.1 went live on ` +
          `2026-08-30; losing them means an image rollback or the route-metrics layer ` +
          `being removed, and the five HTTP dashboard panels are now blank.`,
      ).toBeGreaterThan(0);
    }
  });

  test('structured logging is enabled on the StatefulSet, with the service name pinned', async ({}, testInfo) => {
    const result = await observed(testInfo, 'statefulset env', () =>
      kubectl(
        [
          '-n',
          NAMESPACE,
          'get',
          'statefulset',
          'tidaldb',
          '-o',
          'jsonpath={range .spec.template.spec.containers[0].env[*]}{.name}={.value}{"\\n"}{end}',
        ],
        { timeoutMs: 45_000 },
      ),
    );
    expect(result.code, result.stderr).toBe(0);

    const env = result.stdout
      .trim()
      .split('\n')
      .filter((line) => line.trim() !== '');
    await recordJson(testInfo, 'statefulset-env', env);

    // FLIPPED 2026-08-31 on the roll of m12-harden-20260831, exactly as the previous
    // assertion instructed. The feature was never missing — `logging.rs:85` has
    // implemented it all along; the StatefulSet simply never asked for it, which is
    // why runbook 9.3 read as a gap for months.
    const jsonLogs = env.find((line) => line.startsWith('JSON_LOGS='));
    expect(
      jsonLogs,
      'JSON_LOGS has been removed from the StatefulSet. Structured logs are what make ' +
        'VictoriaLogs `level:` selectors match at all; without it every log-based alert ' +
        'and dashboard silently returns nothing.',
    ).toBe('JSON_LOGS=1');

    // TIDAL_SERVICE_NAME is load-bearing, not cosmetic: enabling JSON_LOGS makes the
    // app's own `service` field win in the fleet's Vector normalize transform
    // (victoria-logs.yaml:230), which would silently rename the log stream from
    // `tidaldb` to `tidal-server` and blind every query keyed on it. The fleet's
    // `_stream_fields` contract pins field NAMES but no legal VALUES, so nothing in
    // k3s-fleet would have caught the flip — `verify-live` now asserts it too.
    const serviceName = env.find((line) => line.startsWith('TIDAL_SERVICE_NAME='));
    expect(
      serviceName,
      'TIDAL_SERVICE_NAME is unset while JSON_LOGS is on — the log stream will silently ' +
        'rename itself to tidal-server and every `service:tidaldb` query will return zero.',
    ).toBe('TIDAL_SERVICE_NAME=tidaldb');
  });
});