tidaldb/tests/e2e/features/04-network-isolation.spec.ts

/**
 * Runbook section 4 — NetworkPolicy enforcement.
 *
 * CAP-008 the metrics port is not reachable from arbitrary pods
 * CAP-009 metrics reach the platform with the labels the dashboard queries
 *
 * A NetworkPolicy that blocks everything is an observability outage and one
 * that blocks nothing is theatre, so both directions are proven in the same
 * test: the foreign pod is refused AND the scraper still collects.
 */

import { expect, test } from '@playwright/test';
import { kubectl, withPortForward } from '../support/cluster';
import { observed, recordJson } from '../support/evidence';
import {
  FOREIGN_NAMESPACE,
  FOREIGN_POD,
  NAMESPACE,
  OBS_NAMESPACE,
  PORT_METRICS,
} from '../support/env';

/**
 * Generous, because a 6-second client timeout truncates this scrape on a loaded
 * node and returns zero lines — indistinguishable from "this pod exports
 * nothing". That false negative briefly looked like a pod had stopped
 * exporting entirely (BUG-002).
 */
const SCRAPE_TIMEOUT_SECONDS = 15;

/** Minimum series a healthy node exports, well below the observed ~330. */
const MIN_EXPECTED_SERIES = 100;

async function podIp(pod: string): Promise<string> {
  const result = await kubectl([
    '-n',
    NAMESPACE,
    'get',
    'pod',
    pod,
    '-o',
    'jsonpath={.status.podIP}',
  ]);
  expect(result.code, `could not read ${pod} IP: ${result.stderr}`).toBe(0);
  expect(result.stdout.trim(), `${pod} should have an IP`).toMatch(/^\d+\.\d+\.\d+\.\d+$/);
  return result.stdout.trim();
}

test.describe('section 4 — network isolation', () => {
  test('the NetworkPolicy exists and selects the cluster pods', async ({}, testInfo) => {
    const result = await observed(testInfo, 'get networkpolicy', () =>
      kubectl(['-n', NAMESPACE, 'get', 'networkpolicy', 'tidaldb', '-o', 'json']),
    );
    expect(result.code, result.stderr).toBe(0);

    const policy = JSON.parse(result.stdout) as {
      spec: { podSelector: { matchLabels: Record<string, string> }; ingress: unknown[] };
    };
    await recordJson(testInfo, 'policy-selector', policy.spec.podSelector);

    expect(
      policy.spec.podSelector.matchLabels['app.kubernetes.io/name'],
      'the policy must select the tidaldb pods',
    ).toBe('tidaldb');
    expect(
      policy.spec.ingress.length,
      'the policy must declare at least one ingress allowance',
    ).toBeGreaterThan(0);
  });

  test('a pod in an unrelated namespace is refused on the metrics port', async ({}, testInfo) => {
    const target = await podIp('tidaldb-0');

    const result = await observed(testInfo, 'foreign pod probes metrics port', () =>
      kubectl([
        '-n',
        FOREIGN_NAMESPACE,
        'exec',
        FOREIGN_POD,
        '--',
        'sh',
        '-c',
        `wget -qO- --timeout=5 http://${target}:${PORT_METRICS}/metrics 2>&1 | head -3`,
      ]),
    );

    const output = `${result.stdout}${result.stderr}`;
    await recordJson(testInfo, 'foreign-probe', {
      from: `${FOREIGN_NAMESPACE}/${FOREIGN_POD}`,
      to: `${target}:${PORT_METRICS}`,
      exitCode: result.code,
      output: output.trim().slice(0, 400),
    });

    // Before the policy existed this returned '# HELP tidaldb_uptime_seconds…'
    // from any pod in the cluster. The refusal is the whole point.
    expect(
      output,
      'a foreign pod must NOT receive metrics — this is the exposure the policy closes',
    ).not.toContain('tidaldb_');
    expect(output.toLowerCase()).toMatch(/refused|timed out|no route|unreachable/);
  });

  test('the scraper still collects metrics through the policy', async ({}, testInfo) => {
    const target = await podIp('tidaldb-0');

    const result = await observed(testInfo, 'scraper collects metrics', () =>
      kubectl(
        [
          '-n',
          OBS_NAMESPACE,
          'exec',
          'deploy/vmagent',
          '--',
          'sh',
          '-c',
          `wget -qO- --timeout=${SCRAPE_TIMEOUT_SECONDS} http://${target}:${PORT_METRICS}/metrics | grep -c '^tidaldb_'`,
        ],
        { timeoutMs: 45_000 },
      ),
    );
    expect(result.code, `scrape failed: ${result.stderr}`).toBe(0);

    const seriesCount = Number.parseInt(result.stdout.trim(), 10);
    await recordJson(testInfo, 'scrape-series-count', { target, seriesCount });

    // A policy that blocked the scraper too would be an observability outage
    // dressed as a security win.
    expect(
      seriesCount,
      'the scraper must still reach the metrics port through the policy',
    ).toBeGreaterThan(MIN_EXPECTED_SERIES);
  });

  test('metrics carry the labels the dashboard templates depend on', async ({
    playwright,
  }, testInfo) => {
    await withPortForward(OBS_NAMESPACE, 'svc/vmsingle', 8428, async (forward) => {
      const context = await playwright.request.newContext();
      try {
        const response = await context.get(
          `http://127.0.0.1:${forward.localPort}/api/v1/series`,
          { params: { 'match[]': 'tidaldb_health_ok' } },
        );
        expect(response.status(), 'vmsingle series query').toBe(200);

        const payload = (await response.json()) as { data: Record<string, string>[] };
        const labelSets = payload.data ?? [];
        await recordJson(testInfo, 'series-labels', {
          seriesCount: labelSets.length,
          labels: labelSets.length > 0 ? Object.keys(labelSets[0]).sort() : [],
        });

        expect(labelSets.length, 'tidaldb_health_ok must exist in the store').toBeGreaterThan(0);

        // The dashboard's $namespace / $pod template variables resolve against
        // these exact label names. A rename leaves every panel blank while the
        // series still exists — a monitoring outage that looks like an outage.
        for (const label of ['namespace', 'pod', 'container', 'partition_id']) {
          expect(
            Object.keys(labelSets[0]),
            `series must carry the '${label}' label the dashboard queries`,
          ).toContain(label);
        }
      } finally {
        await context.dispose();
      }
    });
  });
});