/** * Runbook section 4 — NetworkPolicy enforcement. * * CAP-008 the metrics port is not reachable from arbitrary pods * CAP-009 metrics reach the platform with the labels the dashboard queries * * A NetworkPolicy that blocks everything is an observability outage and one * that blocks nothing is theatre, so both directions are proven in the same * test: the foreign pod is refused AND the scraper still collects. */ import { expect, test } from '@playwright/test'; import { kubectl, withPortForward } from '../support/cluster'; import { observed, recordJson } from '../support/evidence'; import { FOREIGN_NAMESPACE, FOREIGN_POD, NAMESPACE, OBS_NAMESPACE, PORT_METRICS, } from '../support/env'; /** * Generous, because a 6-second client timeout truncates this scrape on a loaded * node and returns zero lines — indistinguishable from "this pod exports * nothing". That false negative briefly looked like a pod had stopped * exporting entirely (BUG-002). */ const SCRAPE_TIMEOUT_SECONDS = 15; /** Minimum series a healthy node exports, well below the observed ~330. */ const MIN_EXPECTED_SERIES = 100; async function podIp(pod: string): Promise { const result = await kubectl([ '-n', NAMESPACE, 'get', 'pod', pod, '-o', 'jsonpath={.status.podIP}', ]); expect(result.code, `could not read ${pod} IP: ${result.stderr}`).toBe(0); expect(result.stdout.trim(), `${pod} should have an IP`).toMatch(/^\d+\.\d+\.\d+\.\d+$/); return result.stdout.trim(); } test.describe('section 4 — network isolation', () => { test('the NetworkPolicy exists and selects the cluster pods', async ({}, testInfo) => { const result = await observed(testInfo, 'get networkpolicy', () => kubectl(['-n', NAMESPACE, 'get', 'networkpolicy', 'tidaldb', '-o', 'json']), ); expect(result.code, result.stderr).toBe(0); const policy = JSON.parse(result.stdout) as { spec: { podSelector: { matchLabels: Record }; ingress: unknown[] }; }; await recordJson(testInfo, 'policy-selector', policy.spec.podSelector); expect( policy.spec.podSelector.matchLabels['app.kubernetes.io/name'], 'the policy must select the tidaldb pods', ).toBe('tidaldb'); expect( policy.spec.ingress.length, 'the policy must declare at least one ingress allowance', ).toBeGreaterThan(0); }); test('a pod in an unrelated namespace is refused on the metrics port', async ({}, testInfo) => { const target = await podIp('tidaldb-0'); const result = await observed(testInfo, 'foreign pod probes metrics port', () => kubectl([ '-n', FOREIGN_NAMESPACE, 'exec', FOREIGN_POD, '--', 'sh', '-c', `wget -qO- --timeout=5 http://${target}:${PORT_METRICS}/metrics 2>&1 | head -3`, ]), ); const output = `${result.stdout}${result.stderr}`; await recordJson(testInfo, 'foreign-probe', { from: `${FOREIGN_NAMESPACE}/${FOREIGN_POD}`, to: `${target}:${PORT_METRICS}`, exitCode: result.code, output: output.trim().slice(0, 400), }); // Before the policy existed this returned '# HELP tidaldb_uptime_seconds…' // from any pod in the cluster. The refusal is the whole point. expect( output, 'a foreign pod must NOT receive metrics — this is the exposure the policy closes', ).not.toContain('tidaldb_'); expect(output.toLowerCase()).toMatch(/refused|timed out|no route|unreachable/); }); test('the scraper still collects metrics through the policy', async ({}, testInfo) => { const target = await podIp('tidaldb-0'); const result = await observed(testInfo, 'scraper collects metrics', () => kubectl( [ '-n', OBS_NAMESPACE, 'exec', 'deploy/vmagent', '--', 'sh', '-c', `wget -qO- --timeout=${SCRAPE_TIMEOUT_SECONDS} http://${target}:${PORT_METRICS}/metrics | grep -c '^tidaldb_'`, ], { timeoutMs: 45_000 }, ), ); expect(result.code, `scrape failed: ${result.stderr}`).toBe(0); const seriesCount = Number.parseInt(result.stdout.trim(), 10); await recordJson(testInfo, 'scrape-series-count', { target, seriesCount }); // A policy that blocked the scraper too would be an observability outage // dressed as a security win. expect( seriesCount, 'the scraper must still reach the metrics port through the policy', ).toBeGreaterThan(MIN_EXPECTED_SERIES); }); test('metrics carry the labels the dashboard templates depend on', async ({ playwright, }, testInfo) => { await withPortForward(OBS_NAMESPACE, 'svc/vmsingle', 8428, async (forward) => { const context = await playwright.request.newContext(); try { const response = await context.get( `http://127.0.0.1:${forward.localPort}/api/v1/series`, { params: { 'match[]': 'tidaldb_health_ok' } }, ); expect(response.status(), 'vmsingle series query').toBe(200); const payload = (await response.json()) as { data: Record[] }; const labelSets = payload.data ?? []; await recordJson(testInfo, 'series-labels', { seriesCount: labelSets.length, labels: labelSets.length > 0 ? Object.keys(labelSets[0]).sort() : [], }); expect(labelSets.length, 'tidaldb_health_ok must exist in the store').toBeGreaterThan(0); // The dashboard's $namespace / $pod template variables resolve against // these exact label names. A rename leaves every panel blank while the // series still exists — a monitoring outage that looks like an outage. for (const label of ['namespace', 'pod', 'container', 'partition_id']) { expect( Object.keys(labelSets[0]), `series must carry the '${label}' label the dashboard queries`, ).toContain(label); } } finally { await context.dispose(); } }); }); });