/** * Cluster-plane access for checks that have no HTTP surface. * * tidalDB serves JSON only — it has no operator web UI — and pod readiness, * NetworkPolicy enforcement, container logs, and Velero state are not reachable * over HTTP at all. Those checks shell out to kubectl. That is a deliberate, * documented choice: the alternative is inventing an HTTP surface that does not * exist and asserting against a fiction. * * Everything here returns real captured output for attachment as evidence, with * secrets redacted on the way out. */ import { execFile, spawn, type ChildProcess } from 'node:child_process'; import { createConnection } from 'node:net'; import { setTimeout as sleep } from 'node:timers/promises'; import { promisify } from 'node:util'; // Explicit `.ts` extension: this module is in the transitive closure of // `tests/e2e/app/harness.ts`, which `npm run app:dev` loads with bare Node's // type stripping — and Node's ESM resolver does not guess extensions. Playwright // resolves either form, so specs elsewhere keep the extensionless style. import { KUBECONFIG, TIDALCTL_BIN, redact } from './env.ts'; const execFileAsync = promisify(execFile); export type CommandResult = { /** argv as run, for the evidence record. */ command: string; code: number; stdout: string; stderr: string; }; const DEFAULT_TIMEOUT_MS = 30_000; /** * Run a binary with no shell. Never throws on a non-zero exit — the caller * asserts on `code`, so a failing command is evidence rather than a crash. */ export async function run( file: string, args: string[], options: { timeoutMs?: number; env?: Record } = {}, ): Promise { const command = redact([file, ...args].join(' ')); try { const { stdout, stderr } = await execFileAsync(file, args, { timeout: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, maxBuffer: 32 * 1024 * 1024, env: { ...process.env, KUBECONFIG, ...options.env }, }); return { command, code: 0, stdout: redact(stdout), stderr: redact(stderr) }; } catch (error) { const err = error as { code?: number | string; stdout?: string; stderr?: string; message: string; }; return { command, code: typeof err.code === 'number' ? err.code : 1, stdout: redact(err.stdout ?? ''), stderr: redact(err.stderr ?? err.message), }; } } /** Run kubectl with the pinned kubeconfig. */ export async function kubectl( args: string[], options: { timeoutMs?: number } = {}, ): Promise { return run('kubectl', args, options); } /** Run kubectl and fail loudly — for prerequisites, where a failure is fatal. */ export async function kubectlOrThrow(args: string[]): Promise { const result = await kubectl(args); if (result.code !== 0) { throw new Error( `kubectl failed (exit ${result.code}): ${result.command}\n${result.stderr}`, ); } return result.stdout; } /** Run the built tidalctl binary. */ export async function tidalctl( args: string[], options: { timeoutMs?: number } = {}, ): Promise { return run(TIDALCTL_BIN, args, options); } /** * Resolve one Secret key to plaintext. Used only to source credentials into the * suite; the value is registered for redaction and never attached. */ export async function secretValue( namespace: string, secret: string, key: string, ): Promise { const result = await kubectl([ '-n', namespace, 'get', 'secret', secret, '-o', `jsonpath={.data.${key}}`, ]); if (result.code !== 0 || result.stdout.trim() === '') return undefined; return Buffer.from(result.stdout.trim(), 'base64').toString('utf8'); } /** True once something accepts a TCP connection on the port. */ async function portAccepts(port: number, host = '127.0.0.1'): Promise { const { promise, resolve } = Promise.withResolvers(); const socket = createConnection({ port, host }); const settle = (ok: boolean) => { socket.destroy(); resolve(ok); }; socket.setTimeout(1_000); socket.once('connect', () => settle(true)); socket.once('timeout', () => settle(false)); socket.once('error', () => settle(false)); return promise; } export type PortForward = { localPort: number; target: string; close: () => Promise; }; /** * Ports are allocated from a high base and bumped per acquisition so parallel * workers never collide. Playwright workers are separate processes, so the * worker index is folded into the base. */ let portCursor = 0; const PORT_BASE = 19_600; const WORKER_STRIDE = 40; /** * Open a port-forward and wait until the local port actually accepts a * connection. * * The manual runbook tells a human to `sleep 8` because a shorter wait races * kubectl's bind and returns an empty body that reads exactly like a dead node. * A harness can do better than a fixed sleep: poll until the port answers, then * proceed. Faster when the bind is quick, and it cannot produce that false * negative when the cluster is slow. */ export async function portForward( namespace: string, target: string, remotePort: number, options: { timeoutMs?: number } = {}, ): Promise { const workerIndex = Number.parseInt(process.env.TEST_PARALLEL_INDEX ?? '0', 10); const localPort = PORT_BASE + workerIndex * WORKER_STRIDE + portCursor; portCursor = (portCursor + 1) % WORKER_STRIDE; const timeoutMs = options.timeoutMs ?? 30_000; const child: ChildProcess = spawn( 'kubectl', ['-n', namespace, 'port-forward', target, `${localPort}:${remotePort}`], { env: { ...process.env, KUBECONFIG }, stdio: ['ignore', 'pipe', 'pipe'] }, ); let exitInfo: string | undefined; let stderr = ''; child.stderr?.on('data', (chunk: Buffer) => { stderr += chunk.toString(); }); child.once('exit', (code, signal) => { exitInfo = `kubectl port-forward exited early (code=${code} signal=${signal}): ${stderr.trim()}`; }); const close = async () => { if (child.exitCode !== null || child.signalCode !== null) return; const { promise, resolve } = Promise.withResolvers(); const hardKill = setTimeout(() => { child.kill('SIGKILL'); resolve(); }, 3_000); child.once('exit', () => { clearTimeout(hardKill); resolve(); }); child.kill('SIGTERM'); await promise; }; const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { if (exitInfo) { await close(); throw new Error(`${exitInfo}\ntarget: ${namespace}/${target}:${remotePort}`); } if (await portAccepts(localPort)) { return { localPort, target: `${namespace}/${target}:${remotePort}`, close }; } await sleep(250); } await close(); throw new Error( `port-forward to ${namespace}/${target}:${remotePort} never accepted a connection ` + `on 127.0.0.1:${localPort} within ${timeoutMs}ms.\nkubectl stderr: ${stderr.trim()}`, ); } /** Run `body` against an open port-forward and always tear it down. */ export async function withPortForward( namespace: string, target: string, remotePort: number, body: (forward: PortForward) => Promise, ): Promise { const forward = await portForward(namespace, target, remotePort); try { return await body(forward); } finally { await forward.close(); } } /** * Pod readiness, polled until every named pod is Ready or `budgetMs` elapses. * * A single sample is the wrong instrument here. This cluster is CONFIGURED to * self-restart: `replication.reseed_self_restart: true` means a node that latches * a reseed marker drains, exits(0), and lets the StatefulSet reinstall it on the * next boot. Observed 2026-08-30 minutes after a deploy — tidaldb-2 exited with * `{"reason":"reseed_self_restart","shard":1}`, reinstalled, and was Ready again * ~20s later. Sampling once mid-restart reports a healthy, self-healing cluster as * broken, and that false red is as corrosive as a false green. * * This is deliberately NOT a Playwright retry (the config sets `retries: 0` on * purpose, so a check that only passes on the second attempt still tells you * something true). It is a bounded convergence window with the whole timeline * returned as evidence: converging is a pass, not-converged-in-budget is a fail, * and the caller can see which pod flapped and when. */ export type ReadinessSample = { at: string; ready: Record }; export type ReadinessOutcome = { converged: boolean; final: Record; samples: ReadinessSample[]; restarts: Record; }; export async function waitForPodsReady( namespace: string, pods: string[], budgetMs = 90_000, intervalMs = 3_000, ): Promise { const deadline = Date.now() + budgetMs; const samples: ReadinessSample[] = []; let final: Record = {}; let restarts: Record = {}; for (;;) { const result = await kubectl([ '-n', namespace, 'get', 'pods', '-l', 'app.kubernetes.io/name=tidaldb', '-o', 'jsonpath={range .items[*]}{.metadata.name}{" "}{.status.containerStatuses[0].ready}{" "}{.status.containerStatuses[0].restartCount}{"\\n"}{end}', ]); final = {}; restarts = {}; if (result.code === 0) { for (const line of result.stdout.trim().split('\n')) { if (line.trim() === '') continue; const [name, ready, restartCount] = line.trim().split(/\s+/); if (!name) continue; final[name] = ready === 'true'; restarts[name] = Number.parseInt(restartCount ?? '0', 10) || 0; } } samples.push({ at: new Date().toISOString(), ready: { ...final } }); const allReady = pods.length > 0 && pods.every((pod) => final[pod] === true); if (allReady) return { converged: true, final, samples, restarts }; if (Date.now() >= deadline) return { converged: false, final, samples, restarts }; await sleep(intervalMs); } }