tidaldb/tests/e2e/support/cluster.ts

/**
 * Cluster-plane access for checks that have no HTTP surface.
 *
 * tidalDB serves JSON only — it has no operator web UI — and pod readiness,
 * NetworkPolicy enforcement, container logs, and Velero state are not reachable
 * over HTTP at all. Those checks shell out to kubectl. That is a deliberate,
 * documented choice: the alternative is inventing an HTTP surface that does not
 * exist and asserting against a fiction.
 *
 * Everything here returns real captured output for attachment as evidence, with
 * secrets redacted on the way out.
 */

import { execFile, spawn, type ChildProcess } from 'node:child_process';
import { createConnection } from 'node:net';
import { setTimeout as sleep } from 'node:timers/promises';
import { promisify } from 'node:util';
// Explicit `.ts` extension: this module is in the transitive closure of
// `tests/e2e/app/harness.ts`, which `npm run app:dev` loads with bare Node's
// type stripping — and Node's ESM resolver does not guess extensions. Playwright
// resolves either form, so specs elsewhere keep the extensionless style.
import { KUBECONFIG, TIDALCTL_BIN, redact } from './env.ts';

const execFileAsync = promisify(execFile);

export type CommandResult = {
  /** argv as run, for the evidence record. */
  command: string;
  code: number;
  stdout: string;
  stderr: string;
};

const DEFAULT_TIMEOUT_MS = 30_000;

/**
 * Run a binary with no shell. Never throws on a non-zero exit — the caller
 * asserts on `code`, so a failing command is evidence rather than a crash.
 */
export async function run(
  file: string,
  args: string[],
  options: { timeoutMs?: number; env?: Record<string, string> } = {},
): Promise<CommandResult> {
  const command = redact([file, ...args].join(' '));
  try {
    const { stdout, stderr } = await execFileAsync(file, args, {
      timeout: options.timeoutMs ?? DEFAULT_TIMEOUT_MS,
      maxBuffer: 32 * 1024 * 1024,
      env: { ...process.env, KUBECONFIG, ...options.env },
    });
    return { command, code: 0, stdout: redact(stdout), stderr: redact(stderr) };
  } catch (error) {
    const err = error as {
      code?: number | string;
      stdout?: string;
      stderr?: string;
      message: string;
    };
    return {
      command,
      code: typeof err.code === 'number' ? err.code : 1,
      stdout: redact(err.stdout ?? ''),
      stderr: redact(err.stderr ?? err.message),
    };
  }
}

/** Run kubectl with the pinned kubeconfig. */
export async function kubectl(
  args: string[],
  options: { timeoutMs?: number } = {},
): Promise<CommandResult> {
  return run('kubectl', args, options);
}

/** Run kubectl and fail loudly — for prerequisites, where a failure is fatal. */
export async function kubectlOrThrow(args: string[]): Promise<string> {
  const result = await kubectl(args);
  if (result.code !== 0) {
    throw new Error(
      `kubectl failed (exit ${result.code}): ${result.command}\n${result.stderr}`,
    );
  }
  return result.stdout;
}

/** Run the built tidalctl binary. */
export async function tidalctl(
  args: string[],
  options: { timeoutMs?: number } = {},
): Promise<CommandResult> {
  return run(TIDALCTL_BIN, args, options);
}

/**
 * Resolve one Secret key to plaintext. Used only to source credentials into the
 * suite; the value is registered for redaction and never attached.
 */
export async function secretValue(
  namespace: string,
  secret: string,
  key: string,
): Promise<string | undefined> {
  const result = await kubectl([
    '-n',
    namespace,
    'get',
    'secret',
    secret,
    '-o',
    `jsonpath={.data.${key}}`,
  ]);
  if (result.code !== 0 || result.stdout.trim() === '') return undefined;
  return Buffer.from(result.stdout.trim(), 'base64').toString('utf8');
}

/** True once something accepts a TCP connection on the port. */
async function portAccepts(port: number, host = '127.0.0.1'): Promise<boolean> {
  const { promise, resolve } = Promise.withResolvers<boolean>();
  const socket = createConnection({ port, host });
  const settle = (ok: boolean) => {
    socket.destroy();
    resolve(ok);
  };
  socket.setTimeout(1_000);
  socket.once('connect', () => settle(true));
  socket.once('timeout', () => settle(false));
  socket.once('error', () => settle(false));
  return promise;
}

export type PortForward = {
  localPort: number;
  target: string;
  close: () => Promise<void>;
};

/**
 * Ports are allocated from a high base and bumped per acquisition so parallel
 * workers never collide. Playwright workers are separate processes, so the
 * worker index is folded into the base.
 */
let portCursor = 0;
const PORT_BASE = 19_600;
const WORKER_STRIDE = 40;

/**
 * Open a port-forward and wait until the local port actually accepts a
 * connection.
 *
 * The manual runbook tells a human to `sleep 8` because a shorter wait races
 * kubectl's bind and returns an empty body that reads exactly like a dead node.
 * A harness can do better than a fixed sleep: poll until the port answers, then
 * proceed. Faster when the bind is quick, and it cannot produce that false
 * negative when the cluster is slow.
 */
export async function portForward(
  namespace: string,
  target: string,
  remotePort: number,
  options: { timeoutMs?: number } = {},
): Promise<PortForward> {
  const workerIndex = Number.parseInt(process.env.TEST_PARALLEL_INDEX ?? '0', 10);
  const localPort = PORT_BASE + workerIndex * WORKER_STRIDE + portCursor;
  portCursor = (portCursor + 1) % WORKER_STRIDE;

  const timeoutMs = options.timeoutMs ?? 30_000;
  const child: ChildProcess = spawn(
    'kubectl',
    ['-n', namespace, 'port-forward', target, `${localPort}:${remotePort}`],
    { env: { ...process.env, KUBECONFIG }, stdio: ['ignore', 'pipe', 'pipe'] },
  );

  let exitInfo: string | undefined;
  let stderr = '';
  child.stderr?.on('data', (chunk: Buffer) => {
    stderr += chunk.toString();
  });
  child.once('exit', (code, signal) => {
    exitInfo = `kubectl port-forward exited early (code=${code} signal=${signal}): ${stderr.trim()}`;
  });

  const close = async () => {
    if (child.exitCode !== null || child.signalCode !== null) return;
    const { promise, resolve } = Promise.withResolvers<void>();
    const hardKill = setTimeout(() => {
      child.kill('SIGKILL');
      resolve();
    }, 3_000);
    child.once('exit', () => {
      clearTimeout(hardKill);
      resolve();
    });
    child.kill('SIGTERM');
    await promise;
  };

  const deadline = Date.now() + timeoutMs;
  while (Date.now() < deadline) {
    if (exitInfo) {
      await close();
      throw new Error(`${exitInfo}\ntarget: ${namespace}/${target}:${remotePort}`);
    }
    if (await portAccepts(localPort)) {
      return { localPort, target: `${namespace}/${target}:${remotePort}`, close };
    }
    await sleep(250);
  }

  await close();
  throw new Error(
    `port-forward to ${namespace}/${target}:${remotePort} never accepted a connection ` +
      `on 127.0.0.1:${localPort} within ${timeoutMs}ms.\nkubectl stderr: ${stderr.trim()}`,
  );
}

/** Run `body` against an open port-forward and always tear it down. */
export async function withPortForward<T>(
  namespace: string,
  target: string,
  remotePort: number,
  body: (forward: PortForward) => Promise<T>,
): Promise<T> {
  const forward = await portForward(namespace, target, remotePort);
  try {
    return await body(forward);
  } finally {
    await forward.close();
  }
}

/**
 * Pod readiness, polled until every named pod is Ready or `budgetMs` elapses.
 *
 * A single sample is the wrong instrument here. This cluster is CONFIGURED to
 * self-restart: `replication.reseed_self_restart: true` means a node that latches
 * a reseed marker drains, exits(0), and lets the StatefulSet reinstall it on the
 * next boot. Observed 2026-08-30 minutes after a deploy — tidaldb-2 exited with
 * `{"reason":"reseed_self_restart","shard":1}`, reinstalled, and was Ready again
 * ~20s later. Sampling once mid-restart reports a healthy, self-healing cluster as
 * broken, and that false red is as corrosive as a false green.
 *
 * This is deliberately NOT a Playwright retry (the config sets `retries: 0` on
 * purpose, so a check that only passes on the second attempt still tells you
 * something true). It is a bounded convergence window with the whole timeline
 * returned as evidence: converging is a pass, not-converged-in-budget is a fail,
 * and the caller can see which pod flapped and when.
 */
export type ReadinessSample = { at: string; ready: Record<string, boolean> };

export type ReadinessOutcome = {
  converged: boolean;
  final: Record<string, boolean>;
  samples: ReadinessSample[];
  restarts: Record<string, number>;
};

export async function waitForPodsReady(
  namespace: string,
  pods: string[],
  budgetMs = 90_000,
  intervalMs = 3_000,
): Promise<ReadinessOutcome> {
  const deadline = Date.now() + budgetMs;
  const samples: ReadinessSample[] = [];
  let final: Record<string, boolean> = {};
  let restarts: Record<string, number> = {};

  for (;;) {
    const result = await kubectl([
      '-n',
      namespace,
      'get',
      'pods',
      '-l',
      'app.kubernetes.io/name=tidaldb',
      '-o',
      'jsonpath={range .items[*]}{.metadata.name}{" "}{.status.containerStatuses[0].ready}{" "}{.status.containerStatuses[0].restartCount}{"\\n"}{end}',
    ]);

    final = {};
    restarts = {};
    if (result.code === 0) {
      for (const line of result.stdout.trim().split('\n')) {
        if (line.trim() === '') continue;
        const [name, ready, restartCount] = line.trim().split(/\s+/);
        if (!name) continue;
        final[name] = ready === 'true';
        restarts[name] = Number.parseInt(restartCount ?? '0', 10) || 0;
      }
    }
    samples.push({ at: new Date().toISOString(), ready: { ...final } });

    const allReady = pods.length > 0 && pods.every((pod) => final[pod] === true);
    if (allReady) return { converged: true, final, samples, restarts };
    if (Date.now() >= deadline) return { converged: false, final, samples, restarts };
    await sleep(intervalMs);
  }
}