tidaldb/tests/e2e/app/harness.ts

/**
 * The content-feed app's lifecycle, in one place.
 *
 * `startApp()` boots a throwaway standalone tidalDB, seeds the fixture catalog,
 * serves the page plus a token-injecting `/api` proxy, and hands back a handle
 * whose `close()` unwinds all of it. The semantic spec, the demo capture and
 * `npm run app:dev` all drive the identical lifecycle, so what a human sees by
 * hand is what the assertions ran against.
 *
 * Two rules shape the design:
 *
 *  - **Relative `/api` only.** `.sdlc/guidance.md:165` forbids a hardcoded
 *    `http://localhost:PORT` in frontend code and prescribes a dev-server proxy.
 *    Independently, a bearer must never reach a browser — a page holding the key
 *    leaks it to anyone who opens devtools, and the demo capture would photograph
 *    it. The proxy satisfies both: the page calls `/api/feed`, the server adds
 *    `Authorization`, the key stays server-side.
 *  - **Poll, never sleep.** A fixed sleep either wastes time or produces the
 *    empty-body false negative that reads exactly like a dead node. This is the
 *    same discipline as `tests/e2e/support/cluster.ts:154` `portForward`.
 *
 * It targets a LOCAL standalone node, never the deployed cluster: `skip` is
 * declared `permanent: true`, so seeding signals into production would be an
 * irreversible mutation of the live corpus.
 */

import { spawn, type ChildProcess } from 'node:child_process';
import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http';
import { createConnection } from 'node:net';
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { existsSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { extname, join, resolve, sep } from 'node:path';
import { setTimeout as sleep } from 'node:timers/promises';

import { run } from '../support/cluster.ts';
import { redact } from '../support/env.ts';
import {
  CATALOG,
  EMBEDDING_DIM,
  PROBE_IDS,
  type FixtureGroundTruth,
} from './fixture-contract.ts';

const REPO_ROOT = resolve(import.meta.dirname, '../../..');
const PUBLIC_DIR = join(import.meta.dirname, 'public');
const SCHEMA_CONFIGMAP = join(REPO_ROOT, 'k8s/cluster/schema-configmap.yaml');
const SERVER_BIN = join(REPO_ROOT, 'target/debug/tidal-server');
const FIXTURE_BIN = join(REPO_ROOT, 'target/debug/feed-fixture');

const BOOT_TIMEOUT_MS = 60_000;
const BUILD_TIMEOUT_MS = 900_000;
const SEED_TIMEOUT_MS = 120_000;
const KILL_GRACE_MS = 3_000;

/**
 * Ask the OS for a free port rather than computing one from a base.
 *
 * A fixed base collides with whatever is already holding it — a leftover node, a
 * second checkout, or `npm run app:dev` running in another terminal while the
 * suite runs. Letting the kernel choose removes that whole class of failure. The
 * handover window between close and the child's bind is microseconds, and a lost
 * race surfaces as the child's own "address in use" stderr rather than a hang.
 */
async function freePort(): Promise<number> {
  const probe = createServer();
  const { promise, resolve: settle, reject } = Promise.withResolvers<number>();
  probe.once('error', reject);
  probe.listen(0, '127.0.0.1', () => {
    const address = probe.address();
    const port = typeof address === 'object' && address !== null ? address.port : 0;
    probe.close(() => settle(port));
  });
  return promise;
}

export type RunningApp = {
  /** The app origin: serves the page and proxies `/api/*`. */
  url: string;
  /** The tidalDB node itself, for probes that bypass the app. */
  nodeUrl: string;
  /** The oracle the fixture binary emitted for this exact corpus. */
  groundTruth: FixtureGroundTruth;
  /** Decay as declared by the schema THIS node loaded, not a copy of it. */
  declaredDecay: DeclaredDecay;
  /**
   * The throwaway data dir. Exposed so a caller can say where a run's state went,
   * and so a long-lived host can install a synchronous last-resort cleanup — an
   * async `close()` that throws would otherwise leave the dir behind.
   */
  dataDir: string;
  close: () => Promise<void>;
};

export type StartAppOptions = {
  /** Bearer to inject into proxied requests. A local standalone needs none. */
  apiKey?: string;
  /** Log lifecycle progress to stdout — for `npm run app:dev`, quiet in tests. */
  verbose?: boolean;
};

/**
 * Extract the schema the DEPLOYED cluster loads, from its ConfigMap wrapper.
 *
 * Not interchangeable with `tidal-server/config/default-schema.yaml`: that file is
 * otherwise identical but declares `dimensions: 128`, while the cluster declares
 * 1536. Seeding 1536-wide vectors against a 128-wide slot is rejected with a 422
 * that reads like a malformed body, so the dimension is asserted here rather than
 * discovered three layers downstream.
 */
async function extractSchema(): Promise<string> {
  const raw = await readFile(SCHEMA_CONFIGMAP, 'utf8');
  const marker = '  schema.yaml: |\n';
  const start = raw.indexOf(marker);
  if (start < 0) {
    throw new Error(`${SCHEMA_CONFIGMAP} has no '  schema.yaml: |' block scalar to extract`);
  }
  const body = raw.slice(start + marker.length);
  const schema = body
    .split('\n')
    .map((line) => (line.startsWith('    ') ? line.slice(4) : line))
    .join('\n');

  if (!schema.startsWith('signals:')) {
    throw new Error(`extracted schema does not start with 'signals:'; got: ${schema.slice(0, 80)}`);
  }
  if (!new RegExp(`dimensions:\\s*${EMBEDDING_DIM}\\b`).test(schema)) {
    throw new Error(
      `extracted schema does not declare dimensions: ${EMBEDDING_DIM}. The fixture ` +
        `contract and the schema must agree or every embedding write is rejected as 422.`,
    );
  }
  return schema;
}

/**
 * The decay each signal declares, read out of the schema the node actually
 * loaded.
 *
 * The decay assertion compares an OBSERVED decay rate against a DECLARED
 * half-life. Hardcoding the declared value in the test would make it a
 * tautology the moment the schema changed, so it is parsed from the same string
 * that was handed to `--schema`.
 *
 * A deliberately small parser rather than a YAML dependency: the shape it reads
 * is four lines of a file this repo owns, and it fails loudly (returns nothing
 * for a signal) rather than guessing.
 */
export type DeclaredDecay = Record<string, number | 'permanent'>;

function parseDeclaredDecay(schema: string): DeclaredDecay {
  const declared: DeclaredDecay = {};
  let current: string | undefined;
  for (const line of schema.split('\n')) {
    const name = /^\s*-\s*name:\s*(\S+)/.exec(line);
    if (name?.[1]) {
      current = name[1];
      continue;
    }
    if (current === undefined) continue;
    const halfLife = /^\s*half_life_seconds:\s*(\d+)/.exec(line);
    if (halfLife?.[1]) {
      declared[current] = Number(halfLife[1]);
      continue;
    }
    if (/^\s*permanent:\s*true/.test(line)) declared[current] = 'permanent';
  }
  return declared;
}

/** True once something accepts a TCP connection on the port. */
async function portAccepts(port: number): Promise<boolean> {
  const { promise, resolve: settle } = Promise.withResolvers<boolean>();
  const socket = createConnection({ port, host: '127.0.0.1' });
  const done = (ok: boolean) => {
    socket.destroy();
    settle(ok);
  };
  socket.setTimeout(1_000);
  socket.once('connect', () => done(true));
  socket.once('timeout', () => done(false));
  socket.once('error', () => done(false));
  return promise;
}

/** True once the node answers `/health` with 2xx. */
async function healthOk(nodeUrl: string): Promise<boolean> {
  try {
    const response = await fetch(`${nodeUrl}/health`, {
      signal: AbortSignal.timeout(2_000),
    });
    return response.ok;
  } catch {
    return false;
  }
}

const buildsInFlight = new Map<string, Promise<void>>();

/**
 * Build a cargo binary if it is not already present, once per process.
 *
 * `feed-fixture` is new, so no existing tree has it; failing with "binary
 * missing, go run cargo" would make the hermetic suite un-runnable from a clean
 * checkout for no reason. Dependencies are already compiled, so this is seconds,
 * not minutes — but the timeout allows for a cold cache.
 */
function ensureBinary(binPath: string, cargoArgs: string[], verbose: boolean): Promise<void> {
  if (existsSync(binPath)) return Promise.resolve();
  const existing = buildsInFlight.get(binPath);
  if (existing) return existing;

  const build = (async () => {
    if (verbose) console.log(`[app] building ${binPath} (missing)`);
    const result = await run('cargo', cargoArgs, { timeoutMs: BUILD_TIMEOUT_MS });
    if (result.code !== 0 || !existsSync(binPath)) {
      throw new Error(
        `could not build ${binPath}\n$ ${result.command}\nexit ${result.code}\n${result.stderr}`,
      );
    }
  })();
  buildsInFlight.set(binPath, build);
  return build;
}

/** MIME types for the handful of things the page is made of. */
const CONTENT_TYPES: Record<string, string> = {
  '.html': 'text/html; charset=utf-8',
  '.js': 'text/javascript; charset=utf-8',
  '.css': 'text/css; charset=utf-8',
  '.json': 'application/json; charset=utf-8',
  '.svg': 'image/svg+xml',
};

/**
 * Read the whole request body as text.
 *
 * Text rather than bytes because the only thing the page ever sends is one small
 * JSON object, and `string` is unambiguously a `BodyInit` — TS's `BufferSource`
 * is `ArrayBufferView<ArrayBuffer>`, which a pooled Node `Buffer` does not
 * satisfy. If this ever needs to forward binary, that is a real change, not a cast.
 */
async function readBody(req: IncomingMessage): Promise<string> {
  const chunks: Buffer[] = [];
  for await (const chunk of req) chunks.push(chunk as Buffer);
  return Buffer.concat(chunks).toString('utf8');
}

/**
 * Forward one `/api/*` request to the node, adding the bearer.
 *
 * A byte pipe: it must never reorder, re-score or filter a response, because the
 * ordering IS the thing under test (`CODING_GUIDELINES.md:88`).
 */
async function proxy(
  req: IncomingMessage,
  res: ServerResponse,
  nodeUrl: string,
  apiKey: string | undefined,
): Promise<void> {
  const path = (req.url ?? '/').slice('/api'.length);
  const headers: Record<string, string> = { accept: 'application/json' };
  const contentType = req.headers['content-type'];
  if (contentType) headers['content-type'] = contentType;
  if (apiKey) headers.authorization = `Bearer ${apiKey}`;

  const method = req.method ?? 'GET';
  const hasBody = method !== 'GET' && method !== 'HEAD';
  const body = hasBody ? await readBody(req) : undefined;

  try {
    const upstream = await fetch(`${nodeUrl}${path}`, {
      method,
      headers,
      body,
      signal: AbortSignal.timeout(30_000),
    });
    const payload = Buffer.from(await upstream.arrayBuffer());
    res.writeHead(upstream.status, {
      'content-type': upstream.headers.get('content-type') ?? 'application/json',
      'content-length': String(payload.byteLength),
    });
    res.end(payload);
  } catch (error) {
    // Surface the upstream fault as a real status rather than hanging the page.
    const message = redact(error instanceof Error ? error.message : String(error));
    const payload = Buffer.from(JSON.stringify({ error: `proxy: ${message}` }));
    res.writeHead(502, {
      'content-type': 'application/json',
      'content-length': String(payload.byteLength),
    });
    res.end(payload);
  }
}

/** Serve one file from the public dir, refusing anything outside it. */
async function serveStatic(req: IncomingMessage, res: ServerResponse): Promise<void> {
  const requested = (req.url ?? '/').split('?')[0] ?? '/';
  const relative = requested === '/' ? 'index.html' : requested.replace(/^\/+/, '');
  const target = resolve(PUBLIC_DIR, relative);
  if (target !== PUBLIC_DIR && !target.startsWith(PUBLIC_DIR + sep)) {
    res.writeHead(403, { 'content-type': 'text/plain' });
    res.end('forbidden');
    return;
  }
  try {
    const body = await readFile(target);
    res.writeHead(200, {
      'content-type': CONTENT_TYPES[extname(target)] ?? 'application/octet-stream',
      'content-length': String(body.byteLength),
    });
    res.end(body);
  } catch {
    res.writeHead(404, { 'content-type': 'text/plain' });
    res.end('not found');
  }
}

/**
 * Boot a standalone node, seed the fixture catalog, and serve the app against it.
 *
 * Every stage that can fail reports why: an early child exit surfaces the node's
 * stderr, and a non-zero seed exit surfaces the seeder's transcript, because a
 * partially seeded corpus would make every downstream assertion meaningless.
 */
export async function startApp(options: StartAppOptions = {}): Promise<RunningApp> {
  const verbose = options.verbose ?? false;
  const log = (message: string) => {
    if (verbose) console.log(`[app] ${message}`);
  };

  await ensureBinary(SERVER_BIN, ['build', '-p', 'tidal-server', '--bin', 'tidal-server'], verbose);
  await ensureBinary(
    FIXTURE_BIN,
    ['build', '-p', 'tidal-stress', '--bin', 'feed-fixture'],
    verbose,
  );

  const dataDir = await mkdtemp(join(tmpdir(), 'tidaldb-feed-app-'));
  const schemaPath = join(dataDir, 'schema.yaml');
  const catalogPath = join(dataDir, 'catalog.json');
  const truthPath = join(dataDir, 'truth.json');
  const schema = await extractSchema();
  const declaredDecay = parseDeclaredDecay(schema);
  await writeFile(schemaPath, schema, 'utf8');
  await writeFile(catalogPath, JSON.stringify(CATALOG), 'utf8');
  // `--data-dir` must already exist: the server treats a missing directory as a
  // config error rather than creating it (verified against its own stderr).
  await mkdir(join(dataDir, 'data'), { recursive: true });

  const nodePort = await freePort();
  const nodeUrl = `http://127.0.0.1:${nodePort}`;

  // `--listen` also reads the PORT env var (tidal-server/src/main.rs:99). An
  // explicit flag wins in clap, but dropping PORT removes the ambiguity entirely.
  const childEnv = { ...process.env };
  delete childEnv.PORT;
  delete childEnv.TIDAL_CONFIG;

  log(`booting node on ${nodeUrl} (data ${dataDir})`);
  const child: ChildProcess = spawn(
    SERVER_BIN,
    [
      'standalone',
      '--listen',
      `127.0.0.1:${nodePort}`,
      '--schema',
      schemaPath,
      '--data-dir',
      join(dataDir, 'data'),
    ],
    { cwd: REPO_ROOT, env: childEnv, stdio: ['ignore', 'pipe', 'pipe'] },
  );

  let nodeStderr = '';
  let exitInfo: string | undefined;
  child.stderr?.on('data', (chunk: Buffer) => {
    nodeStderr += chunk.toString();
  });
  child.stdout?.on('data', () => {
    /* the node's own log; kept off the test transcript unless it fails */
  });
  child.once('exit', (code, signal) => {
    exitInfo = `tidal-server exited early (code=${code} signal=${signal})`;
  });

  const stopNode = async () => {
    if (child.exitCode !== null || child.signalCode !== null) return;
    const { promise, resolve: settle } = Promise.withResolvers<void>();
    const hardKill = setTimeout(() => {
      child.kill('SIGKILL');
      settle();
    }, KILL_GRACE_MS);
    child.once('exit', () => {
      clearTimeout(hardKill);
      settle();
    });
    child.kill('SIGTERM');
    await promise;
  };

  const cleanup = async (server?: Server) => {
    if (server) {
      const { promise, resolve: settle } = Promise.withResolvers<void>();
      server.close(() => settle());
      server.closeAllConnections?.();
      await promise;
    }
    await stopNode();
    await rm(dataDir, { recursive: true, force: true });
  };

  try {
    const deadline = Date.now() + BOOT_TIMEOUT_MS;
    let ready = false;
    while (Date.now() < deadline) {
      if (exitInfo) {
        throw new Error(`${exitInfo}\n--- node stderr ---\n${redact(nodeStderr.trim())}`);
      }
      if (await healthOk(nodeUrl)) {
        ready = true;
        break;
      }
      await sleep(150);
    }
    if (!ready) {
      throw new Error(
        `tidal-server never answered ${nodeUrl}/health within ${BOOT_TIMEOUT_MS}ms\n` +
          `--- node stderr ---\n${redact(nodeStderr.trim())}`,
      );
    }
    log(`node healthy; seeding ${CATALOG.length} items`);

    const seed = await run(
      FIXTURE_BIN,
      [
        '--base-url',
        nodeUrl,
        '--catalog',
        catalogPath,
        '--out',
        truthPath,
        '--dim',
        String(EMBEDDING_DIM),
        ...PROBE_IDS.flatMap((id) => ['--probe', String(id)]),
      ],
      { timeoutMs: SEED_TIMEOUT_MS },
    );
    if (seed.code !== 0) {
      throw new Error(
        `fixture seeding failed — a partial corpus makes every assertion ` +
          `meaningless, so this is fatal.\n$ ${seed.command}\nexit ${seed.code}\n` +
          `${seed.stdout}\n${seed.stderr}`,
      );
    }
    const groundTruth = JSON.parse(await readFile(truthPath, 'utf8')) as FixtureGroundTruth;

    const route = async (req: IncomingMessage, res: ServerResponse): Promise<void> => {
      if (req.url?.startsWith('/api/') === true) {
        return proxy(req, res, nodeUrl, options.apiKey);
      }
      if ((req.url ?? '/').split('?')[0] === '/catalog.json') {
        // The page joins ranked ids to titles, so it needs the catalog — served
        // from the same contract the seeder used, never a second copy.
        const payload = Buffer.from(JSON.stringify(CATALOG));
        res.writeHead(200, {
          'content-type': 'application/json; charset=utf-8',
          'content-length': String(payload.byteLength),
        });
        res.end(payload);
        return;
      }
      return serveStatic(req, res);
    };

    const server = createServer((req, res) => {
      route(req, res).catch((error: unknown) => {
        if (res.headersSent) {
          res.end();
          return;
        }
        res.writeHead(500, { 'content-type': 'text/plain' });
        res.end(redact(error instanceof Error ? error.message : String(error)));
      });
    });

    // Bind port 0 and read back what the kernel gave us: no probe, no race.
    const listening = Promise.withResolvers<number>();
    server.once('error', (error) => listening.reject(error));
    server.listen(0, '127.0.0.1', () => {
      const address = server.address();
      listening.resolve(typeof address === 'object' && address !== null ? address.port : 0);
    });
    const appPort = await listening.promise;

    const url = `http://127.0.0.1:${appPort}`;
    // Prove the app origin is actually reachable before handing it to a caller,
    // so a bind that silently failed cannot look like a page-render bug later.
    if (!(await portAccepts(appPort))) {
      throw new Error(`app server bound ${url} but the port does not accept connections`);
    }
    log(`app ready at ${url}`);

    return {
      url,
      nodeUrl,
      groundTruth,
      declaredDecay,
      dataDir,
      close: () => cleanup(server),
    };
  } catch (error) {
    await cleanup();
    throw error;
  }
}