tidaldb/tests/e2e/features/08-backups.spec.ts

/**
 * Runbook section 8 — the fleet backup completed and captured every volume.
 *
 * CAP-013 the fleet backup completed and captured every volume
 *
 * The selector is the whole lesson here. Sorting every Backup by creation
 * timestamp and taking the newest returns a `restore-canary-*` run — 20 items,
 * one volume — which reports Completed and proves nothing about the fleet. The
 * check must filter on the schedule label that the freshness alert actually
 * watches. See BUG-004.
 */

import { expect, test } from '@playwright/test';
import { kubectl } from '../support/cluster';
import { observed, recordJson } from '../support/evidence';
import { BACKUP_NAMESPACE, BACKUP_SCHEDULE } from '../support/env';

/** A fleet backup covering thousands of objects, not a canary's handful. */
const MIN_FLEET_ITEMS = 1_000;

/** Age beyond which a "most recent" backup is itself the finding. */
const MAX_BACKUP_AGE_HOURS = 48;

type BackupStatus = {
  phase?: string;
  errors?: number;
  startTimestamp?: string;
  completionTimestamp?: string;
  progress?: { itemsBackedUp?: number; totalItems?: number };
};

test.describe('section 8 — backups', () => {
  test('the newest fleet-schedule backup completed with every item and every volume', async ({}, testInfo) => {
    const list = await observed(testInfo, 'list fleet-schedule backups', () =>
      kubectl(
        [
          '-n',
          BACKUP_NAMESPACE,
          'get',
          'backup.velero.io',
          '-l',
          `velero.io/schedule-name=${BACKUP_SCHEDULE}`,
          '--sort-by=.metadata.creationTimestamp',
          '-o',
          'jsonpath={range .items[*]}{.metadata.name}{"\\n"}{end}',
        ],
        { timeoutMs: 45_000 },
      ),
    );
    expect(list.code, list.stderr).toBe(0);

    const names = list.stdout
      .trim()
      .split('\n')
      .filter((line) => line.trim() !== '');
    expect(
      names.length,
      `no Backup carries label velero.io/schedule-name=${BACKUP_SCHEDULE} — either the ` +
        `schedule was renamed or it has never run, and the freshness alert watches this label`,
    ).toBeGreaterThan(0);

    // Pick the newest backup that has actually FINISHED. Selecting `names.at(-1)`
    // unconditionally made this test unpassable during the daily backup window:
    // the schedule fires at 03:30 and measured runs take 9.1–24.8 min, so for
    // ~25 minutes every day the newest object is legitimately `InProgress` and
    // this asserted `phase === 'Completed'` against it. That is a false failure by
    // construction — it says "backups are broken" when a backup is working.
    // The in-flight case has its own bounded assertion below (see the stuck test).
    const finished: string[] = [];
    for (const name of names) {
      const phaseProbe = await kubectl(
        ['-n', BACKUP_NAMESPACE, 'get', 'backup.velero.io', name, '-o', 'jsonpath={.status.phase}'],
        { timeoutMs: 45_000 },
      );
      if (phaseProbe.code === 0 && phaseProbe.stdout.trim() !== 'InProgress') {
        finished.push(name);
      }
    }
    expect(
      finished.length,
      'every fleet-schedule backup is still in flight — nothing has ever finished, which is ' +
        'a real failure rather than a timing artifact',
    ).toBeGreaterThan(0);
    const newest = finished[finished.length - 1];

    // Guard the selector itself. A canary backup slipping through means the
    // filter regressed and this whole test would be verifying the wrong object.
    expect(
      newest,
      'the selected backup must come from the fleet schedule, not a restore canary',
    ).not.toMatch(/restore-canary/);

    const detail = await observed(testInfo, `describe ${newest}`, () =>
      kubectl(
        ['-n', BACKUP_NAMESPACE, 'get', 'backup.velero.io', newest, '-o', 'jsonpath={.status}'],
        { timeoutMs: 45_000 },
      ),
    );
    expect(detail.code, detail.stderr).toBe(0);

    const status = JSON.parse(detail.stdout) as BackupStatus;
    const ageHours = status.completionTimestamp
      ? (Date.now() - new Date(status.completionTimestamp).getTime()) / 3_600_000
      : Number.POSITIVE_INFINITY;

    await recordJson(testInfo, 'backup-status', {
      name: newest,
      phase: status.phase,
      errors: status.errors ?? 0,
      itemsBackedUp: status.progress?.itemsBackedUp,
      totalItems: status.progress?.totalItems,
      completedAt: status.completionTimestamp,
      ageHours: Number.isFinite(ageHours) ? Number(ageHours.toFixed(1)) : null,
    });

    expect(status.phase, `${newest} must have completed cleanly`).toBe('Completed');
    expect(status.errors ?? 0, `${newest} reported errors`).toBe(0);
    expect(
      status.progress?.itemsBackedUp,
      'every discovered item must be backed up',
    ).toBe(status.progress?.totalItems);
    expect(
      status.progress?.totalItems ?? 0,
      'a fleet backup should cover thousands of items — a small count means the wrong object',
    ).toBeGreaterThan(MIN_FLEET_ITEMS);
    expect(
      ageHours,
      `the newest fleet backup is ${ageHours.toFixed(1)}h old — the schedule may have stopped`,
    ).toBeLessThan(MAX_BACKUP_AGE_HOURS);

    const volumes = await observed(testInfo, `pod volume backups for ${newest}`, () =>
      kubectl(
        [
          '-n',
          BACKUP_NAMESPACE,
          'get',
          'podvolumebackups',
          '-l',
          `velero.io/backup-name=${newest}`,
          '-o',
          'jsonpath={range .items[*]}{.status.phase}{"\\n"}{end}',
        ],
        { timeoutMs: 45_000 },
      ),
    );
    expect(volumes.code, volumes.stderr).toBe(0);

    const phases: Record<string, number> = {};
    for (const phase of volumes.stdout.trim().split('\n')) {
      if (phase.trim() === '') continue;
      phases[phase.trim()] = (phases[phase.trim()] ?? 0) + 1;
    }
    await recordJson(testInfo, 'pod-volume-backup-phases', phases);

    expect(
      Object.keys(phases).length,
      'the fleet backup must have captured pod volumes',
    ).toBeGreaterThan(0);

    // One failed PVB marks the whole Backup PartiallyFailed and freezes
    // velero_backup_last_successful_timestamp, so the fleet alert fires even
    // when everything that matters was captured. Every volume must be clean.
    expect(
      Object.keys(phases).sort(),
      `not every PodVolumeBackup completed: ${JSON.stringify(phases)}`,
    ).toEqual(['Completed']);
  });

  test('no Velero backup is in flight beyond the measured completion envelope', async ({}, testInfo) => {
    const result = await observed(testInfo, 'all backup phases', () =>
      kubectl(
        [
          '-n',
          BACKUP_NAMESPACE,
          'get',
          'backup.velero.io',
          '-o',
          'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.phase}{"\\t"}{.status.startTimestamp}{"\\n"}{end}',
        ],
        { timeoutMs: 45_000 },
      ),
    );
    expect(result.code, result.stderr).toBe(0);

    // CALIBRATED 2026-08-31 against every fleet-daily run in the namespace:
    // successful backups complete in 9.1–24.8 min (n=15), and the observed
    // FAILURE mode is a hard 240.0 min timeout that lands PartiallyFailed
    // (2026-08-17/19/25). So "in flight" is normal and "in flight for an hour" is
    // not. 60 min is ~2.4x the slowest success and a quarter of the timeout.
    //
    // The previous assertion was `InProgress → fail`, full stop. The daily fires
    // at 03:30, so for ~25 minutes every day this test reported the fleet
    // unprotected while it was actively being protected. A gate that cries wolf on
    // a schedule gets muted, and then it is not a gate.
    const STUCK_AFTER_MIN = 60;

    const inFlight = result.stdout
      .trim()
      .split('\n')
      .filter((line) => line.trim() !== '')
      .map((line) => {
        const [name, phase, startTimestamp] = line.split('\t');
        const ageMin = startTimestamp
          ? (Date.now() - new Date(startTimestamp).getTime()) / 60_000
          : Number.POSITIVE_INFINITY;
        return { name, phase, startTimestamp, ageMin: Number(ageMin.toFixed(1)) };
      })
      .filter((backup) => backup.phase === 'InProgress' || backup.phase === 'Deleting');

    const stuck = inFlight.filter((backup) => backup.ageMin > STUCK_AFTER_MIN);

    await recordJson(testInfo, 'in-flight-backups', {
      stuckAfterMin: STUCK_AFTER_MIN,
      inFlight,
      stuck,
    });

    // A backup wedged InProgress blocks the next scheduled run and silently
    // stops the whole fleet from being protected.
    expect(
      stuck,
      `backups in flight past ${STUCK_AFTER_MIN} min: ` +
        `${stuck.map((b) => `${b.name}=${b.phase} (${b.ageMin} min)`).join(', ')}`,
    ).toEqual([]);
  });
});