/** * Runbook section 8 — the fleet backup completed and captured every volume. * * CAP-013 the fleet backup completed and captured every volume * * The selector is the whole lesson here. Sorting every Backup by creation * timestamp and taking the newest returns a `restore-canary-*` run — 20 items, * one volume — which reports Completed and proves nothing about the fleet. The * check must filter on the schedule label that the freshness alert actually * watches. See BUG-004. */ import { expect, test } from '@playwright/test'; import { kubectl } from '../support/cluster'; import { observed, recordJson } from '../support/evidence'; import { BACKUP_NAMESPACE, BACKUP_SCHEDULE } from '../support/env'; /** A fleet backup covering thousands of objects, not a canary's handful. */ const MIN_FLEET_ITEMS = 1_000; /** Age beyond which a "most recent" backup is itself the finding. */ const MAX_BACKUP_AGE_HOURS = 48; type BackupStatus = { phase?: string; errors?: number; startTimestamp?: string; completionTimestamp?: string; progress?: { itemsBackedUp?: number; totalItems?: number }; }; test.describe('section 8 — backups', () => { test('the newest fleet-schedule backup completed with every item and every volume', async ({}, testInfo) => { const list = await observed(testInfo, 'list fleet-schedule backups', () => kubectl( [ '-n', BACKUP_NAMESPACE, 'get', 'backup.velero.io', '-l', `velero.io/schedule-name=${BACKUP_SCHEDULE}`, '--sort-by=.metadata.creationTimestamp', '-o', 'jsonpath={range .items[*]}{.metadata.name}{"\\n"}{end}', ], { timeoutMs: 45_000 }, ), ); expect(list.code, list.stderr).toBe(0); const names = list.stdout .trim() .split('\n') .filter((line) => line.trim() !== ''); expect( names.length, `no Backup carries label velero.io/schedule-name=${BACKUP_SCHEDULE} — either the ` + `schedule was renamed or it has never run, and the freshness alert watches this label`, ).toBeGreaterThan(0); const newest = names[names.length - 1]; // Guard the selector itself. A canary backup slipping through means the // filter regressed and this whole test would be verifying the wrong object. expect( newest, 'the selected backup must come from the fleet schedule, not a restore canary', ).not.toMatch(/restore-canary/); const detail = await observed(testInfo, `describe ${newest}`, () => kubectl( ['-n', BACKUP_NAMESPACE, 'get', 'backup.velero.io', newest, '-o', 'jsonpath={.status}'], { timeoutMs: 45_000 }, ), ); expect(detail.code, detail.stderr).toBe(0); const status = JSON.parse(detail.stdout) as BackupStatus; const ageHours = status.completionTimestamp ? (Date.now() - new Date(status.completionTimestamp).getTime()) / 3_600_000 : Number.POSITIVE_INFINITY; await recordJson(testInfo, 'backup-status', { name: newest, phase: status.phase, errors: status.errors ?? 0, itemsBackedUp: status.progress?.itemsBackedUp, totalItems: status.progress?.totalItems, completedAt: status.completionTimestamp, ageHours: Number.isFinite(ageHours) ? Number(ageHours.toFixed(1)) : null, }); expect(status.phase, `${newest} must have completed cleanly`).toBe('Completed'); expect(status.errors ?? 0, `${newest} reported errors`).toBe(0); expect( status.progress?.itemsBackedUp, 'every discovered item must be backed up', ).toBe(status.progress?.totalItems); expect( status.progress?.totalItems ?? 0, 'a fleet backup should cover thousands of items — a small count means the wrong object', ).toBeGreaterThan(MIN_FLEET_ITEMS); expect( ageHours, `the newest fleet backup is ${ageHours.toFixed(1)}h old — the schedule may have stopped`, ).toBeLessThan(MAX_BACKUP_AGE_HOURS); const volumes = await observed(testInfo, `pod volume backups for ${newest}`, () => kubectl( [ '-n', BACKUP_NAMESPACE, 'get', 'podvolumebackups', '-l', `velero.io/backup-name=${newest}`, '-o', 'jsonpath={range .items[*]}{.status.phase}{"\\n"}{end}', ], { timeoutMs: 45_000 }, ), ); expect(volumes.code, volumes.stderr).toBe(0); const phases: Record = {}; for (const phase of volumes.stdout.trim().split('\n')) { if (phase.trim() === '') continue; phases[phase.trim()] = (phases[phase.trim()] ?? 0) + 1; } await recordJson(testInfo, 'pod-volume-backup-phases', phases); expect( Object.keys(phases).length, 'the fleet backup must have captured pod volumes', ).toBeGreaterThan(0); // One failed PVB marks the whole Backup PartiallyFailed and freezes // velero_backup_last_successful_timestamp, so the fleet alert fires even // when everything that matters was captured. Every volume must be clean. expect( Object.keys(phases).sort(), `not every PodVolumeBackup completed: ${JSON.stringify(phases)}`, ).toEqual(['Completed']); }); test('no Velero backup in the namespace is stuck in progress', async ({}, testInfo) => { const result = await observed(testInfo, 'all backup phases', () => kubectl( [ '-n', BACKUP_NAMESPACE, 'get', 'backup.velero.io', '-o', 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.phase}{"\\n"}{end}', ], { timeoutMs: 45_000 }, ), ); expect(result.code, result.stderr).toBe(0); const stuck = result.stdout .trim() .split('\n') .filter((line) => line.trim() !== '') .map((line) => { const [name, phase] = line.split('\t'); return { name, phase }; }) .filter((backup) => backup.phase === 'InProgress' || backup.phase === 'Deleting'); await recordJson(testInfo, 'in-flight-backups', stuck); // A backup wedged InProgress blocks the next scheduled run and silently // stops the whole fleet from being protected. expect( stuck, `backups stuck in flight: ${stuck.map((b) => `${b.name}=${b.phase}`).join(', ')}`, ).toEqual([]); }); });