/** * Runbook section 5 — the Grafana dashboard. * * CAP-010 the dashboard is loaded and its panels render * * This is the only genuine browser surface in the whole evidence chain, and it * is the reason the visual pass matters. An API check proves the dashboard * OBJECT exists; it does not prove an operator opening it sees anything. The * dashboard shipped referencing datasource uid `${DS_PROMETHEUS}` — a Grafana * export-for-sharing placeholder with no `__inputs` block to resolve it — so * every panel queried a datasource that did not exist and the whole board was * blank. The API said "loaded". See BUG-007. * * Grafana here runs behind an auth proxy with the login form disabled * (GF_AUTH_DISABLE_LOGIN_FORM=true), so the suite authenticates with the * cluster's own grafana-admin basic credential rather than inventing an * identity or driving a form that does not exist. */ import { expect, test, type Page } from '@playwright/test'; import { withPortForward } from '../support/cluster'; import { recordJson } from '../support/evidence'; import { DASHBOARD_UID, OBS_NAMESPACE, grafanaPassword } from '../support/env'; const GRAFANA_PORT = 3000; /** Panels whose queries need metrics the running image does not emit. */ const EXPECTED_EMPTY_PANEL_IDS = [2, 3, 4, 5, 6]; function basicAuth(): string { return `Basic ${Buffer.from(`admin:${grafanaPassword()}`).toString('base64')}`; } type PanelState = { id: number; title: string; populated: boolean; noData: boolean; }; /** * Read every panel's rendered state. * * Grafana virtualizes: a panel below the fold is never queried and never draws. * So this scrolls the whole board first, then classifies — otherwise every * off-screen panel looks broken. */ async function readPanelStates(page: Page): Promise { await page.evaluate(async () => { const scroller = document.querySelector('[class*="scrollbar-view"]') ?? document.scrollingElement ?? document.body; const step = 400; for (let y = 0; y < scroller.scrollHeight; y += step) { scroller.scrollTop = y; await new Promise((resolve) => setTimeout(resolve, 250)); } scroller.scrollTop = 0; await new Promise((resolve) => setTimeout(resolve, 500)); }); return page.evaluate(() => { const panels: PanelState[] = []; for (const element of Array.from(document.querySelectorAll('[data-panelid]'))) { const id = Number.parseInt(element.getAttribute('data-panelid') ?? '0', 10); const text = element instanceof HTMLElement ? element.innerText : ''; const header = element.querySelector('h2, h6, [class*="panel-title"]'); // Everything below the panel title. const body = text.replace(/^[^\n]*\n?/, '').trim(); const noData = /no data/i.test(body); // A populated panel drew a chart OR rendered a value. Requiring a DIGIT // would be wrong: a stat panel with value mappings legitimately renders // words — "OK" for health, "none" for reseed pending — and those are the // most important readings on the board. const populated = !!element.querySelector('canvas') || !!element.querySelector('.uplot') || (body.length > 0 && !noData); panels.push({ id, title: (header?.textContent ?? text.split('\n')[0] ?? '').trim(), populated, noData, }); } return panels; }); } /** * Every `datasource.uid` anywhere in a dashboard model — panels, targets, * template variables, annotations. Only these are datasource references; the * dashboard's own `uid` and any panel `libraryPanel.uid` are not. */ function collectDatasourceUids(node: unknown, found: string[] = []): string[] { if (Array.isArray(node)) { for (const item of node) collectDatasourceUids(item, found); return found; } if (node === null || typeof node !== 'object') return found; for (const [key, value] of Object.entries(node)) { if (key === 'datasource' && value !== null && typeof value === 'object' && 'uid' in value) { const uid = value.uid; if (typeof uid === 'string') found.push(uid); } collectDatasourceUids(value, found); } return found; } test.describe('section 5 — metrics dashboard', () => { test('the dashboard is provisioned with a resolvable datasource', async ({ playwright, }, testInfo) => { await withPortForward(OBS_NAMESPACE, 'deploy/grafana', GRAFANA_PORT, async (forward) => { const context = await playwright.request.newContext({ extraHTTPHeaders: { authorization: basicAuth() }, }); try { const base = `http://127.0.0.1:${forward.localPort}`; const datasources = await context.get(`${base}/api/datasources`); expect(datasources.status(), 'datasource list').toBe(200); const provisioned = (await datasources.json()) as { uid: string; type: string }[]; const knownUids = provisioned.map((ds) => ds.uid); const dashboard = await context.get(`${base}/api/dashboards/uid/${DASHBOARD_UID}`); expect(dashboard.status(), `dashboard ${DASHBOARD_UID} must be provisioned`).toBe(200); const payload = (await dashboard.json()) as { dashboard: { title: string; panels: { id: number; type: string; title?: string }[] }; meta: { folderTitle?: string }; }; const serialized = JSON.stringify(payload.dashboard); // Walk for `datasource.uid` specifically. A blanket /"uid":"…"/ scan // also catches the dashboard's OWN uid and every library-panel uid, // and reports them as missing datasources. const referencedUids = [...new Set(collectDatasourceUids(payload.dashboard))]; const unresolvable = referencedUids.filter((uid) => !knownUids.includes(uid)); await recordJson(testInfo, 'datasource-wiring', { provisioned: knownUids, referenced: referencedUids, unresolvable, }); // The defect that made every panel blank. `${DS_PROMETHEUS}` is the // placeholder Grafana writes when you "export for sharing"; under // ConfigMap file-provisioning there is no import step to substitute it. expect( serialized, 'the dashboard must not ship an unresolved export placeholder', ).not.toContain('DS_PROMETHEUS'); expect( unresolvable, `dashboard references datasource uids that do not exist: ${unresolvable.join(', ')}`, ).toEqual([]); expect(payload.meta.folderTitle, 'dashboard folder').toBe('Databases'); const nonRowPanels = payload.dashboard.panels.filter((panel) => panel.type !== 'row'); // 15, not 13: the single "Fleet state" panel was split into three // single-query stat panels (Cluster health / Reseed pending / Indexed // vectors) so each gets its own thresholds and value mappings. See // BUG-008 and BUG-009. expect(nonRowPanels.length, 'expected the full 15-panel board').toBe(15); } finally { await context.dispose(); } }); }); test('every panel query returns data from the live store', async ({ playwright, }, testInfo) => { await withPortForward(OBS_NAMESPACE, 'deploy/grafana', GRAFANA_PORT, async (forward) => { const context = await playwright.request.newContext({ extraHTTPHeaders: { authorization: basicAuth(), 'content-type': 'application/json' }, }); try { const base = `http://127.0.0.1:${forward.localPort}`; const dashboard = await context.get(`${base}/api/dashboards/uid/${DASHBOARD_UID}`); const payload = (await dashboard.json()) as { dashboard: { panels: { id: number; type: string; title?: string; targets?: { expr?: string; refId?: string }[]; datasource?: { type?: string; uid?: string }; }[]; }; }; const results: Record = {}; for (const panel of payload.dashboard.panels) { if (panel.type === 'row' || !panel.targets) continue; for (const [index, target] of panel.targets.entries()) { if (!target.expr) continue; const response = await context.post(`${base}/api/ds/query`, { data: { queries: [ { refId: 'A', datasource: panel.datasource ?? { type: 'prometheus', uid: 'victoriametrics' }, expr: target.expr, instant: true, }, ], from: 'now-6h', to: 'now', }, }); const body = (await response.json()) as { results?: { A?: { frames?: unknown[]; status?: number; error?: string } }; }; results[`panel-${panel.id}-${index}`] = { expr: target.expr.slice(0, 90), frames: body.results?.A?.frames?.length ?? 0, status: response.status(), }; } } await recordJson(testInfo, 'panel-query-results', results); // Every query must at least EXECUTE. A query that errors is a broken // panel regardless of whether the underlying metric exists yet. const failed = Object.entries(results).filter(([, r]) => r.status !== 200); expect( failed.map(([k, r]) => `${k}: HTTP ${r.status}`), 'every panel query must execute against a real datasource', ).toEqual([]); } finally { await context.dispose(); } }); }); test('an operator opening the dashboard sees populated charts', async ({ page }, testInfo) => { await withPortForward(OBS_NAMESPACE, 'deploy/grafana', GRAFANA_PORT, async (forward) => { const consoleErrors: string[] = []; page.on('console', (message) => { if (message.type() !== 'error') return; const text = message.text(); // Grafana Live opens a WebSocket; a browser cannot attach a basic-auth // header to a WS upgrade, so this retry loop is an artefact of how the // suite authenticates and not a dashboard defect. if (text.includes('/api/live/ws')) return; consoleErrors.push(text.slice(0, 200)); }); await page.setExtraHTTPHeaders({ authorization: basicAuth() }); await page.goto( `http://127.0.0.1:${forward.localPort}/d/${DASHBOARD_UID}/?from=now-6h&to=now&refresh=`, { waitUntil: 'networkidle' }, ); // Wait for real plotted output, not merely for the shell to mount. await page .waitForFunction(() => document.querySelectorAll('canvas, .uplot').length > 3, { timeout: 90_000, }) .catch(() => undefined); const panels = await readPanelStates(page); const populated = panels.filter((panel) => panel.populated); const dataPanels = panels.filter((panel) => panel.id > 0 && !isRow(panel)); await recordJson(testInfo, 'rendered-panel-states', { total: panels.length, populated: populated.length, consoleErrors, panels, }); await testInfo.attach('dashboard.png', { body: await page.screenshot({ fullPage: true }), contentType: 'image/png', }); expect(consoleErrors, `dashboard raised console errors: ${consoleErrors.join(' | ')}`).toEqual( [], ); // The whole point: charts must actually be drawn. Before the datasource // fix this was zero while the API happily reported the dashboard loaded. expect( populated.length, `no panel drew anything — the board is blank to an operator. Populated: ` + `${populated.map((p) => p.id).join(',')}`, ).toBeGreaterThan(0); // The five HTTP panels legitimately have no data on this image; the rest // must render. Asserting the split means a real regression in the other // eight is visible instead of being absorbed into "some panels are empty". const blankNonHttp = dataPanels.filter( (panel) => !panel.populated && !EXPECTED_EMPTY_PANEL_IDS.includes(panel.id), ); expect( blankNonHttp.map((panel) => `${panel.id} ${panel.title}`), 'these panels should have data but drew nothing', ).toEqual([]); }); }); }); /** Row headers carry no data and must not be counted as blank panels. */ function isRow(panel: PanelState): boolean { return [1, 7, 11, 15].includes(panel.id); }