tidaldb/tests/e2e/features/05-metrics-dashboard.spec.ts

/**
 * Runbook section 5 — the Grafana dashboard.
 *
 * CAP-010 the dashboard is loaded and its panels render
 *
 * This is the only genuine browser surface in the whole evidence chain, and it
 * is the reason the visual pass matters. An API check proves the dashboard
 * OBJECT exists; it does not prove an operator opening it sees anything. The
 * dashboard shipped referencing datasource uid `${DS_PROMETHEUS}` — a Grafana
 * export-for-sharing placeholder with no `__inputs` block to resolve it — so
 * every panel queried a datasource that did not exist and the whole board was
 * blank. The API said "loaded". See BUG-007.
 *
 * Grafana here runs behind an auth proxy with the login form disabled
 * (GF_AUTH_DISABLE_LOGIN_FORM=true), so the suite authenticates with the
 * cluster's own grafana-admin basic credential rather than inventing an
 * identity or driving a form that does not exist.
 */

import { expect, test, type Page } from '@playwright/test';
import { withPortForward } from '../support/cluster';
import { recordJson } from '../support/evidence';
import { DASHBOARD_UID, OBS_NAMESPACE, grafanaPassword } from '../support/env';

const GRAFANA_PORT = 3000;

/** Panels whose queries need metrics the running image does not emit. */
const EXPECTED_EMPTY_PANEL_IDS = [2, 3, 4, 5, 6];

function basicAuth(): string {
  return `Basic ${Buffer.from(`admin:${grafanaPassword()}`).toString('base64')}`;
}

type PanelState = {
  id: number;
  title: string;
  populated: boolean;
  noData: boolean;
};

/**
 * Read every panel's rendered state.
 *
 * Grafana virtualizes: a panel below the fold is never queried and never draws.
 * So this scrolls the whole board first, then classifies — otherwise every
 * off-screen panel looks broken.
 */
async function readPanelStates(page: Page): Promise<PanelState[]> {
  await page.evaluate(async () => {
    const scroller =
      document.querySelector('[class*="scrollbar-view"]') ??
      document.scrollingElement ??
      document.body;
    const step = 400;
    for (let y = 0; y < scroller.scrollHeight; y += step) {
      scroller.scrollTop = y;
      await new Promise((resolve) => setTimeout(resolve, 250));
    }
    scroller.scrollTop = 0;
    await new Promise((resolve) => setTimeout(resolve, 500));
  });

  return page.evaluate(() => {
    const panels: PanelState[] = [];
    for (const element of Array.from(document.querySelectorAll('[data-panelid]'))) {
      const id = Number.parseInt(element.getAttribute('data-panelid') ?? '0', 10);
      const text = element instanceof HTMLElement ? element.innerText : '';
      const header = element.querySelector('h2, h6, [class*="panel-title"]');
      // Everything below the panel title.
      const body = text.replace(/^[^\n]*\n?/, '').trim();
      const noData = /no data/i.test(body);
      // A populated panel drew a chart OR rendered a value. Requiring a DIGIT
      // would be wrong: a stat panel with value mappings legitimately renders
      // words — "OK" for health, "none" for reseed pending — and those are the
      // most important readings on the board.
      const populated =
        !!element.querySelector('canvas') ||
        !!element.querySelector('.uplot') ||
        (body.length > 0 && !noData);
      panels.push({
        id,
        title: (header?.textContent ?? text.split('\n')[0] ?? '').trim(),
        populated,
        noData,
      });
    }
    return panels;
  });
}

/**
 * Every `datasource.uid` anywhere in a dashboard model — panels, targets,
 * template variables, annotations. Only these are datasource references; the
 * dashboard's own `uid` and any panel `libraryPanel.uid` are not.
 */
function collectDatasourceUids(node: unknown, found: string[] = []): string[] {
  if (Array.isArray(node)) {
    for (const item of node) collectDatasourceUids(item, found);
    return found;
  }
  if (node === null || typeof node !== 'object') return found;

  for (const [key, value] of Object.entries(node)) {
    if (key === 'datasource' && value !== null && typeof value === 'object' && 'uid' in value) {
      const uid = value.uid;
      if (typeof uid === 'string') found.push(uid);
    }
    collectDatasourceUids(value, found);
  }
  return found;
}

test.describe('section 5 — metrics dashboard', () => {
  test('the dashboard is provisioned with a resolvable datasource', async ({
    playwright,
  }, testInfo) => {
    await withPortForward(OBS_NAMESPACE, 'deploy/grafana', GRAFANA_PORT, async (forward) => {
      const context = await playwright.request.newContext({
        extraHTTPHeaders: { authorization: basicAuth() },
      });
      try {
        const base = `http://127.0.0.1:${forward.localPort}`;

        const datasources = await context.get(`${base}/api/datasources`);
        expect(datasources.status(), 'datasource list').toBe(200);
        const provisioned = (await datasources.json()) as { uid: string; type: string }[];
        const knownUids = provisioned.map((ds) => ds.uid);

        const dashboard = await context.get(`${base}/api/dashboards/uid/${DASHBOARD_UID}`);
        expect(dashboard.status(), `dashboard ${DASHBOARD_UID} must be provisioned`).toBe(200);
        const payload = (await dashboard.json()) as {
          dashboard: { title: string; panels: { id: number; type: string; title?: string }[] };
          meta: { folderTitle?: string };
        };

        const serialized = JSON.stringify(payload.dashboard);
        // Walk for `datasource.uid` specifically. A blanket /"uid":"…"/ scan
        // also catches the dashboard's OWN uid and every library-panel uid,
        // and reports them as missing datasources.
        const referencedUids = [...new Set(collectDatasourceUids(payload.dashboard))];
        const unresolvable = referencedUids.filter((uid) => !knownUids.includes(uid));

        await recordJson(testInfo, 'datasource-wiring', {
          provisioned: knownUids,
          referenced: referencedUids,
          unresolvable,
        });

        // The defect that made every panel blank. `${DS_PROMETHEUS}` is the
        // placeholder Grafana writes when you "export for sharing"; under
        // ConfigMap file-provisioning there is no import step to substitute it.
        expect(
          serialized,
          'the dashboard must not ship an unresolved export placeholder',
        ).not.toContain('DS_PROMETHEUS');
        expect(
          unresolvable,
          `dashboard references datasource uids that do not exist: ${unresolvable.join(', ')}`,
        ).toEqual([]);

        expect(payload.meta.folderTitle, 'dashboard folder').toBe('Databases');
        const nonRowPanels = payload.dashboard.panels.filter((panel) => panel.type !== 'row');
        // 15, not 13: the single "Fleet state" panel was split into three
        // single-query stat panels (Cluster health / Reseed pending / Indexed
        // vectors) so each gets its own thresholds and value mappings. See
        // BUG-008 and BUG-009.
        expect(nonRowPanels.length, 'expected the full 15-panel board').toBe(15);
      } finally {
        await context.dispose();
      }
    });
  });

  test('every panel query returns data from the live store', async ({
    playwright,
  }, testInfo) => {
    await withPortForward(OBS_NAMESPACE, 'deploy/grafana', GRAFANA_PORT, async (forward) => {
      const context = await playwright.request.newContext({
        extraHTTPHeaders: { authorization: basicAuth(), 'content-type': 'application/json' },
      });
      try {
        const base = `http://127.0.0.1:${forward.localPort}`;
        const dashboard = await context.get(`${base}/api/dashboards/uid/${DASHBOARD_UID}`);
        const payload = (await dashboard.json()) as {
          dashboard: {
            panels: {
              id: number;
              type: string;
              title?: string;
              targets?: { expr?: string; refId?: string }[];
              datasource?: { type?: string; uid?: string };
            }[];
          };
        };

        const results: Record<string, { expr: string; frames: number; status: number }> = {};
        for (const panel of payload.dashboard.panels) {
          if (panel.type === 'row' || !panel.targets) continue;
          for (const [index, target] of panel.targets.entries()) {
            if (!target.expr) continue;
            const response = await context.post(`${base}/api/ds/query`, {
              data: {
                queries: [
                  {
                    refId: 'A',
                    datasource: panel.datasource ?? { type: 'prometheus', uid: 'victoriametrics' },
                    expr: target.expr,
                    instant: true,
                  },
                ],
                from: 'now-6h',
                to: 'now',
              },
            });
            const body = (await response.json()) as {
              results?: { A?: { frames?: unknown[]; status?: number; error?: string } };
            };
            results[`panel-${panel.id}-${index}`] = {
              expr: target.expr.slice(0, 90),
              frames: body.results?.A?.frames?.length ?? 0,
              status: response.status(),
            };
          }
        }

        await recordJson(testInfo, 'panel-query-results', results);

        // Every query must at least EXECUTE. A query that errors is a broken
        // panel regardless of whether the underlying metric exists yet.
        const failed = Object.entries(results).filter(([, r]) => r.status !== 200);
        expect(
          failed.map(([k, r]) => `${k}: HTTP ${r.status}`),
          'every panel query must execute against a real datasource',
        ).toEqual([]);
      } finally {
        await context.dispose();
      }
    });
  });

  test('an operator opening the dashboard sees populated charts', async ({ page }, testInfo) => {
    await withPortForward(OBS_NAMESPACE, 'deploy/grafana', GRAFANA_PORT, async (forward) => {
      const consoleErrors: string[] = [];
      page.on('console', (message) => {
        if (message.type() !== 'error') return;
        const text = message.text();
        // Grafana Live opens a WebSocket; a browser cannot attach a basic-auth
        // header to a WS upgrade, so this retry loop is an artefact of how the
        // suite authenticates and not a dashboard defect.
        if (text.includes('/api/live/ws')) return;
        consoleErrors.push(text.slice(0, 200));
      });

      await page.setExtraHTTPHeaders({ authorization: basicAuth() });
      await page.goto(
        `http://127.0.0.1:${forward.localPort}/d/${DASHBOARD_UID}/?from=now-6h&to=now&refresh=`,
        { waitUntil: 'networkidle' },
      );

      // Wait for real plotted output, not merely for the shell to mount.
      await page
        .waitForFunction(() => document.querySelectorAll('canvas, .uplot').length > 3, {
          timeout: 90_000,
        })
        .catch(() => undefined);

      const panels = await readPanelStates(page);
      const populated = panels.filter((panel) => panel.populated);
      const dataPanels = panels.filter((panel) => panel.id > 0 && !isRow(panel));

      await recordJson(testInfo, 'rendered-panel-states', {
        total: panels.length,
        populated: populated.length,
        consoleErrors,
        panels,
      });

      await testInfo.attach('dashboard.png', {
        body: await page.screenshot({ fullPage: true }),
        contentType: 'image/png',
      });

      expect(consoleErrors, `dashboard raised console errors: ${consoleErrors.join(' | ')}`).toEqual(
        [],
      );

      // The whole point: charts must actually be drawn. Before the datasource
      // fix this was zero while the API happily reported the dashboard loaded.
      expect(
        populated.length,
        `no panel drew anything — the board is blank to an operator. Populated: ` +
          `${populated.map((p) => p.id).join(',')}`,
      ).toBeGreaterThan(0);

      // The five HTTP panels legitimately have no data on this image; the rest
      // must render. Asserting the split means a real regression in the other
      // eight is visible instead of being absorbed into "some panels are empty".
      const blankNonHttp = dataPanels.filter(
        (panel) => !panel.populated && !EXPECTED_EMPTY_PANEL_IDS.includes(panel.id),
      );
      expect(
        blankNonHttp.map((panel) => `${panel.id} ${panel.title}`),
        'these panels should have data but drew nothing',
      ).toEqual([]);
    });
  });
});

/** Row headers carry no data and must not be counted as blank panels. */
function isRow(panel: PanelState): boolean {
  return [1, 7, 11, 15].includes(panel.id);
}