check_dashboard_coverage.js

#!/usr/bin/env node
//
// Fails when the benchmark dashboard would silently render an empty card.
//
// The dashboard parses BenchmarkDotNet result names with regexes, and a name that does
// not match is dropped without a trace: the data publishes to gh-pages correctly and the
// card just renders blank. That has happened twice — EnumMap / EnumSet declare no
// [Params] at all, and DisjointSet once named its params property ElementCount — and
// neither produced any CI signal.
//
// This check closes that gap. It lifts the COLLECTIONS tables and the two name parsers
// straight out of the dashboard HTML rather than reimplementing them, so it validates
// the parser that actually ships and cannot drift from it.
//
// Structural checks, run on every PR (no benchmark run needed):
//   1. index.html and detail.html agree on the collection keys and their item counts;
//   2. every charted collection has a matching benchmark class registered in the
//      CoreBenchmarks array of src/Celerity.Benchmarks/Program.cs.
//
// Report checks, run in the benchmark job once the sharded suite has been merged:
//   3. every benchmark name in the report is understood by one of the dashboard parsers;
//   4. every (collection, op) pair the dashboard draws a card for resolves to both a BCL
//      and a Celerity measurement.
//
// Usage:
//   node scripts/check_dashboard_coverage.js                             # 1-2 only
//   node scripts/check_dashboard_coverage.js <joined-report-full.json>   # 1-4
// Run from the repository root.

'use strict';

const fs = require('fs');
const path = require('path');

const INDEX_HTML = path.join('web', 'dev', 'bench', 'index.html');
const DETAIL_HTML = path.join('web', 'dev', 'bench', 'detail.html');
const PROGRAM_CS = path.join('src', 'Celerity.Benchmarks', 'Program.cs');
const SELF = path.join('scripts', 'check_dashboard_coverage.js');

function fail(message) {
  console.error(`error: ${message}`);
  process.exit(1);
}

// ---- Lift declarations out of the dashboard source ----------------------------------
// Each pattern is anchored on the closing token at its known indentation, so a partial
// match is impossible: either the whole declaration comes out or extraction fails loudly.

function extract(source, file, label, pattern) {
  const m = source.match(pattern);
  if (!m) {
    fail(
      `could not extract ${label} from ${file}. The dashboard source was restructured; ` +
      `update the patterns in ${SELF} to match.`
    );
  }
  return m[0];
}

function loadDashboard(file) {
  const source = fs.readFileSync(file, 'utf8');
  const parts = [
    extract(source, file, 'NO_SWEEP', /var NO_SWEEP = [^;]+;/),
    extract(source, file, 'BCL_TYPES', /var BCL_TYPES = new Set\(\[[^\]]*\]\);/),
    extract(source, file, 'COLLECTIONS', /var COLLECTIONS = \[[\s\S]*?\n {2}\];/),
    extract(source, file, 'parseName', /function parseName\(name\) \{[\s\S]*?\n {2}\}/),
  ];
  // Only index.html carries the key builder and the hasher parser; detail.html is
  // consulted for its COLLECTIONS table alone.
  const idxKey = source.match(/function idxKey\([\s\S]*?\n {2}\}/);
  const parseHasher = source.match(/function parseHasher\(name, value\) \{[\s\S]*?\n {2}\}/);
  if (idxKey) parts.push(idxKey[0]);
  if (parseHasher) parts.push(parseHasher[0]);
  parts.push(
    'return { NO_SWEEP: NO_SWEEP, COLLECTIONS: COLLECTIONS, parseName: parseName,' +
    ' idxKey: typeof idxKey === "function" ? idxKey : null,' +
    ' parseHasher: typeof parseHasher === "function" ? parseHasher : null };'
  );
  return new Function(parts.join('\n'))();
}

// The CI-tracked suite, as `{Prefix}Benchmark` type names.
function loadCoreBenchmarks() {
  const source = fs.readFileSync(PROGRAM_CS, 'utf8');
  const block = source.match(/CoreBenchmarks\s*=\s*\{([\s\S]*?)\};/);
  if (!block) {
    fail(`could not find the CoreBenchmarks array in ${PROGRAM_CS}; update the pattern in ${SELF}.`);
  }
  return new Set([...block[1].matchAll(/typeof\((\w+)\)/g)].map((m) => m[1]));
}

function loadBenchmarkNames(reportPath) {
  let report;
  try {
    report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
  } catch (e) {
    fail(`could not read ${reportPath}: ${e.message}`);
  }
  if (!report || !Array.isArray(report.Benchmarks)) {
    fail(`${reportPath} has no Benchmarks array — is it a BenchmarkDotNet *-report-full.json?`);
  }
  return report.Benchmarks.map((b) => b.FullName).filter(Boolean);
}

// ---- Checks -------------------------------------------------------------------------

function main() {
  const reportPath = process.argv[2] || null;

  const index = loadDashboard(INDEX_HTML);
  const detail = loadDashboard(DETAIL_HTML);
  if (!index.idxKey) fail(`could not extract idxKey from ${INDEX_HTML}`);
  if (!index.parseHasher) fail(`could not extract parseHasher from ${INDEX_HTML}`);

  const problems = [];

  // (1) The two tables must agree, or a card that renders on the grid dead-ends on click.
  const detailByKey = new Map(detail.COLLECTIONS.map((c) => [c.key, c]));
  for (const col of index.COLLECTIONS) {
    const twin = detailByKey.get(col.key);
    if (!twin) {
      problems.push(`${col.key} is charted in ${INDEX_HTML} but missing from ${DETAIL_HTML}'s COLLECTIONS — its detail page will show "Unknown benchmark".`);
      continue;
    }
    const a = JSON.stringify(col.items || null);
    const b = JSON.stringify(twin.items || null);
    if (a !== b) {
      problems.push(`${col.key} declares items ${a} in ${INDEX_HTML} but ${b} in ${DETAIL_HTML} — the detail page will reject the item count the grid links to.`);
    }
  }
  for (const col of detail.COLLECTIONS) {
    if (!index.COLLECTIONS.some((c) => c.key === col.key)) {
      problems.push(`${col.key} is listed in ${DETAIL_HTML} but missing from ${INDEX_HTML}'s COLLECTIONS — it has no card on the grid.`);
    }
  }

  // (2) A charted collection whose class is not in the CI suite can never receive data.
  const core = loadCoreBenchmarks();
  for (const col of index.COLLECTIONS) {
    if (!core.has(`${col.key}Benchmark`)) {
      problems.push(`${col.key} is charted in ${INDEX_HTML} but ${col.key}Benchmark is not registered in the CoreBenchmarks array of ${PROGRAM_CS} — it will never be measured in CI.`);
    }
  }

  if (reportPath) {
    const names = loadBenchmarkNames(reportPath);

    // (3) Nothing in the report may be silently unrenderable.
    const unparsed = names.filter((n) => !index.parseName(n) && !index.parseHasher(n, 0));
    if (unparsed.length > 0) {
      const classes = [...new Set(unparsed.map((n) => n.split('.')[0]))];
      problems.push(
        `${unparsed.length} benchmark name(s) match no dashboard parser and would be dropped at render time, ` +
        `from: ${classes.join(', ')}. First: "${unparsed[0]}". ` +
        `The usual cause is a [Params] property not named ItemCount — rename it, or teach both dashboard parsers the new shape.`
      );
    }

    // (4) Every card the dashboard draws must have both series behind it.
    const idx = {};
    for (const name of names) {
      const p = index.parseName(name);
      if (!p) continue;
      const key = index.idxKey(p.collection, p.op, p.itemCount);
      if (!idx[key]) idx[key] = {};
      idx[key][p.isBcl ? 'bcl' : 'celerity'] = true;
    }

    for (const col of index.COLLECTIONS) {
      // The primary count, exactly as index.html picks it: a card's ratio text may fall
      // back to the smaller count, but its chart is always seriesFor(..., primaryN), and
      // that is what decides between a sparkline and "awaiting data". So the primary
      // count is the one that has to resolve.
      const items = col.items || [1000, 100000];
      const primaryN = items[items.length - 1];
      for (const op of col.ops) {
        const pair = idx[index.idxKey(col.key, op, primaryN)];
        if (!pair || !pair.bcl || !pair.celerity) {
          const at = primaryN == null ? 'no sweep' : primaryN;
          problems.push(`${col.key}.${op} has no BCL+Celerity pair at ${at} — that card renders "awaiting data".`);
        }
      }
    }
  }

  if (problems.length > 0) {
    console.error(`Dashboard coverage check failed (${problems.length} problem(s)):\n`);
    for (const p of problems) console.error(`  - ${p}`);
    console.error(
      `\nEvery collection in the COLLECTIONS arrays of ${INDEX_HTML} and ${DETAIL_HTML} must be measured ` +
      `by the CI suite and must resolve to real measurements. See CONTRIBUTING.md, "The dashboard".`
    );
    process.exit(1);
  }

  const cards = index.COLLECTIONS.reduce((acc, c) => acc + c.ops.length, 0);
  console.log(
    `Dashboard coverage OK: ${cards} cards across ${index.COLLECTIONS.length} collections` +
    (reportPath ? ` resolve from the joined report.` : ` are wired to registered benchmark classes (structural checks only — pass a joined report to verify the data).`)
  );
}

main();