test_pages_report_history.mjs

import assert from 'node:assert/strict';
import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises';
import http from 'node:http';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';

import { buildPagesReport } from '../../scripts/build-pages-report.mjs';
import {
  findShellReport,
  testRunDump,
} from '../../scripts/omarchy-shell-evidence.mjs';
import { reportCases } from '../../scripts/report-cases.mjs';
import { renderReportSummary } from '../../scripts/render-ci-report-summary.mjs';
import { verifyPublishedReport } from '../../scripts/verify-pages-report.mjs';

test('publishes the Markdown evidence table for same-repository pull requests', async () => {
  for (const workflow of [
    'midscene-ubuntu-22.04.yml',
    'midscene-omarchy-4.0.3.yml',
  ]) {
    const source = await readFile(
      new URL(`../../.github/workflows/${workflow}`, import.meta.url),
      'utf8',
    );
    const pagesJob = source.slice(source.indexOf('  pages-report:'));
    assert.match(
      pagesJob,
      /github\.event\.pull_request\.head\.repo\.full_name == github\.repository/,
      `${workflow} must publish its report and Summary table for trusted PRs`,
    );
    assert.doesNotMatch(
      pagesJob,
      /github\.ref_name == vars\.PAGES_REPORT_BRANCH/,
      `${workflow} must not suppress the PR Summary table behind a branch check`,
    );
  }
});

test('records every shard and renders the Summary before Pages deployment', async () => {
  for (const workflow of [
    'midscene-ubuntu-22.04.yml',
    'midscene-omarchy-4.0.3.yml',
  ]) {
    const source = await readFile(
      new URL(`../../.github/workflows/${workflow}`, import.meta.url),
      'utf8',
    );
    assert.match(source, /Record shard result for report aggregation/);
    assert.match(
      source,
      /if: always\(\) && !cancelled\(\)/,
      `${workflow} must preserve failure evidence without extending manually cancelled runs`,
    );
    assert.match(
      source,
      /^\s*max-parallel: 5$/m,
      `${workflow} must run its independent VM shards in parallel`,
    );
    assert.match(source, /if-no-files-found: error/);
    assert.match(source, /Create bundle even when every shard failed early/);
    assert.match(
      source,
      /Upload .*shard report[\s\S]*?overwrite: true/,
      `${workflow} must replace a rerun shard artifact instead of keeping duplicate names`,
    );
    assert.match(
      source,
      /Upload combined report bundle[\s\S]*?overwrite: true/,
      `${workflow} must replace the previous attempt's combined bundle`,
    );
  }

  const deploy = await readFile(
    new URL('../../.github/workflows/deploy-midscene-report.yml', import.meta.url),
    'utf8',
  );
  const summary = deploy.indexOf('Add complete evidence table');
  const deployment = deploy.indexOf('Deploy report history to GitHub Pages');
  const deployJob = deploy.indexOf('\n  deploy:');
  assert.ok(summary > 0 && summary < deployment);
  assert.match(
    deploy.slice(summary, deployment),
    /steps\.build-report\.outcome == 'success'/,
  );
  assert.ok(deployJob > summary);
  assert.doesNotMatch(deploy.slice(0, deployJob), /environment:/);
  assert.match(
    deploy,
    /name: \$\{\{ inputs\.artifact-name \}\}-pages-\$\{\{ github\.run_attempt \}\}/,
  );
  assert.match(
    deploy,
    /artifact_name: \$\{\{ inputs\.artifact-name \}\}-pages-\$\{\{ github\.run_attempt \}\}/,
  );
  assert.match(
    deploy,
    /name: \$\{\{ inputs\.artifact-name \}\}-manifest-\$\{\{ github\.run_attempt \}\}/,
  );
});

test('rerun aggregation excludes the previous combined report artifact', async () => {
  const source = await readFile(
    new URL('../../.github/workflows/midscene-omarchy-4.0.3.yml', import.meta.url),
    'utf8',
  );
  assert.match(source, /pattern: omarchy-midscene-omarchy-\*/);
  assert.doesNotMatch(source, /pattern: omarchy-midscene-\*\s*$/m);
});

test('assigns every product case to exactly one balanced shard', async () => {
  const shardCounts = new Map();
  let caseCount = 0;
  for (const file of [
    'onboarding.yaml',
    'onboarding-regressions.yaml',
    'runtime.yaml',
  ]) {
    const source = await readFile(
      new URL(`../e2e/cases/${file}`, import.meta.url),
      'utf8',
    );
    const cases = source
      .split(/(?=^  - name:)/m)
      .filter((section) => section.startsWith('  - name:'));
    for (const testCase of cases) {
      const tags = [...testCase.matchAll(/^    tags: \[([^\]]+)\]$/gm)];
      assert.equal(tags.length, 1, testCase.split('\n')[0]);
      const shards = tags[0][1].split(',').map((tag) => tag.trim())
        .filter((tag) => /^shard-[1-4]$/.test(tag));
      assert.equal(shards.length, 1, testCase.split('\n')[0]);
      shardCounts.set(shards[0], (shardCounts.get(shards[0]) ?? 0) + 1);
      caseCount += 1;
    }
  }
  assert.equal(caseCount, 14);
  assert.deepEqual(Object.fromEntries(shardCounts), {
    'shard-1': 2,
    'shard-2': 3,
    'shard-3': 5,
    'shard-4': 4,
  });
});

test('keeps visual E2E interactions at task level with proven tap exceptions', async () => {
  for (const file of [
    'onboarding.yaml',
    'onboarding-regressions.yaml',
    'polishing.yaml',
    'runtime.yaml',
  ]) {
    const source = await readFile(
      new URL(`../e2e/cases/${file}`, import.meta.url),
      'utf8',
    );
    const oneShotActions = source.match(
      /^\s+-\s+(?:aiTap|aiScroll|aiInput|computer\.inputText|computer\.tapPoint|computer\.selectTriggerPreset):.*$/gm,
    ) ?? [];
    assert.deepEqual(
      oneShotActions.map((line) => line.trim()),
      file === 'onboarding.yaml'
        ? [
            '- aiTap: The large blue full-width Open Doubao sign-in button near the bottom of the central Sign in panel, around 50% width and 73% screen height',
            '- aiTap: The blue Simulate successful sign-in button in the CI-only modal, around 50% width and 43% screen height',
            '- aiTap: Check microphone · 3 seconds button on the Microphone step',
            '- aiTap: Right-arrow Next button in the fixed top navigation',
            '- aiScroll:',
            '- aiTap: Test endpoint button in the Voice polishing section',
            '- aiTap: Right-arrow Next button in the fixed top navigation',
            '- aiTap: Start voice test button',
            '- aiTap: Finish & check result button',
            '- aiTap: Enabled Finish setup button that says Voice test passed and nothing pasted or sent',
          ]
        : file === 'onboarding-regressions.yaml'
          ? [
              '- aiTap: The large blue full-width Open Doubao sign-in button near the bottom of the central Sign in panel, around 50% width and 73% screen height',
              '- aiTap: The large blue full-width Open Doubao sign-in button near the bottom of the central Sign in panel, around 50% width and 73% screen height',
              '- aiTap: The blue Simulate successful sign-in button in the CI-only modal, around 50% width and 43% screen height',
              '- aiTap: The large blue full-width Open Doubao sign-in button near the bottom of the central Sign in panel, around 50% width and 73% screen height',
              '- aiTap: The blue Simulate successful sign-in button in the CI-only modal, around 50% width and 43% screen height',
              '- computer.tapPoint: { target: Left-arrow Previous button in the fixed top navigation, point: { x: 240, y: 203 } }',
              '- computer.tapPoint: { target: Right-arrow Next button in the fixed top navigation, point: { x: 1038, y: 203 } }',
              '- computer.selectTriggerPreset: { preset: Disabled }',
              '- computer.tapPoint: { target: Right-arrow Next button in the fixed top navigation, point: { x: 1038, y: 203 } }',
              '- computer.selectTriggerPreset: { preset: F8 }',
              '- computer.tapPoint: { target: Right-arrow Next button in the fixed top navigation, point: { x: 1038, y: 203 } }',
              '- computer.tapPoint: { target: Left-arrow Previous button in the fixed top navigation, point: { x: 240, y: 203 } }',
              '- aiScroll:',
              '- computer.tapPoint: { target: Enabled Voice polishing switch, point: { x: 1180, y: 316 } }',
              '- computer.tapPoint: { target: Disabled Voice polishing switch in the collapsed card, point: { x: 1166, y: 554 } }',
              '- aiScroll:',
              '- aiScroll:',
              '- computer.inputText: { target: The text entry to the right of Model containing synthetic-model, value: ci-fail-model, point: { x: 850, y: 477 } }',
              '- aiTap: Test endpoint button in the Voice polishing section',
              '- computer.inputText: { target: The text entry to the right of Model containing ci-fail-model, value: ci-ok-model, point: { x: 850, y: 477 } }',
              '- aiTap: Test failed — try again button in the Voice polishing section',
              '- aiTap: Right-arrow Next button in the fixed top navigation',
              '- aiScroll:',
              '- computer.tapPoint: { target: Left-arrow Previous button in the fixed top navigation, point: { x: 240, y: 203 } }',
              '- aiTap: Check microphone · 3 seconds button on the Microphone step',
              '- aiTap: Right-arrow Next button in the fixed top navigation',
              '- aiScroll:',
              '- aiTap: Start voice test button in the Voice test section',
              '- aiTap: Cancel test button in the active voice test',
              '- aiTap: Start voice test button in the Voice test section',
              '- aiTap: Finish & check result button in the active voice test',
              '- aiTap: The Recognition service dropdown currently showing Doubao account',
              '- aiTap: Volcengine API option in the open Recognition service dropdown',
              '- aiTap: Test API key button on the Recognition step',
              '- aiTap: Right-arrow Next button in the fixed top navigation',
              '- aiTap: Recognition service dropdown currently showing Doubao account',
              '- aiTap: Deepgram Nova-3 (English) option in the opened dropdown',
              '- aiScroll:',
              '- computer.inputText:',
              '- aiTap: Test API key button below the API Key field',
              '- aiTap: The microphone dropdown currently showing Synthetic microphone one',
              '- aiTap: Synthetic microphone two option in the open microphone dropdown',
              '- aiTap: Right-arrow Next button in the fixed top navigation',
              '- aiScroll:',
              '- aiTap: Left-arrow Previous button in the fixed top navigation',
              '- aiTap: Check microphone · 3 seconds button on the Microphone step',
              '- aiTap: Right-arrow Next button in the fixed top navigation',
              '- aiScroll:',
              '- aiTap: The trigger selection dropdown currently showing fn',
              '- aiTap: Record a shortcut… option in the open trigger selection dropdown',
              '- aiTap: Right-arrow Next button in the fixed top navigation',
              '- aiTap: Left-arrow Previous button in the fixed top navigation',
            ]
          : [],
      `${file} must express visual interactions as task-level aiAct steps except for proven failures`,
    );
    assert.doesNotMatch(
      source,
      /^\s+-\s+wait:/gm,
      `${file} must wait for a visible state with aiWaitFor instead of sleeping`,
    );
  }
});

test('captures both legacy and current Midscene report step lists', async () => {
  const source = await readFile(
    new URL('../e2e/capture-report-preview.mjs', import.meta.url),
    'utf8',
  );
  assert.match(
    source,
    /\.runner-detail-step-group > button, .*\.runner-detail-step-list > button/,
  );
  assert.match(
    source,
    /\.runner-detail-step-group > button\.is-selected, .*\.runner-detail-step-list > button\.is-selected/,
  );
  assert.match(source, /waitUntil: 'domcontentloaded'/);
  assert.match(source, /timeout: 120_000/);
});

function runnerScript({
  assertionCount = 0,
  assertionAttempts,
  project,
  status = 'success',
  startedAt,
  screenshotMode = 'inline',
}) {
  const attempts =
    assertionAttempts ??
    (assertionCount
      ? [Array.from({ length: assertionCount }, () => 'success')]
      : [[status === 'success' ? 'success' : 'failed']]);
  const reportId = 'runner-report-1';
  const executions = attempts.flatMap((statuses, attemptIndex) =>
    statuses.map((stepStatus, stepIndex) => ({
      id: `execution-${attemptIndex}-${stepIndex}`,
      tasks: [
        {
          status: stepStatus === 'success' ? 'finished' : 'failed',
          type: 'Insight',
          subType: 'Assert',
          uiContext: {
            screenshot: {
              id: `screenshot-${attemptIndex}-${stepIndex}`,
              mimeType: 'image/jpeg',
              storage: 'inline',
            },
          },
          thought:
            stepStatus === 'success'
              ? `AI explanation ${attemptIndex}-${stepIndex}`
              : `AI failure explanation ${attemptIndex}-${stepIndex}`,
        },
      ],
    })),
  );
  const run = {
    startedAt,
    status,
    summary: {
      total: 1,
      passed: status === 'success' ? 1 : 0,
      failed: status === 'success' ? 0 : 1,
    },
    projects: [
      {
        name: project,
        documents: [
              {
                cases: [
                  {
                    caseId: `case-${project}`,
                    name: `${project} visual case`,
                    status,
                    attempts: attempts.map((statuses, attemptIndex) => ({
                      status: statuses.every((item) => item === 'success')
                        ? 'success'
                        : 'failed',
                      durationMs: 12000 + attemptIndex * 1000,
                      steps: statuses.map((stepStatus, stepIndex) => ({
                        id: `assert-${attemptIndex}-${stepIndex}`,
                        node: assertionCount || assertionAttempts ? 'aiAssert' : 'aiAct',
                        title: `Evidence ${attemptIndex}-${stepIndex}`,
                        status: stepStatus,
                        durationMs: 4000 + stepIndex * 500,
                        ...(stepStatus === 'failed'
                          ? {
                              error: {
                                message: `Fixture error ${attemptIndex}-${stepIndex}`,
                              },
                            }
                          : {}),
                        agentDetails: [{ reportId, executionId: `execution-${attemptIndex}-${stepIndex}` }],
                      })),
                    })),
                  },
                ],
              },
            ],
      },
    ],
  };
  return `<script type="midscene_web_dump" data-report-id="${reportId}"${
    screenshotMode === 'directory'
      ? ' data-screenshot-mode="directory"'
      : ''
  }>${JSON.stringify({ executions })}</script>
${
  screenshotMode === 'inline'
    ? attempts
        .flatMap((statuses, attemptIndex) =>
          statuses.map(
            (_stepStatus, stepIndex) =>
              `<script type="midscene-image" data-id="screenshot-${attemptIndex}-${stepIndex}">data:image/jpeg;base64,/9j/2Q==</script>`,
          ),
        )
        .join('\n')
    : ''
}
<script type="midscene_test_run_dump">${JSON.stringify(run)}</script>`;
}

function runnerWithoutScreenshot({ project, startedAt }) {
  const run = {
    startedAt,
    status: 'failed',
    summary: { total: 2, passed: 0, failed: 1, notRun: 1 },
    projects: [
      {
        name: project,
        documents: [
          {
            cases: [
              {
                caseId: `case-${project}`,
                name: `${project} damaged-agent case`,
                status: 'failed',
                attempts: [
                  {
                    status: 'failed',
                    durationMs: 480000,
                    steps: [
                      {
                        id: 'act-without-execution',
                        node: 'aiAct',
                        title: 'Timed-out action',
                        status: 'failed',
                        error: {
                          message:
                            'AI call hard timeout; no stable Agent execution reference was captured',
                        },
                      },
                    ],
                  },
                ],
              },
            ],
          },
          {
            cases: [
              {
                caseId: `not-run-${project}`,
                name: `${project} not-run case`,
                status: 'not-run',
                attempts: [],
              },
            ],
          },
        ],
      },
    ],
  };
  return `<script type="midscene_test_run_dump">${JSON.stringify(run)}</script>`;
}

const fixtureHtml = `<!doctype html><html><body>report
<script type="midscene_web_dump">{"usage":{"_midscene_call_id":"call-1","time_cost":2500,"total_tokens":120}}</script>
<script type="midscene_web_dump">{"message":"raw
control character","usage":{"_midscene_call_id":"call-2","time_cost":3500,"total_tokens":180}}</script>
${runnerScript({ project: 'ubuntu', startedAt: '2026-09-15T12:00:00Z' })}
</body></html>`;

const shellPrompts = [
  'The Omarchy system menu is open and Shutdown is readable.',
  'Exactly one row in the open system menu is highlighted.',
  'The Omarchy bar is visible on the left.',
];
const shellFixtureHtml = `<!doctype html><html><body>${shellPrompts
  .map(
    (prompt, index) => `
<script type="midscene_web_dump">${JSON.stringify({
      executions: [
        {
          tasks: [
            {
              status: 'finished',
              subType: 'Assert',
              param: { dataDemand: prompt },
              output: true,
              uiContext: { screenshot: { id: `image-${index}` } },
            },
          ],
        },
      ],
    })}</script>
<script type="midscene-image" data-id="image-${index}">data:image/jpeg;base64,/9j/2Q==</script>`,
  )
  .join('')}${runnerScript({ assertionCount: 3, project: 'omarchy-shell', startedAt: '2026-09-15T12:05:00Z' })}</body></html>`;

async function fixtureDirectory(root) {
  const reportDirectory = path.join(root, 'artifact', 'report');
  await mkdir(reportDirectory, { recursive: true });
  await writeFile(path.join(reportDirectory, 'test-run-ubuntu.html'), fixtureHtml);
  await writeFile(path.join(reportDirectory, 'agent-detail.html'), '<html>intermediate Agent report</html>');
  await writeFile(path.join(path.dirname(reportDirectory), 'report-preview.png'), 'preview');
  await writeFile(
    path.join(path.dirname(reportDirectory), 'case-preview-ubuntu-case-ubuntu.jpg'),
    'ubuntu case preview',
  );
  return path.dirname(reportDirectory);
}

async function startServer(handler) {
  const server = http.createServer(handler);
  await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
  const address = server.address();
  return {
    url: `http://127.0.0.1:${address.port}/`,
    close: () => new Promise((resolve) => server.close(resolve)),
  };
}

function options(reportDirectory, siteDirectory, pagesUrl, runId = '200') {
  return {
    'report-dir': reportDirectory,
    'site-dir': siteDirectory,
    'run-id': runId,
    'workflow-url': `https://github.com/quanru/doubao-say/actions/runs/${runId}`,
    'generated-at': '2026-09-15T12:00:00Z',
    retention: '10',
    'pages-url': pagesUrl,
    label: 'Ubuntu 22.04',
    'primary-project': 'ubuntu',
  };
}

test('simulates a first deployment when Pages returns 404', async (context) => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-first-'));
  const reportDirectory = await fixtureDirectory(root);
  const siteDirectory = path.join(root, 'site');
  const server = await startServer((_request, response) => {
    response.writeHead(404).end();
  });
  context.after(server.close);

  const manifest = await buildPagesReport({
    ...options(reportDirectory, siteDirectory, server.url),
    'auxiliary-project': 'ubuntu-polishing',
    'auxiliary-report-label': 'Polishing overlay report',
  });

  assert.equal(manifest.reports.length, 1);
  assert.equal(manifest.reports[0].runId, '200');
  assert.equal(manifest.reports[0].reportPath, 'reports/200/index.html');
  assert.equal(manifest.reports[0].successRate, 100);
  assert.equal(manifest.reports[0].averageDurationMs, 3000);
  assert.equal(manifest.reports[0].modelCallCount, 2);
  assert.equal(manifest.reports[0].tokenUsage, 300);
  const indexHtml = await readFile(
    path.join(siteDirectory, 'index.html'),
    'utf8',
  );
  assert.match(indexHtml, /Run ID/);
  assert.match(indexHtml, /Distribution/);
  assert.match(indexHtml, /Ubuntu 22\.04/);
  const runIndex = await readFile(
    path.join(siteDirectory, 'reports', '200', 'index.html'),
    'utf8',
  );
  assert.match(runIndex, /Node screenshot/);
  assert.match(runIndex, /AI response \/ error/);
  assert.match(runIndex, /ubuntu visual case/);
  assert.match(
    runIndex,
    /native-report\.html#runner-step=assert-0-0/,
  );
  assert.equal(
    await readFile(
      path.join(siteDirectory, 'reports', '200', 'native-report.html'),
      'utf8',
    ),
    fixtureHtml,
  );
  assert.deepEqual(manifest.reports[0].files, [
    'reports/200/index.html',
    'reports/200/native-report.html',
    'reports/200/report-preview.png',
    'reports/200/case-preview-ubuntu-case-ubuntu.jpg',
  ]);
  assert.equal(manifest.reports[0].entries.length, 1);
  assert.equal(
    await readFile(
      path.join(siteDirectory, 'reports', '200', 'report-preview.png'),
      'utf8',
    ),
    'preview',
  );
});

test('publishes 1.13.0 directory-mode screenshots next to the native report', async (context) => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-directory-'));
  const reportRoot = path.join(root, 'artifact');
  const htmlDirectory = path.join(
    reportRoot,
    'report',
    'midscene-e2e-20260921045854-d9ab4bd0',
  );
  await mkdir(path.join(htmlDirectory, 'screenshots'), {
    recursive: true,
  });
  const html = runnerScript({
    project: 'ubuntu',
    startedAt: '2026-09-15T12:00:00Z',
    screenshotMode: 'directory',
  });
  await writeFile(path.join(htmlDirectory, 'index.html'), html);
  await writeFile(
    path.join(htmlDirectory, 'screenshots', 'screenshot-0-0.jpeg'),
    'directory screenshot bytes',
  );
  // 1.13.0 also writes standalone computer-*.html agent reports without a
  // runner dump; they must not be treated as undeclared report projects.
  await writeFile(
    path.join(reportRoot, 'report', 'computer-2026-09-21_04-58-55.html'),
    '<html>standalone computer agent report</html>',
  );
  await writeFile(
    path.join(reportRoot, 'report-preview.png'),
    'preview',
  );
  await writeFile(
    path.join(reportRoot, 'case-preview-ubuntu-case-ubuntu.jpg'),
    'ubuntu case preview',
  );
  const siteDirectory = path.join(root, 'site');
  const server = await startServer((_request, response) => {
    response.writeHead(404).end();
  });
  context.after(server.close);

  const manifest = await buildPagesReport(
    options(reportRoot, siteDirectory, server.url),
  );

  assert.equal(
    await readFile(
      path.join(
        siteDirectory,
        'reports',
        '200',
        'screenshots',
        'screenshot-0-0.jpeg',
      ),
      'utf8',
    ),
    'directory screenshot bytes',
  );
  assert.ok(
    manifest.reports[0].files.includes(
      'reports/200/screenshots/screenshot-0-0.jpeg',
    ),
  );
  assert.deepEqual(manifest.reports[0].files, [
    'reports/200/index.html',
    'reports/200/native-report.html',
    'reports/200/report-preview.png',
    'reports/200/case-preview-ubuntu-case-ubuntu.jpg',
    'reports/200/screenshots/screenshot-0-0.jpeg',
  ]);
});

test('combines independently executed shards into one report table', async (context) => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-shards-'));
  const reportDirectory = path.join(root, 'bundle');
  for (const project of ['ubuntu-shard-1', 'ubuntu-shard-2']) {
    const shardDirectory = path.join(reportDirectory, project);
    const htmlDirectory = path.join(shardDirectory, 'report');
    await mkdir(htmlDirectory, { recursive: true });
    await writeFile(
      path.join(htmlDirectory, `test-run-${project}.html`),
      runnerScript({ project, startedAt: '2026-09-15T12:00:00Z' }),
    );
    await writeFile(
      path.join(shardDirectory, `report-preview-${project}.png`),
      `${project} preview`,
    );
    await writeFile(
      path.join(
        shardDirectory,
        `case-preview-${project}-case-${project}.jpg`,
      ),
      `${project} case preview`,
    );
  }
  const siteDirectory = path.join(root, 'site');
  const server = await startServer((_request, response) =>
    response.writeHead(404).end(),
  );
  context.after(server.close);

  const manifest = await buildPagesReport({
    ...options(reportDirectory, siteDirectory, server.url),
    'report-groups': JSON.stringify([
      {
        role: 'primary',
        label: 'Doubao Say',
        projects: ['ubuntu-shard-1', 'ubuntu-shard-2'],
      },
    ]),
  });

  const [entry] = manifest.reports[0].entries;
  assert.deepEqual(entry.projects, ['ubuntu-shard-1', 'ubuntu-shard-2']);
  assert.equal(entry.cases.length, 2);
  assert.equal(entry.scenarios.total, 2);
  assert.notEqual(entry.cases[0].reportPath, entry.cases[1].reportPath);
  const runIndex = await readFile(
    path.join(siteDirectory, 'reports', '200', 'index.html'),
    'utf8',
  );
  assert.match(
    runIndex,
    /native-report-ubuntu-shard-1\.html#runner-step=assert-0-0/,
  );
  assert.match(
    runIndex,
    /native-report-ubuntu-shard-2\.html#runner-step=assert-0-0/,
  );
  const summary = renderReportSummary({
    manifest,
    pagesUrl: 'https://example.test/doubao-say/',
    producerResult: 'success',
    runId: '200',
    summaryTitle: 'Ubuntu',
  });
  assert.match(
    summary,
    /native-report-ubuntu-shard-1\.html#runner-step=assert-0-0/,
  );
  assert.match(
    summary,
    /native-report-ubuntu-shard-2\.html#runner-step=assert-0-0/,
  );
  assert.match(summary, /\| ubuntu-shard-1 \| \[ubuntu-shard-1 visual case\]/);
  assert.match(summary, /\| ubuntu-shard-2 \| \[ubuntu-shard-2 visual case\]/);
});

test('keeps a complete Markdown table when a shard produces no native report', async (context) => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-partial-shards-'));
  const reportDirectory = path.join(root, 'reports');
  const shardDirectory = path.join(reportDirectory, 'ubuntu-shard-1');
  await mkdir(path.join(shardDirectory, 'report'), { recursive: true });
  await writeFile(
    path.join(shardDirectory, 'report', 'test-run-ubuntu-shard-1.html'),
    runnerScript({
      project: 'ubuntu-shard-1',
      startedAt: '2026-09-15T12:00:00Z',
    }),
  );
  await writeFile(
    path.join(shardDirectory, 'report-preview-ubuntu-shard-1.png'),
    'preview',
  );
  await writeFile(
    path.join(
      shardDirectory,
      'case-preview-ubuntu-shard-1-case-ubuntu-shard-1.jpg',
    ),
    'case preview',
  );
  await writeFile(
    path.join(shardDirectory, 'ci-shard-status-ubuntu-shard-2.json'),
    JSON.stringify({
      project: 'ubuntu-shard-2',
      result: 'failure',
      stage: 'workflow setup before test execution',
    }),
  );
  const siteDirectory = path.join(root, 'site');
  const server = await startServer((_request, response) =>
    response.writeHead(404).end(),
  );
  context.after(server.close);

  const manifest = await buildPagesReport({
    ...options(reportDirectory, siteDirectory, server.url),
    'report-groups': JSON.stringify([
      {
        role: 'primary',
        label: 'Doubao Say',
        projects: ['ubuntu-shard-1', 'ubuntu-shard-2'],
      },
    ]),
  });
  const [entry] = manifest.reports[0].entries;
  assert.equal(entry.cases.length, 2);
  assert.equal(entry.cases[1].selection, 'workflow-failure');
  assert.match(entry.cases[1].description, /workflow setup/);
  await readFile(
    path.join(
      siteDirectory,
      'reports',
      '200',
      'shard-failure-ubuntu-shard-2.svg',
    ),
  );

  const summary = renderReportSummary({
    manifest,
    pagesUrl: 'https://example.test/doubao-say/',
    producerResult: 'failure',
    runId: '200',
    summaryTitle: 'Ubuntu',
  });
  assert.match(summary, /ubuntu-shard-1 visual case/);
  assert.match(summary, /ubuntu-shard-2 CI shard/);
  assert.match(summary, /CI failure before node capture/);
  assert.match(summary, /workflow setup before test execution/);
  assert.doesNotMatch(summary, /runner-step=.*infrastructure-ubuntu-shard-2/);
});

test('uses a failure row when node evidence extraction fails', async (context) => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-capture-failed-'));
  const reportDirectory = await fixtureDirectory(root);
  await writeFile(
    path.join(reportDirectory, 'ci-shard-status-ubuntu.json'),
    JSON.stringify({
      project: 'ubuntu',
      result: 'failure',
      stage: 'report evidence capture',
      testOutcome: 'success',
      captureOutcome: 'failure',
    }),
  );
  const siteDirectory = path.join(root, 'site');
  const server = await startServer((_request, response) =>
    response.writeHead(404).end(),
  );
  context.after(server.close);

  const manifest = await buildPagesReport({
    ...options(reportDirectory, siteDirectory, server.url),
    'report-groups': JSON.stringify([
      { role: 'primary', label: 'Doubao Say', projects: ['ubuntu'] },
    ]),
  });
  const [testCase] = manifest.reports[0].entries[0].cases;
  assert.equal(testCase.selection, 'workflow-failure');
  assert.match(testCase.description, /report evidence capture/);
});

test('publishes the Doubao Say report as the primary CI entrance', async (context) => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-omarchy-'));
  const reportDirectory = await fixtureDirectory(root);
  await writeFile(
    path.join(reportDirectory, 'report', 'test-run-shell.html'),
    shellFixtureHtml,
  );
  await writeFile(path.join(reportDirectory, 'report-preview.png'), 'preview');
  await writeFile(
    path.join(reportDirectory, 'auxiliary-report-preview.png'),
    'auxiliary preview',
  );
  await writeFile(
    path.join(reportDirectory, 'case-preview-omarchy-shell-case-omarchy-shell.jpg'),
    'shell case preview',
  );
  const siteDirectory = path.join(root, 'site');
  const server = await startServer((_request, response) =>
    response.writeHead(404).end(),
  );
  context.after(server.close);

  const manifest = await buildPagesReport({
    ...options(reportDirectory, siteDirectory, server.url),
    'auxiliary-project': 'omarchy-shell',
    'auxiliary-report-label': 'Omarchy visual checks',
  });

  assert.equal(manifest.reports[0].testCount, 2);
  assert.equal(manifest.reports[0].successRate, 100);
  assert.deepEqual(manifest.reports[0].entries, [
    {
      role: 'primary',
      project: 'ubuntu',
      label: 'Doubao Say',
      status: 'success',
      previewStep: 'last',
      reportPath: 'reports/200/native-report.html',
      previewPath: 'reports/200/report-preview.png',
      scenarios: { passed: 1, total: 1 },
      assertions: { passed: 0, total: 0 },
      cases: [
        {
          caseId: 'case-ubuntu',
          name: 'ubuntu visual case',
          status: 'success',
          durationMs: 12000,
          stepId: 'assert-0-0',
          selection: 'last-screenshot',
          description: 'AI explanation 0-0',
          descriptionKind: 'ai',
          previewPath: 'reports/200/case-preview-ubuntu-case-ubuntu.jpg',
        },
      ],
    },
    {
      role: 'auxiliary',
      project: 'omarchy-shell',
      label: 'Omarchy visual checks',
      status: 'success',
      previewStep: 'last',
      reportPath: 'reports/200/auxiliary-report.html',
      previewPath: 'reports/200/auxiliary-report-preview.png',
      scenarios: { passed: 1, total: 1 },
      assertions: { passed: 3, total: 3 },
      cases: [
        {
          caseId: 'case-omarchy-shell',
          name: 'omarchy-shell visual case',
          status: 'success',
          durationMs: 12000,
          stepId: 'assert-0-2',
          selection: 'last-screenshot',
          description: 'AI explanation 0-2',
          descriptionKind: 'ai',
          previewPath:
            'reports/200/case-preview-omarchy-shell-case-omarchy-shell.jpg',
        },
      ],
    },
  ]);
  assert.deepEqual(manifest.reports[0].files, [
    'reports/200/index.html',
    'reports/200/native-report.html',
    'reports/200/auxiliary-report.html',
    'reports/200/report-preview.png',
    'reports/200/auxiliary-report-preview.png',
    'reports/200/case-preview-ubuntu-case-ubuntu.jpg',
    'reports/200/case-preview-omarchy-shell-case-omarchy-shell.jpg',
  ]);
  const publishedIndex = await readFile(
    path.join(siteDirectory, 'reports', '200', 'index.html'),
    'utf8',
  );
  assert.match(publishedIndex, /Doubao Say/);
  assert.match(publishedIndex, /Omarchy visual checks/);
  assert.match(publishedIndex, /AI response \/ error/);
  assert.equal(
    await readFile(
      path.join(siteDirectory, 'reports', '200', 'native-report.html'),
      'utf8',
    ),
    fixtureHtml,
  );
  assert.equal(
    await readFile(
      path.join(siteDirectory, 'reports', '200', 'auxiliary-report.html'),
      'utf8',
    ),
    shellFixtureHtml,
  );
  assert.equal(
    await readFile(
      path.join(siteDirectory, 'reports', '200', 'report-preview.png'),
      'utf8',
    ),
    'preview',
  );
  assert.equal(
    await readFile(
      path.join(
        siteDirectory,
        'reports',
        '200',
        'auxiliary-report-preview.png',
      ),
      'utf8',
    ),
    'auxiliary preview',
  );
});

test('reports visual assertions from the final retry attempt only', async (context) => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-retry-'));
  const reportDirectory = path.join(root, 'artifact', 'report');
  await mkdir(reportDirectory, { recursive: true });
  const retriedReport = `<!doctype html><html><body>${runnerScript({
    assertionAttempts: [['failed'], ['success', 'success']],
    project: 'ubuntu',
    startedAt: '2026-09-15T12:00:00Z',
  })}</body></html>`;
  await writeFile(
    path.join(reportDirectory, 'test-run-retried.html'),
    retriedReport,
  );
  await writeFile(
    path.join(path.dirname(reportDirectory), 'report-preview.png'),
    'preview',
  );
  await writeFile(
    path.join(
      path.dirname(reportDirectory),
      'case-preview-ubuntu-case-ubuntu.jpg',
    ),
    'retry preview',
  );
  const siteDirectory = path.join(root, 'site');
  const server = await startServer((_request, response) =>
    response.writeHead(404).end(),
  );
  context.after(server.close);

  const manifest = await buildPagesReport(
    options(path.dirname(reportDirectory), siteDirectory, server.url),
  );

  assert.deepEqual(manifest.reports[0].entries[0].assertions, {
    passed: 2,
    total: 2,
  });
});

test('publishes a failed shell report with its result in history', async (context) => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-failed-'));
  const reportDirectory = await fixtureDirectory(root);
  await writeFile(
    path.join(reportDirectory, 'report', 'test-run-shell.html'),
    shellFixtureHtml
      .replace('"output":true', '"output":false')
      .replace(
        '"status":"success","summary":{"total":1,"passed":1,"failed":0}',
        '"status":"failed","summary":{"total":1,"passed":0,"failed":1}',
      ),
  );
  await writeFile(path.join(reportDirectory, 'report-preview.png'), 'failed');
  await writeFile(
    path.join(reportDirectory, 'auxiliary-report-preview.png'),
    'auxiliary failed',
  );
  await writeFile(
    path.join(
      reportDirectory,
      'case-preview-omarchy-shell-case-omarchy-shell.jpg',
    ),
    'shell failure preview',
  );
  const siteDirectory = path.join(root, 'site');
  const server = await startServer((_request, response) =>
    response.writeHead(404).end(),
  );
  context.after(server.close);

  const manifest = await buildPagesReport(
    {
      ...options(reportDirectory, siteDirectory, server.url),
      'auxiliary-project': 'omarchy-shell',
      'auxiliary-report-label': 'Omarchy visual checks',
    },
  );

  assert.equal(manifest.reports[0].successRate, 50);
  assert.equal(
    await readFile(
      path.join(siteDirectory, 'reports', '200', 'report-preview.png'),
      'utf8',
    ),
    'failed',
  );
});

test('keeps only the latest failed retry for an Omarchy project', async (context) => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-retries-'));
  const reportDirectory = path.join(root, 'artifact');
  const reportPath = path.join(reportDirectory, 'report');
  await mkdir(reportPath, { recursive: true });
  const firstFailure = `<!doctype html><html><body>first failure${runnerScript({ project: 'omarchy-onboarding', status: 'failed', startedAt: '2026-09-15T13:00:00Z' })}</body></html>`;
  const finalFailure = `<!doctype html><html><body>final failure${runnerScript({ project: 'omarchy-onboarding', status: 'failed', startedAt: '2026-09-15T14:00:00Z' })}</body></html>`;
  await writeFile(path.join(reportPath, 'test-run-retry-1.html'), firstFailure);
  await writeFile(path.join(reportPath, 'test-run-retry-2.html'), finalFailure);
  await writeFile(path.join(reportDirectory, 'report-preview.png'), 'failed');
  await writeFile(
    path.join(
      reportDirectory,
      'case-preview-omarchy-onboarding-case-omarchy-onboarding.jpg',
    ),
    'failure preview',
  );
  const siteDirectory = path.join(root, 'site');
  const server = await startServer((_request, response) =>
    response.writeHead(404).end(),
  );
  context.after(server.close);

  const manifest = await buildPagesReport({
    ...options(reportDirectory, siteDirectory, server.url),
    label: 'Omarchy 4.0.3',
    'primary-project': 'omarchy-onboarding',
  });

  assert.equal(manifest.reports[0].successRate, 0);
  assert.equal(manifest.reports[0].testCount, 1);
  assert.deepEqual(manifest.reports[0].files, [
    'reports/200/index.html',
    'reports/200/native-report.html',
    'reports/200/report-preview.png',
    'reports/200/case-preview-omarchy-onboarding-case-omarchy-onboarding.jpg',
  ]);
  const failedRunIndex = await readFile(
    path.join(siteDirectory, 'reports', '200', 'index.html'),
    'utf8',
  );
  assert.match(failedRunIndex, /❌ Failed/);
  assert.match(failedRunIndex, /Error:/);
  assert.equal(
    await readFile(
      path.join(siteDirectory, 'reports', '200', 'native-report.html'),
      'utf8',
    ),
    finalFailure,
  );
});

test('restores retained history before adding the new run', async (context) => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-history-'));
  const reportDirectory = await fixtureDirectory(root);
  const siteDirectory = path.join(root, 'site');
  const oldReport = '<!doctype html><html><body>old report</body></html>';
  const oldEntry = {
    runId: '100',
    generatedAt: '2026-09-14T12:00:00.000Z',
    label: 'Omarchy 4.0.3',
    successRate: 100,
    testCount: 1,
    averageDurationMs: 4000,
    modelCallCount: 1,
    tokenUsage: null,
    workflowUrl: 'https://github.com/quanru/doubao-say/actions/runs/100',
    reportPath: 'reports/100/index.html',
  };
  let restoreAttempts = 0;
  const server = await startServer((request, response) => {
    if (request.url === '/reports/manifest.json') {
      response.setHeader('content-type', 'application/json');
      response.end(JSON.stringify({ version: 1, reports: [oldEntry] }));
    } else if (request.url === '/reports/100/index.html') {
      restoreAttempts++;
      if (restoreAttempts === 1) return response.writeHead(503).end();
      response.setHeader('content-type', 'text/html; charset=utf-8');
      response.end(oldReport);
    } else {
      response.writeHead(404).end();
    }
  });
  context.after(server.close);

  const manifest = await buildPagesReport(
    options(reportDirectory, siteDirectory, server.url),
  );

  assert.deepEqual(
    manifest.reports.map((report) => report.runId),
    ['200', '100'],
  );
  assert.equal(restoreAttempts, 2);
  assert.equal(
    await readFile(
      path.join(siteDirectory, 'reports', '100', 'index.html'),
      'utf8',
    ),
    oldReport,
  );
  const writtenManifest = JSON.parse(
    await readFile(
      path.join(siteDirectory, 'reports', 'manifest.json'),
      'utf8',
    ),
  );
  assert.equal(writtenManifest.version, 7);
  assert.equal(writtenManifest.reports[0].reportPath, 'reports/200/index.html');
  assert.equal(
    writtenManifest.reports[0].workflowUrl,
    'https://github.com/quanru/doubao-say/actions/runs/200',
  );
});

test('retries rate-limited history files before failing the report build', async (context) => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-rate-limit-'));
  const reportDirectory = await fixtureDirectory(root);
  const siteDirectory = path.join(root, 'site');
  const file = 'reports/100/index.html';
  let attempts = 0;
  const server = await startServer((request, response) => {
    if (request.url === '/reports/manifest.json') {
      response.setHeader('content-type', 'application/json');
      response.end(JSON.stringify({
        version: 1,
        reports: [{
          runId: '100',
          generatedAt: '2026-09-14T12:00:00.000Z',
          label: 'Omarchy 4.0.3',
          successRate: 100,
          testCount: 1,
          averageDurationMs: 4000,
          modelCallCount: 1,
          tokenUsage: null,
          workflowUrl: 'https://github.com/quanru/doubao-say/actions/runs/100',
          reportPath: file,
        }],
      }));
    } else if (request.url === `/${file}`) {
      attempts++;
      if (attempts === 1) {
        response.writeHead(429, { 'retry-after': '0' }).end();
      } else {
        response.writeHead(200, { 'content-type': 'text/html' });
        response.end('<html>restored</html>');
      }
    } else {
      response.writeHead(404).end();
    }
  });
  context.after(server.close);

  await buildPagesReport(options(reportDirectory, siteDirectory, server.url));
  assert.equal(attempts, 2);
  assert.equal(await readFile(path.join(siteDirectory, file), 'utf8'), '<html>restored</html>');
});

test('restores large report histories with bounded parallel downloads', async (context) => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-parallel-history-'));
  const reportDirectory = await fixtureDirectory(root);
  const siteDirectory = path.join(root, 'site');
  const files = [
    'reports/100/index.html',
    ...Array.from({ length: 15 }, (_, index) =>
      `reports/100/screenshots/shot-${index}.jpeg`,
    ),
  ];
  let active = 0;
  let peak = 0;
  const server = await startServer((request, response) => {
    if (request.url === '/reports/manifest.json') {
      response.setHeader('content-type', 'application/json');
      response.end(JSON.stringify({
        version: 7,
        reports: [{
          runId: '100',
          generatedAt: '2026-09-14T12:00:00.000Z',
          label: 'Omarchy 4.0.3',
          successRate: 100,
          testCount: 1,
          averageDurationMs: 4000,
          modelCallCount: 1,
          tokenUsage: null,
          workflowUrl: 'https://github.com/quanru/doubao-say/actions/runs/100',
          reportPath: 'reports/100/index.html',
          files,
        }],
      }));
      return;
    }
    active++;
    peak = Math.max(peak, active);
    setTimeout(() => {
      response.setHeader(
        'content-type',
        request.url.endsWith('.html') ? 'text/html' : 'image/jpeg',
      );
      response.end(request.url);
      active--;
    }, 30);
  });
  context.after(server.close);

  await buildPagesReport(options(reportDirectory, siteDirectory, server.url));

  assert.ok(peak > 1, `expected parallel requests, saw ${peak}`);
  assert.ok(peak <= 8, `expected at most 8 requests, saw ${peak}`);
  assert.equal(
    await readFile(path.join(siteDirectory, files.at(-1)), 'utf8'),
    `/${files.at(-1)}`,
  );
});

test('restores version 4 history created before node text evidence', async (context) => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-v4-history-'));
  const reportDirectory = await fixtureDirectory(root);
  const siteDirectory = path.join(root, 'site');
  const files = [
    'reports/100/index.html',
    'reports/100/report-preview.png',
    'reports/100/case-preview-ubuntu-old-case.jpg',
  ];
  const oldEntry = {
    runId: '100',
    generatedAt: '2026-09-14T12:00:00.000Z',
    label: 'Ubuntu 22.04',
    successRate: 100,
    testCount: 1,
    modelCallCount: 1,
    averageDurationMs: 4000,
    tokenUsage: 100,
    workflowUrl: 'https://github.com/quanru/doubao-say/actions/runs/100',
    reportPath: files[0],
    files,
    entries: [{
      role: 'primary', project: 'ubuntu', label: 'Doubao Say',
      status: 'success', previewStep: 'last', reportPath: files[0],
      previewPath: files[1], scenarios: { passed: 1, total: 1 },
      assertions: { passed: 1, total: 1 },
      cases: [{
        caseId: 'old-case', name: 'Old visual case', status: 'success',
        stepId: 'old-step', selection: 'last-screenshot', previewPath: files[2],
      }],
    }],
  };
  const server = await startServer((request, response) => {
    if (request.url === '/reports/manifest.json') {
      response.setHeader('content-type', 'application/json');
      response.end(JSON.stringify({ version: 4, reports: [oldEntry] }));
      return;
    }
    const file = files.find((candidate) => `/${candidate}` === request.url);
    if (!file) return response.writeHead(404).end();
    response.setHeader(
      'content-type',
      file.endsWith('.html') ? 'text/html; charset=utf-8' : 'image/png',
    );
    response.end(file);
  });
  context.after(server.close);

  const manifest = await buildPagesReport(
    options(reportDirectory, siteDirectory, server.url),
  );
  assert.deepEqual(manifest.reports.map((report) => report.runId), ['200', '100']);
  assert.equal(manifest.reports[1].entries[0].cases[0].description, undefined);
});

test('restores version 6 history with directory-mode screenshots', async (context) => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-v6-history-'));
  const reportDirectory = await fixtureDirectory(root);
  const siteDirectory = path.join(root, 'site');
  const files = [
    'reports/100/index.html',
    'reports/100/native-report.html',
    'reports/100/report-preview.png',
    'reports/100/case-preview-ubuntu-old-case.jpg',
    'reports/100/screenshots/node-shot-1.jpeg',
  ];
  const oldEntry = {
    runId: '100',
    generatedAt: '2026-09-20T12:00:00.000Z',
    label: 'Ubuntu 22.04',
    successRate: 100,
    testCount: 1,
    modelCallCount: 1,
    averageDurationMs: 4000,
    tokenUsage: 100,
    workflowUrl: 'https://github.com/quanru/doubao-say/actions/runs/100',
    reportPath: files[0],
    files,
    entries: [
      {
        role: 'primary',
        project: 'ubuntu',
        label: 'Doubao Say',
        status: 'success',
        previewStep: 'last',
        reportPath: files[1],
        previewPath: files[2],
        scenarios: { passed: 1, total: 1 },
        assertions: { passed: 1, total: 1 },
        cases: [
          {
            caseId: 'old-case',
            name: 'Old visual case',
            status: 'success',
            stepId: 'old-step',
            selection: 'last-screenshot',
            description: 'Old AI explanation',
            descriptionKind: 'ai',
            previewPath: files[3],
          },
        ],
      },
    ],
  };
  const server = await startServer((request, response) => {
    if (request.url === '/reports/manifest.json') {
      response.setHeader('content-type', 'application/json');
      response.end(JSON.stringify({ version: 6, reports: [oldEntry] }));
      return;
    }
    const file = files.find((candidate) => `/${candidate}` === request.url);
    if (!file) return response.writeHead(404).end();
    response.setHeader(
      'content-type',
      file.endsWith('.html') ? 'text/html; charset=utf-8' : 'image/jpeg',
    );
    response.end(file);
  });
  context.after(server.close);

  const manifest = await buildPagesReport(
    options(reportDirectory, siteDirectory, server.url),
  );
  assert.deepEqual(manifest.reports.map((report) => report.runId), ['200', '100']);
  assert.equal(
    await readFile(path.join(siteDirectory, files[4]), 'utf8'),
    files[4],
  );
});

test('stops when a manifest exists but an old report cannot be restored', async (context) => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-failure-'));
  const reportDirectory = await fixtureDirectory(root);
  const siteDirectory = path.join(root, 'site');
  const server = await startServer((request, response) => {
    if (request.url === '/reports/manifest.json') {
      response.setHeader('content-type', 'application/json');
      response.end(
        JSON.stringify({
          version: 1,
          reports: [
            {
              runId: '100',
              workflowUrl:
                'https://github.com/quanru/doubao-say/actions/runs/100',
              reportPath: 'reports/100/index.html',
            },
          ],
        }),
      );
    } else {
      response.writeHead(503).end();
    }
  });
  context.after(server.close);

  await assert.rejects(
    buildPagesReport(options(reportDirectory, siteDirectory, server.url)),
    /Cannot restore report for run 100: HTTP 503/,
  );
});

test('rejects retention outside the supported range', async (context) => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-retention-'));
  const reportDirectory = await fixtureDirectory(root);
  const server = await startServer((_request, response) => {
    response.writeHead(404).end();
  });
  context.after(server.close);
  const invalid = options(reportDirectory, path.join(root, 'site'), server.url);
  invalid.retention = '51';

  await assert.rejects(buildPagesReport(invalid), /integer from 1 to 50/);
});

test('rejects an undeclared report project', async (context) => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-project-'));
  const reportDirectory = await fixtureDirectory(root);
  await writeFile(
    path.join(reportDirectory, 'report', 'test-run-shell.html'),
    shellFixtureHtml,
  );
  const server = await startServer((_request, response) =>
    response.writeHead(404).end(),
  );
  context.after(server.close);

  await assert.rejects(
    buildPagesReport(
      options(reportDirectory, path.join(root, 'site'), server.url),
    ),
    /Unexpected Midscene Test project\(s\): omarchy-shell/,
  );
});

test('renders Rome-style passed-case appendix with linked screenshots', () => {
  const manifest = {
    reports: [
      {
        runId: '200',
        entries: [
          {
            role: 'primary',
            project: 'ubuntu',
            label: 'Doubao Say',
            status: 'success',
            previewStep: 'last',
            reportPath: 'reports/200/index.html',
            previewPath: 'reports/200/report-preview.png',
            scenarios: { passed: 6, total: 6 },
            assertions: { passed: 5, total: 5 },
            cases: [
              {
                caseId: 'case-ubuntu',
                name: 'Launch and finish onboarding',
                status: 'success',
                durationMs: 73000,
                stepId: 'case-ubuntu:steps:8',
                selection: 'last-screenshot',
                description: 'The setup completion state is visible.',
                descriptionKind: 'ai',
                previewPath:
                  'reports/200/case-preview-ubuntu-case-ubuntu.jpg',
              },
            ],
          },
          {
            role: 'auxiliary',
            project: 'ubuntu-polishing',
            label: 'Polishing overlay report',
            status: 'success',
            previewStep: 'last',
            reportPath: 'reports/200/auxiliary-report.html',
            previewPath: 'reports/200/auxiliary-report-preview.png',
            scenarios: { passed: 1, total: 1 },
            assertions: { passed: 8, total: 8 },
            cases: [
              {
                caseId: 'case-polishing',
                name: 'Polish selected text',
                status: 'success',
                durationMs: 52000,
                stepId: 'case-polishing:steps:5',
                selection: 'last-screenshot',
                description: 'The polished text is visible.',
                descriptionKind: 'ai',
                previewPath:
                  'reports/200/case-preview-ubuntu-polishing-case-polishing.jpg',
              },
            ],
          },
        ],
      },
    ],
  };
  const summary = renderReportSummary({
    manifest,
    pagesUrl: 'https://example.test/doubao-say/',
    producerResult: 'success',
    runId: '200',
    summaryTitle: 'Ubuntu',
  });

  assert.match(summary, /Ubuntu × Midscene · passed/);
  assert.match(summary, /\*\*✅ 0 need attention · 2 passed\*\*/);
  assert.match(summary, /🎉 All 2 cases passed\./);
  assert.doesNotMatch(summary, /### Needs attention/);
  assert.match(
    summary,
    /index\.html#runner-step=case-ubuntu%3Asteps%3A8/,
  );
  assert.match(summary, /\| Shard \| Case \| Screenshot \| Status \| Duration \|/);
  assert.match(summary, /\| ubuntu \| \[Launch and finish onboarding\].*\| ✅ Passed \| 1m13s \|/);
  assert.match(summary, /\| ubuntu-polishing \| \[Polish selected text\].*\| ✅ Passed \| 52s \|/);
  assert.match(
    summary,
    /<img src="[^"]*case-preview-ubuntu-case-ubuntu\.jpg"[^>]*width="160">/,
  );
  assert.match(summary, /<summary>Appendix: passed cases \(2\)<\/summary>/);
  assert.match(summary, /Open the published HTML report/);
});

test('renders singular failure copy for one available report', () => {
  const manifest = {
    reports: [
      {
        runId: '200',
        entries: [
          {
            role: 'primary',
            project: 'ubuntu',
            label: 'Doubao Say',
            status: 'failed',
            previewStep: 'last-error',
            reportPath: 'reports/200/index.html',
            previewPath: 'reports/200/report-preview.png',
            scenarios: { passed: 0, total: 1 },
            assertions: { passed: 1, total: 2 },
            cases: [
              {
                caseId: 'case-fail',
                name: 'Broken flow',
                status: 'failed',
                stepId: 'case-fail:steps:2',
          selection: 'first-failing-screenshot',
                description: 'Node "aiAssert" failed: expected state missing.',
                descriptionKind: 'error',
                previewPath:
                  'reports/200/case-preview-ubuntu-case-fail.jpg',
              },
            ],
          },
        ],
      },
    ],
  };
  const summary = renderReportSummary({
    manifest,
    pagesUrl: 'https://example.test/doubao-say',
    producerResult: 'failure',
    runId: '200',
    summaryTitle: 'Ubuntu',
  });

  assert.match(summary, /Ubuntu × Midscene · failure captured/);
  assert.match(summary, /\*\*1 need attention · 0 passed\*\*/);
  assert.match(summary, /### Needs attention/);
  assert.match(summary, /\| Shard \| Case \| Screenshot \| Status \/ reason \| Duration \|/);
  assert.match(summary, /\| ubuntu \| \[Broken flow\]/);
  assert.match(summary, /<img src="[^"]*case-preview-ubuntu-case-fail\.jpg"[^>]*width="160">/);
  assert.match(
    summary,
    /Error: Node "aiAssert" failed: expected state missing\./,
  );
  assert.match(
    summary,
    /index\.html#runner-step=case-fail%3Asteps%3A2/,
  );
  assert.match(summary, /case-preview-ubuntu-case-fail\.jpg/);
  assert.match(summary, /<summary>Appendix: passed cases \(0\)<\/summary>/);
});

test('selects the last screenshot for success and first failed screenshot for failure', async () => {
  const run = {
    projects: [
      {
        name: 'ubuntu',
        documents: [
          {
            cases: [
              {
                caseId: 'passed-case',
                name: 'Passed case',
                status: 'success',
                attempts: [
                  {
                    status: 'success',
                    steps: [
                      { id: 'pass-1', status: 'success', agentDetails: [{}] },
                      { id: 'pass-2', status: 'success' },
                    ],
                  },
                ],
              },
              {
                caseId: 'failed-case',
                name: 'Failed case',
                status: 'failed',
                attempts: [
                  {
                    status: 'failed',
                    steps: [
                      { id: 'fail-1', status: 'failed', agentDetails: [{}] },
                      { id: 'fail-2', status: 'failed', agentDetails: [{}] },
                    ],
                  },
                ],
              },
            ],
          },
        ],
      },
    ],
  };

  assert.deepEqual(
    (
      await reportCases(run, 'ubuntu')
    ).map(({ caseId, stepId, selection }) => ({
      caseId,
      stepId,
      selection,
    })),
    [
      {
        caseId: 'passed-case',
        stepId: 'pass-1',
        selection: 'last-screenshot',
      },
      {
        caseId: 'failed-case',
        stepId: 'fail-1',
        selection: 'first-failing-screenshot',
      },
    ],
  );
});

test('pairs an original node screenshot with its AI text or error', async () => {
  const successHtml = runnerScript({
    project: 'ubuntu',
    startedAt: '2026-09-15T12:00:00Z',
  });
  const success = (
    await reportCases(testRunDump(successHtml), 'ubuntu', {
      reportHtml: successHtml,
    })
  )[0];
  assert.equal(success.descriptionKind, 'ai');
  assert.equal(success.description, 'AI explanation 0-0');
  assert.deepEqual(success.screenshot.bytes, Buffer.from('/9j/2Q==', 'base64'));

  const failureHtml = runnerScript({
    project: 'ubuntu',
    startedAt: '2026-09-15T12:00:00Z',
    status: 'failed',
  });
  const failure = (
    await reportCases(testRunDump(failureHtml), 'ubuntu', {
      reportHtml: failureHtml,
    })
  )[0];
  assert.equal(failure.descriptionKind, 'error');
  assert.equal(failure.description, 'Fixture error 0-0');
  assert.deepEqual(failure.screenshot.bytes, Buffer.from('/9j/2Q==', 'base64'));
});

test('recovers a sole Agent screenshot from an unlinked custom visual step', async () => {
  const html = runnerScript({
    project: 'ubuntu',
    startedAt: '2026-09-26T07:32:45Z',
  });
  const run = testRunDump(html);
  const step = run.projects[0].documents[0].cases[0].attempts[0].steps[0];
  step.node = 'review.assertConfiguredVisual';
  delete step.agentDetails;

  const [testCase] = await reportCases(run, 'ubuntu', { reportHtml: html });
  assert.equal(testCase.selection, 'last-screenshot');
  assert.equal(testCase.descriptionKind, 'ai');
  assert.equal(testCase.description, 'AI explanation 0-0');
  assert.deepEqual(testCase.screenshot.bytes, Buffer.from('/9j/2Q==', 'base64'));
});

test('matches unlinked custom-step screenshots by execution time', async () => {
  const html = runnerScript({
    assertionCount: 2,
    project: 'ubuntu',
    startedAt: '2026-09-26T07:32:45Z',
  });
  const run = testRunDump(html);
  const steps = run.projects[0].documents[0].cases[0].attempts[0].steps;
  for (const [index, step] of steps.entries()) {
    step.node = 'review.assertConfiguredVisual';
    step.startedAt = `2026-09-26T07:32:${index ? '55' : '45'}.000Z`;
    step.endedAt = `2026-09-26T07:33:${index ? '05' : '00'}.000Z`;
    delete step.agentDetails;
  }
  const dumpMatch = html.match(/<script type="midscene_web_dump"[^>]*>(\{[\s\S]*?)<\/script>/);
  const dump = JSON.parse(dumpMatch[1]);
  dump.executions[0].logTime = Date.parse('2026-09-26T07:32:47.000Z');
  dump.executions[1].logTime = Date.parse('2026-09-26T07:32:57.000Z');
  const reportHtml = html.replace(dumpMatch[1], JSON.stringify(dump));

  const [testCase] = await reportCases(run, 'ubuntu', { reportHtml });
  assert.equal(testCase.stepId, steps[1].id);
  assert.equal(testCase.selection, 'last-screenshot');
  assert.equal(testCase.description, 'AI explanation 0-1');
});

test('keeps an ambiguous custom-step success in the table without inventing a screenshot', async (context) => {
  const html = runnerScript({
    assertionCount: 2,
    project: 'ubuntu',
    startedAt: '2026-09-26T07:32:45Z',
  });
  const run = testRunDump(html);
  for (const step of run.projects[0].documents[0].cases[0].attempts[0].steps) {
    step.node = 'review.assertConfiguredVisual';
    delete step.agentDetails;
  }
  const reportHtml = html.replace(
    /<script type="midscene_test_run_dump">[\s\S]*?<\/script>/,
    `<script type="midscene_test_run_dump">${JSON.stringify(run)}</script>`,
  );
  const [testCase] = await reportCases(run, 'ubuntu', { reportHtml });
  assert.equal(testCase.selection, 'last-no-screenshot');
  assert.equal(testCase.screenshot, undefined);
  assert.equal(testCase.descriptionKind, 'result');

  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-unlinked-step-'));
  const reportDirectory = path.join(root, 'artifact');
  await mkdir(path.join(reportDirectory, 'report'), { recursive: true });
  await writeFile(path.join(reportDirectory, 'report', 'test-run-ubuntu.html'), reportHtml);
  await writeFile(path.join(reportDirectory, 'report-preview.png'), 'preview');
  const server = await startServer((_request, response) => response.writeHead(404).end());
  context.after(server.close);
  const manifest = await buildPagesReport(
    options(reportDirectory, path.join(root, 'site'), server.url),
  );
  const [publishedCase] = manifest.reports[0].entries[0].cases;
  assert.equal(publishedCase.status, 'success');
  assert.equal(publishedCase.selection, 'last-no-screenshot');
  assert.equal(publishedCase.previewPath, undefined);
  assert.equal(publishedCase.description, 'Step passed; no per-step AI response was recorded.');
  const summary = renderReportSummary({
    manifest,
    pagesUrl: server.url,
    producerResult: 'success',
    runId: '200',
    summaryTitle: 'Ubuntu',
  });
  assert.match(summary, /Passed case|ubuntu visual case/);
  assert.match(summary, /\| — \| ✅ Passed \|/);
});

test('still rejects missing screenshots from explicitly linked AI steps', async () => {
  const html = runnerScript({
    project: 'ubuntu',
    startedAt: '2026-09-26T07:32:45Z',
  }).replace(/<script type="midscene-image"[^>]*>[\s\S]*?<\/script>/g, '');
  await assert.rejects(
    reportCases(testRunDump(html), 'ubuntu', { reportHtml: html }),
    /has no embedded node screenshot/,
  );
});

test('preserves a failed case and skips a not-run case after agent damage', async () => {
  const html = runnerWithoutScreenshot({
    project: 'ubuntu',
    startedAt: '2026-09-21T12:00:00Z',
  });
  const cases = await reportCases(testRunDump(html), 'ubuntu', {
    reportHtml: html,
  });
  assert.equal(cases.length, 1);
  const [testCase] = cases;

  assert.equal(testCase.name, 'ubuntu damaged-agent case');
  assert.equal(testCase.stepId, 'act-without-execution');
  assert.equal(testCase.selection, 'first-failing-no-screenshot');
  assert.equal(testCase.previewFile, undefined);
  assert.equal(testCase.screenshot, undefined);
  assert.equal(testCase.descriptionKind, 'error');
  assert.match(testCase.description, /no stable Agent execution reference/);
});

test('publishes all case metadata when one report has no node screenshot', async (context) => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'pages-no-node-shot-'));
  const reportDirectory = path.join(root, 'artifact');
  const nativeDirectory = path.join(reportDirectory, 'report');
  await mkdir(nativeDirectory, { recursive: true });
  const html = runnerWithoutScreenshot({
    project: 'ubuntu',
    startedAt: '2026-09-21T12:00:00Z',
  });
  await writeFile(path.join(nativeDirectory, 'test-run-ubuntu.html'), html);
  await writeFile(path.join(reportDirectory, 'report-preview.png'), 'preview');
  const siteDirectory = path.join(root, 'site');
  const server = await startServer((_request, response) =>
    response.writeHead(404).end(),
  );
  context.after(server.close);

  const manifest = await buildPagesReport({
    ...options(reportDirectory, siteDirectory, server.url),
  });
  const [testCase] = manifest.reports[0].entries[0].cases;
  assert.equal(manifest.version, 7);
  assert.equal(testCase.selection, 'first-failing-no-screenshot');
  assert.equal(testCase.previewPath, undefined);
  assert.match(testCase.description, /no stable Agent execution reference/);
  assert.doesNotMatch(JSON.stringify(manifest.reports[0].files), /case-preview/);

  const runIndex = await readFile(
    path.join(siteDirectory, 'reports', '200', 'index.html'),
    'utf8',
  );
  assert.match(runIndex, /ubuntu damaged-agent case/);
  assert.match(runIndex, /Not available/);

  const summary = renderReportSummary({
    manifest,
    pagesUrl: 'https://example.test/doubao-say/',
    producerResult: 'failure',
    runId: '200',
    summaryTitle: 'Ubuntu',
  });
  assert.match(summary, /1 need attention · 0 passed/);
  assert.match(summary, /### Needs attention/);
  assert.match(summary, /\| ubuntu \| \[ubuntu damaged-agent case\].*\| — \| ❌ Failed:/);
  assert.match(summary, /Appendix: passed cases \(0\)/);
  assert.match(summary, /ubuntu damaged-agent case/);
});

test('finds Omarchy evidence by project instead of assertion wording', async () => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'shell-project-'));
  const reportDirectory = path.join(root, 'report');
  await mkdir(reportDirectory, { recursive: true });
  await writeFile(
    path.join(reportDirectory, 'test-run-shell.html'),
    shellFixtureHtml
      .replace(shellPrompts[0], 'Renamed menu assertion.')
      .replace(shellPrompts[1], 'Renamed focus assertion.')
      .replace(shellPrompts[2], 'Renamed bar assertion.'),
  );

  const report = await findShellReport(root);
  assert.deepEqual(
    report.checks.map(({ key, passed }) => ({ key, passed })),
    [
      { key: 'shutdown', passed: true },
      { key: 'focus', passed: true },
      { key: 'bar', passed: true },
    ],
  );
});

test('uses the successful Omarchy retry instead of failed earlier assertions', async () => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'shell-retry-'));
  const reportDirectory = path.join(root, 'report');
  await mkdir(reportDirectory, { recursive: true });
  const failedAttempt = `<script type="midscene_web_dump">${JSON.stringify({
    executions: [true, false].map((output) => ({
      tasks: [{ status: 'finished', subType: 'Assert', output }],
    })),
  })}</script>`;
  await writeFile(
    path.join(reportDirectory, 'test-run-shell.html'),
    shellFixtureHtml.replace('<body>', `<body>${failedAttempt}`),
  );

  const report = await findShellReport(root);
  assert.deepEqual(report.checks.map(({ passed }) => passed), [true, true, true]);
});

test('reads Midscene Test 1.13.0 node screenshots from the screenshots directory', async () => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'directory-shots-'));
  const reportDirectory = path.join(
    root,
    'report',
    'midscene-e2e-20260921045854-d9ab4bd0',
  );
  await mkdir(path.join(reportDirectory, 'screenshots'), {
    recursive: true,
  });
  const html = runnerScript({
    project: 'ubuntu',
    startedAt: '2026-09-21T12:00:00Z',
    screenshotMode: 'directory',
  });
  const reportFile = path.join(reportDirectory, 'index.html');
  await writeFile(reportFile, html);
  await writeFile(
    path.join(reportDirectory, 'screenshots', 'screenshot-0-0.jpeg'),
    Buffer.from('/9j/2Q==', 'base64'),
  );

  const [testCase] = await reportCases(testRunDump(html), 'ubuntu', {
    reportHtml: html,
    reportFile,
  });
  assert.equal(testCase.descriptionKind, 'ai');
  assert.equal(testCase.description, 'AI explanation 0-0');
  assert.equal(testCase.screenshot.extension, 'jpg');
  assert.deepEqual(
    testCase.screenshot.bytes,
    Buffer.from('/9j/2Q==', 'base64'),
  );
});

test('reads 1.13.0 shell screenshots from the screenshots directory', async () => {
  const root = await mkdtemp(path.join(os.tmpdir(), 'shell-directory-'));
  const reportDirectory = path.join(root, 'report');
  await mkdir(path.join(reportDirectory, 'screenshots'), {
    recursive: true,
  });
  const html = shellFixtureHtml.replace(
    /<script\s+type=["']midscene-image["'][^>]*>[\s\S]*?<\/script>/g,
    '',
  );
  await writeFile(
    path.join(reportDirectory, 'test-run-shell.html'),
    html,
  );
  for (const index of [0, 1, 2]) {
    await writeFile(
      path.join(reportDirectory, 'screenshots', `image-${index}.jpeg`),
      Buffer.from('/9j/2Q==', 'base64'),
    );
  }

  const report = await findShellReport(root);
  assert.deepEqual(
    report.checks.map(({ key, passed, screenshot }) => ({
      key,
      passed,
      hasScreenshot: screenshot !== null,
    })),
    [
      { key: 'shutdown', passed: true, hasScreenshot: true },
      { key: 'focus', passed: true, hasScreenshot: true },
      { key: 'bar', passed: true, hasScreenshot: true },
    ],
  );
});

test('verifies every file and its expected content type', async () => {
  const requested = [];
  const manifest = {
    reports: [
      {
        runId: '200',
        files: [
          'reports/200/index.html',
          'reports/200/auxiliary-report.html',
          'reports/200/report-preview.png',
          'reports/200/auxiliary-report-preview.png',
          'reports/200/case-preview-ubuntu-case-ubuntu.jpg',
          'reports/200/screenshots/node-shot-1.jpeg',
        ],
      },
    ],
  };
  await verifyPublishedReport({
    manifest,
    pagesUrl: 'https://example.test/doubao-say/',
    runId: '200',
    fetchImpl: async (url) => {
      requested.push(url.href);
      return new Response(null, {
        status: 200,
        headers: {
          'content-type': url.pathname.endsWith('.png')
            ? 'image/png'
            : url.pathname.endsWith('.jpg') ||
                url.pathname.endsWith('.jpeg')
              ? 'image/jpeg'
              : 'text/html; charset=utf-8',
        },
      });
    },
  });

  assert.deepEqual(requested, [
    'https://example.test/doubao-say/',
    'https://example.test/doubao-say/reports/200/index.html',
    'https://example.test/doubao-say/reports/200/auxiliary-report.html',
    'https://example.test/doubao-say/reports/200/report-preview.png',
    'https://example.test/doubao-say/reports/200/auxiliary-report-preview.png',
    'https://example.test/doubao-say/reports/200/case-preview-ubuntu-case-ubuntu.jpg',
    'https://example.test/doubao-say/reports/200/screenshots/node-shot-1.jpeg',
  ]);
});