diff --git a/tools/check-coverage.test.ts b/tools/check-coverage.test.ts new file mode 100644 index 000000000..6cc7ec95a --- /dev/null +++ b/tools/check-coverage.test.ts @@ -0,0 +1,119 @@ +import { assertEquals, assertRejects, assertStringIncludes } from '@std/assert'; +import { + classifyTestExit, + describeNativeCrash, + runTestSuiteWithCrashRetry, +} from './check-coverage.ts'; + +function scriptedRunner( + codes: number[], +): { runner: () => Promise<{ code: number }>; calls: () => number } { + let calls = 0; + return { + runner: () => { + const code = codes[Math.min(calls, codes.length - 1)]; + calls++; + return Promise.resolve({ code }); + }, + calls: () => calls, + }; +} + +Deno.test('classifyTestExit separates real failures from native crashes (#1278)', () => { + assertEquals(classifyTestExit(0), 'ok'); + // deno test reports assertion failures as exit code 1; usage and spawn + // errors stay below the 128 + signal floor. None of these may be retried. + assertEquals(classifyTestExit(1), 'test-failure'); + assertEquals(classifyTestExit(2), 'test-failure'); + assertEquals(classifyTestExit(127), 'test-failure'); + // Signal-terminated processes surface as 128 + signal number. + assertEquals(classifyTestExit(128), 'native-crash'); + assertEquals(classifyTestExit(132), 'native-crash'); // SIGILL + assertEquals(classifyTestExit(134), 'native-crash'); // SIGABRT + assertEquals(classifyTestExit(139), 'native-crash'); // SIGSEGV +}); + +Deno.test('describeNativeCrash names known signals and stays explicit for unknown ones', () => { + assertStringIncludes(describeNativeCrash(139), 'SIGSEGV'); + assertStringIncludes(describeNativeCrash(139), '139'); + assertStringIncludes(describeNativeCrash(134), 'SIGABRT'); + assertStringIncludes(describeNativeCrash(200), 'signal 72'); + assertStringIncludes(describeNativeCrash(200), '200'); +}); + +Deno.test('a crash without any assertion failure is retried and can recover', async () => { + const { runner, calls } = scriptedRunner([139, 139, 0]); + const crashes: number[] = []; + + const result = await runTestSuiteWithCrashRetry(runner, { + maxAttempts: 3, + onCrash: ({ attempt }) => crashes.push(attempt), + }); + + assertEquals(result, { crashes: 2 }); + assertEquals(calls(), 3); + // Every crash is reported loudly, in order, so flakes stay countable. + assertEquals(crashes, [1, 2]); +}); + +Deno.test('a real assertion failure fails immediately without any retry', async () => { + const { runner, calls } = scriptedRunner([1, 0]); + let crashes = 0; + + const error = await assertRejects( + () => runTestSuiteWithCrashRetry(runner, { maxAttempts: 3, onCrash: () => crashes++ }), + Error, + ); + + assertStringIncludes(error.message, 'tests failed with code 1'); + assertEquals(calls(), 1); + assertEquals(crashes, 0); +}); + +Deno.test('crash exhaustion fails loudly after the bounded attempt count', async () => { + const { runner, calls } = scriptedRunner([139]); + const crashes: Array<{ attempt: number; maxAttempts: number; code: number }> = []; + + const error = await assertRejects( + () => + runTestSuiteWithCrashRetry(runner, { + maxAttempts: 3, + onCrash: (event) => crashes.push(event), + }), + Error, + ); + + assertEquals(calls(), 3); + assertStringIncludes(error.message, 'crashed natively'); + assertStringIncludes(error.message, 'SIGSEGV'); + assertStringIncludes(error.message, '3'); + assertEquals(crashes, [ + { attempt: 1, maxAttempts: 3, code: 139 }, + { attempt: 2, maxAttempts: 3, code: 139 }, + { attempt: 3, maxAttempts: 3, code: 139 }, + ]); +}); + +Deno.test('different crash signals across attempts still count toward the same bound', async () => { + const { runner, calls } = scriptedRunner([134, 139, 0]); + + const result = await runTestSuiteWithCrashRetry(runner, { maxAttempts: 3 }); + + assertEquals(result, { crashes: 2 }); + assertEquals(calls(), 3); +}); + +Deno.test('maxAttempts must be a positive integer', async () => { + const { runner, calls } = scriptedRunner([0]); + await assertRejects( + () => runTestSuiteWithCrashRetry(runner, { maxAttempts: 0 }), + Error, + 'positive integer', + ); + await assertRejects( + () => runTestSuiteWithCrashRetry(runner, { maxAttempts: 1.5 }), + Error, + 'positive integer', + ); + assertEquals(calls(), 0); +}); diff --git a/tools/check-coverage.ts b/tools/check-coverage.ts index 58d03bf2e..4b083c210 100644 --- a/tools/check-coverage.ts +++ b/tools/check-coverage.ts @@ -20,26 +20,117 @@ function getNumberArg(flag: string, fallback: number): number { return value; } -async function runCoverage(): Promise { +// Issue #1278: the `deno test --coverage` subprocess has repeatedly died by +// native crash (observed exit 139 = SIGSEGV, rolldown/workerd class) with no +// test assertion failure. Signal-terminated processes surface as exit code +// 128 + signal number; only those are retryable. Any exit code below the +// floor — including 1, the deno test assertion-failure code — is a real +// failure and must fail the gate immediately, never retried. +const NATIVE_CRASH_FLOOR = 128; + +const SIGNAL_NAMES: Record = { + 4: 'SIGILL', + 5: 'SIGTRAP', + 6: 'SIGABRT', + 7: 'SIGBUS', + 8: 'SIGFPE', + 11: 'SIGSEGV', +}; + +export type TestExitKind = 'ok' | 'test-failure' | 'native-crash'; + +export function classifyTestExit(code: number): TestExitKind { + if (code === 0) return 'ok'; + return code >= NATIVE_CRASH_FLOOR ? 'native-crash' : 'test-failure'; +} + +export function describeNativeCrash(code: number): string { + const signal = code - NATIVE_CRASH_FLOOR; + const name = SIGNAL_NAMES[signal]; + return name ? `signal ${name} (exit code ${code})` : `signal ${signal} (exit code ${code})`; +} + +export interface CrashRetryEvent { + attempt: number; + maxAttempts: number; + code: number; +} + +// Runs the coverage test suite with a bounded, fail-loud native-crash retry: +// every crash is reported through onCrash, real test failures abort without +// retry, and exhausting maxAttempts on crashes alone throws. Returns the +// number of crashes observed so the caller can keep recovered flakes visible. +export async function runTestSuiteWithCrashRetry( + runner: () => Promise<{ code: number }>, + options: { maxAttempts: number; onCrash?: (event: CrashRetryEvent) => void }, +): Promise<{ crashes: number }> { + const { maxAttempts, onCrash } = options; + if (!Number.isInteger(maxAttempts) || maxAttempts < 1) { + throw new Error('maxAttempts must be a positive integer'); + } + let crashes = 0; + for (let attempt = 1;; attempt++) { + const { code } = await runner(); + const kind = classifyTestExit(code); + if (kind === 'ok') return { crashes }; + if (kind === 'test-failure') throw new Error(`tests failed with code ${code}`); + crashes++; + onCrash?.({ attempt, maxAttempts, code }); + if (attempt >= maxAttempts) { + throw new Error( + `coverage test run crashed natively (${describeNativeCrash(code)}) on all ` + + `${maxAttempts} attempts; refusing to pass the gate on repeated native ` + + 'crashes (#1278)', + ); + } + } +} + +async function runCoverage(crashRetries: number): Promise { const coverageDir = '.coverage-check'; try { - const test = await new Deno.Command(Deno.execPath(), { - args: [ - 'test', - '--no-lock', - `--coverage=${coverageDir}`, - '--allow-read', - '--allow-write', - '--allow-env', - '--allow-net', - '--allow-run', - '--allow-ffi', - '--allow-sys', - ], - stdout: 'inherit', - stderr: 'inherit', - }).spawn().status; - if (!test.success) throw new Error(`tests failed with code ${test.code}`); + const { crashes } = await runTestSuiteWithCrashRetry( + async () => { + // A crashed attempt can leave partial coverage profiles behind that + // `deno coverage` would choke on; each attempt starts from a clean dir. + await Deno.remove(coverageDir, { recursive: true }).catch(() => undefined); + return await new Deno.Command(Deno.execPath(), { + args: [ + 'test', + '--no-lock', + `--coverage=${coverageDir}`, + '--allow-read', + '--allow-write', + '--allow-env', + '--allow-net', + '--allow-run', + '--allow-ffi', + '--allow-sys', + ], + stdout: 'inherit', + stderr: 'inherit', + }).spawn().status; + }, + { + maxAttempts: crashRetries + 1, + onCrash: ({ attempt, maxAttempts, code }) => { + console.error( + `\n[check-coverage] NATIVE CRASH: deno test terminated by ${ + describeNativeCrash(code) + } ` + + `on attempt ${attempt}/${maxAttempts} with no test assertion failure (#1278). ` + + (attempt < maxAttempts ? 'Retrying.' : 'No attempts left.'), + ); + }, + }, + ); + if (crashes > 0) { + console.error( + `\n[check-coverage] WARNING: coverage run recovered after ${crashes} native ` + + `crash(es) (#1278). The gate passed, but the flake stays visible — count ` + + 'these lines in CI logs when trending the crash rate.', + ); + } const report = await new Deno.Command(Deno.execPath(), { args: ['coverage', coverageDir, '--lcov'], @@ -80,7 +171,11 @@ function formatMetric(name: string, metric: CoverageMetric, threshold: number): async function main(): Promise { await ensureWwwBuildOutput(); - const lcov = await runCoverage(); + // Bounded crash retry for #1278: one clean run plus `--crash-retries` + // retries that only fire on native-crash exits (>= 128 + signal), never on + // assertion failures. Loud by design: every crash prints to stderr. + const crashRetries = getNumberArg('--crash-retries', 2); + const lcov = await runCoverage(crashRetries); const profiledFiles = lcovFilePaths(lcov); // Threshold baseline: 2026-08-04 (v0.42.0-alpha.14 cycle), measured with the