Skip to content

Commit 09ddedd

Browse files
SisyphusZhengDevBot
SisyphusZheng
and
DevBot
authored
fix(tools): fail-loud native-crash retry for test:coverage:check (#1278) (#1292)
The deno test --coverage subprocess has died by SIGSEGV (exit 139) three times under load with no test assertion failure, blocking a release-tier gate by random crash. Classify the test-run exit: codes below 128 are real failures and fail immediately without retry; signal exits (128 + signo) are retried up to --crash-retries (default 2), each crash reported loudly on stderr, and crash exhaustion fails the gate explicitly. Coverage semantics unchanged: thresholds, scopes and full denominator untouched. Co-authored-by: DevBot <devbot@openelement.dev>
1 parent 493cf2a commit 09ddedd

2 files changed

Lines changed: 233 additions & 19 deletions

File tree

‎tools/check-coverage.test.ts‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
import { assertEquals, assertRejects, assertStringIncludes } from '@std/assert';
2+
import {
3+
classifyTestExit,
4+
describeNativeCrash,
5+
runTestSuiteWithCrashRetry,
6+
} from './check-coverage.ts';
7+
8+
function scriptedRunner(
9+
codes: number[],
10+
): { runner: () => Promise<{ code: number }>; calls: () => number } {
11+
let calls = 0;
12+
return {
13+
runner: () => {
14+
const code = codes[Math.min(calls, codes.length - 1)];
15+
calls++;
16+
return Promise.resolve({ code });
17+
},
18+
calls: () => calls,
19+
};
20+
}
21+
22+
Deno.test('classifyTestExit separates real failures from native crashes (#1278)', () => {
23+
assertEquals(classifyTestExit(0), 'ok');
24+
// deno test reports assertion failures as exit code 1; usage and spawn
25+
// errors stay below the 128 + signal floor. None of these may be retried.
26+
assertEquals(classifyTestExit(1), 'test-failure');
27+
assertEquals(classifyTestExit(2), 'test-failure');
28+
assertEquals(classifyTestExit(127), 'test-failure');
29+
// Signal-terminated processes surface as 128 + signal number.
30+
assertEquals(classifyTestExit(128), 'native-crash');
31+
assertEquals(classifyTestExit(132), 'native-crash'); // SIGILL
32+
assertEquals(classifyTestExit(134), 'native-crash'); // SIGABRT
33+
assertEquals(classifyTestExit(139), 'native-crash'); // SIGSEGV
34+
});
35+
36+
Deno.test('describeNativeCrash names known signals and stays explicit for unknown ones', () => {
37+
assertStringIncludes(describeNativeCrash(139), 'SIGSEGV');
38+
assertStringIncludes(describeNativeCrash(139), '139');
39+
assertStringIncludes(describeNativeCrash(134), 'SIGABRT');
40+
assertStringIncludes(describeNativeCrash(200), 'signal 72');
41+
assertStringIncludes(describeNativeCrash(200), '200');
42+
});
43+
44+
Deno.test('a crash without any assertion failure is retried and can recover', async () => {
45+
const { runner, calls } = scriptedRunner([139, 139, 0]);
46+
const crashes: number[] = [];
47+
48+
const result = await runTestSuiteWithCrashRetry(runner, {
49+
maxAttempts: 3,
50+
onCrash: ({ attempt }) => crashes.push(attempt),
51+
});
52+
53+
assertEquals(result, { crashes: 2 });
54+
assertEquals(calls(), 3);
55+
// Every crash is reported loudly, in order, so flakes stay countable.
56+
assertEquals(crashes, [1, 2]);
57+
});
58+
59+
Deno.test('a real assertion failure fails immediately without any retry', async () => {
60+
const { runner, calls } = scriptedRunner([1, 0]);
61+
let crashes = 0;
62+
63+
const error = await assertRejects(
64+
() => runTestSuiteWithCrashRetry(runner, { maxAttempts: 3, onCrash: () => crashes++ }),
65+
Error,
66+
);
67+
68+
assertStringIncludes(error.message, 'tests failed with code 1');
69+
assertEquals(calls(), 1);
70+
assertEquals(crashes, 0);
71+
});
72+
73+
Deno.test('crash exhaustion fails loudly after the bounded attempt count', async () => {
74+
const { runner, calls } = scriptedRunner([139]);
75+
const crashes: Array<{ attempt: number; maxAttempts: number; code: number }> = [];
76+
77+
const error = await assertRejects(
78+
() =>
79+
runTestSuiteWithCrashRetry(runner, {
80+
maxAttempts: 3,
81+
onCrash: (event) => crashes.push(event),
82+
}),
83+
Error,
84+
);
85+
86+
assertEquals(calls(), 3);
87+
assertStringIncludes(error.message, 'crashed natively');
88+
assertStringIncludes(error.message, 'SIGSEGV');
89+
assertStringIncludes(error.message, '3');
90+
assertEquals(crashes, [
91+
{ attempt: 1, maxAttempts: 3, code: 139 },
92+
{ attempt: 2, maxAttempts: 3, code: 139 },
93+
{ attempt: 3, maxAttempts: 3, code: 139 },
94+
]);
95+
});
96+
97+
Deno.test('different crash signals across attempts still count toward the same bound', async () => {
98+
const { runner, calls } = scriptedRunner([134, 139, 0]);
99+
100+
const result = await runTestSuiteWithCrashRetry(runner, { maxAttempts: 3 });
101+
102+
assertEquals(result, { crashes: 2 });
103+
assertEquals(calls(), 3);
104+
});
105+
106+
Deno.test('maxAttempts must be a positive integer', async () => {
107+
const { runner, calls } = scriptedRunner([0]);
108+
await assertRejects(
109+
() => runTestSuiteWithCrashRetry(runner, { maxAttempts: 0 }),
110+
Error,
111+
'positive integer',
112+
);
113+
await assertRejects(
114+
() => runTestSuiteWithCrashRetry(runner, { maxAttempts: 1.5 }),
115+
Error,
116+
'positive integer',
117+
);
118+
assertEquals(calls(), 0);
119+
});

‎tools/check-coverage.ts‎

Lines changed: 114 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -20,26 +20,117 @@ function getNumberArg(flag: string, fallback: number): number {
2020
return value;
2121
}
2222

23-
async function runCoverage(): Promise<string> {
23+
// Issue #1278: the `deno test --coverage` subprocess has repeatedly died by
24+
// native crash (observed exit 139 = SIGSEGV, rolldown/workerd class) with no
25+
// test assertion failure. Signal-terminated processes surface as exit code
26+
// 128 + signal number; only those are retryable. Any exit code below the
27+
// floor — including 1, the deno test assertion-failure code — is a real
28+
// failure and must fail the gate immediately, never retried.
29+
const NATIVE_CRASH_FLOOR = 128;
30+
31+
const SIGNAL_NAMES: Record<number, string> = {
32+
4: 'SIGILL',
33+
5: 'SIGTRAP',
34+
6: 'SIGABRT',
35+
7: 'SIGBUS',
36+
8: 'SIGFPE',
37+
11: 'SIGSEGV',
38+
};
39+
40+
export type TestExitKind = 'ok' | 'test-failure' | 'native-crash';
41+
42+
export function classifyTestExit(code: number): TestExitKind {
43+
if (code === 0) return 'ok';
44+
return code >= NATIVE_CRASH_FLOOR ? 'native-crash' : 'test-failure';
45+
}
46+
47+
export function describeNativeCrash(code: number): string {
48+
const signal = code - NATIVE_CRASH_FLOOR;
49+
const name = SIGNAL_NAMES[signal];
50+
return name ? `signal ${name} (exit code ${code})` : `signal ${signal} (exit code ${code})`;
51+
}
52+
53+
export interface CrashRetryEvent {
54+
attempt: number;
55+
maxAttempts: number;
56+
code: number;
57+
}
58+
59+
// Runs the coverage test suite with a bounded, fail-loud native-crash retry:
60+
// every crash is reported through onCrash, real test failures abort without
61+
// retry, and exhausting maxAttempts on crashes alone throws. Returns the
62+
// number of crashes observed so the caller can keep recovered flakes visible.
63+
export async function runTestSuiteWithCrashRetry(
64+
runner: () => Promise<{ code: number }>,
65+
options: { maxAttempts: number; onCrash?: (event: CrashRetryEvent) => void },
66+
): Promise<{ crashes: number }> {
67+
const { maxAttempts, onCrash } = options;
68+
if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
69+
throw new Error('maxAttempts must be a positive integer');
70+
}
71+
let crashes = 0;
72+
for (let attempt = 1;; attempt++) {
73+
const { code } = await runner();
74+
const kind = classifyTestExit(code);
75+
if (kind === 'ok') return { crashes };
76+
if (kind === 'test-failure') throw new Error(`tests failed with code ${code}`);
77+
crashes++;
78+
onCrash?.({ attempt, maxAttempts, code });
79+
if (attempt >= maxAttempts) {
80+
throw new Error(
81+
`coverage test run crashed natively (${describeNativeCrash(code)}) on all ` +
82+
`${maxAttempts} attempts; refusing to pass the gate on repeated native ` +
83+
'crashes (#1278)',
84+
);
85+
}
86+
}
87+
}
88+
89+
async function runCoverage(crashRetries: number): Promise<string> {
2490
const coverageDir = '.coverage-check';
2591
try {
26-
const test = await new Deno.Command(Deno.execPath(), {
27-
args: [
28-
'test',
29-
'--no-lock',
30-
`--coverage=${coverageDir}`,
31-
'--allow-read',
32-
'--allow-write',
33-
'--allow-env',
34-
'--allow-net',
35-
'--allow-run',
36-
'--allow-ffi',
37-
'--allow-sys',
38-
],
39-
stdout: 'inherit',
40-
stderr: 'inherit',
41-
}).spawn().status;
42-
if (!test.success) throw new Error(`tests failed with code ${test.code}`);
92+
const { crashes } = await runTestSuiteWithCrashRetry(
93+
async () => {
94+
// A crashed attempt can leave partial coverage profiles behind that
95+
// `deno coverage` would choke on; each attempt starts from a clean dir.
96+
await Deno.remove(coverageDir, { recursive: true }).catch(() => undefined);
97+
return await new Deno.Command(Deno.execPath(), {
98+
args: [
99+
'test',
100+
'--no-lock',
101+
`--coverage=${coverageDir}`,
102+
'--allow-read',
103+
'--allow-write',
104+
'--allow-env',
105+
'--allow-net',
106+
'--allow-run',
107+
'--allow-ffi',
108+
'--allow-sys',
109+
],
110+
stdout: 'inherit',
111+
stderr: 'inherit',
112+
}).spawn().status;
113+
},
114+
{
115+
maxAttempts: crashRetries + 1,
116+
onCrash: ({ attempt, maxAttempts, code }) => {
117+
console.error(
118+
`\n[check-coverage] NATIVE CRASH: deno test terminated by ${
119+
describeNativeCrash(code)
120+
} ` +
121+
`on attempt ${attempt}/${maxAttempts} with no test assertion failure (#1278). ` +
122+
(attempt < maxAttempts ? 'Retrying.' : 'No attempts left.'),
123+
);
124+
},
125+
},
126+
);
127+
if (crashes > 0) {
128+
console.error(
129+
`\n[check-coverage] WARNING: coverage run recovered after ${crashes} native ` +
130+
`crash(es) (#1278). The gate passed, but the flake stays visible — count ` +
131+
'these lines in CI logs when trending the crash rate.',
132+
);
133+
}
43134

44135
const report = await new Deno.Command(Deno.execPath(), {
45136
args: ['coverage', coverageDir, '--lcov'],
@@ -80,7 +171,11 @@ function formatMetric(name: string, metric: CoverageMetric, threshold: number):
80171

81172
async function main(): Promise<void> {
82173
await ensureWwwBuildOutput();
83-
const lcov = await runCoverage();
174+
// Bounded crash retry for #1278: one clean run plus `--crash-retries`
175+
// retries that only fire on native-crash exits (>= 128 + signal), never on
176+
// assertion failures. Loud by design: every crash prints to stderr.
177+
const crashRetries = getNumberArg('--crash-retries', 2);
178+
const lcov = await runCoverage(crashRetries);
84179
const profiledFiles = lcovFilePaths(lcov);
85180

86181
// Threshold baseline: 2026-08-04 (v0.42.0-alpha.14 cycle), measured with the

0 commit comments

Comments
 (0)