Skip to content

Commit 480b381

Browse files
benleshbenlesh
committed
ci: harden hosted runner contracts
1 parent 36e9121 commit 480b381

14 files changed

Lines changed: 292 additions & 33 deletions

‎.github/workflows/ci_main.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ jobs:
5656

5757
steps:
5858
- uses: actions/checkout@v4
59+
with:
60+
fetch-depth: 0
5961

6062
- name: Install dependencies
6163
uses: ./.github/actions/install-dependencies

‎packages/observable-polyfill/test/wpt/lib/runner.mjs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,7 @@ export async function runWpt({
340340
'0',
341341
'--binary-arg=--js-flags=--expose-gc',
342342
'--binary-arg=--disable-dev-shm-usage',
343+
...(process.platform === 'linux' ? ['--binary-arg=--no-sandbox'] : []),
343344
...browserIsolationArgs.map((argument) => `--binary-arg=${argument}`),
344345
'--no-fail-on-unexpected',
345346
'--log-wptreport',

‎packages/rxjs/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
"test:unit:polyfill": "RXJS_NEXT_TEST_MODE=polyfill vitest --run --config vitest.ported.config.ts",
3131
"test:unit:native": "RXJS_NEXT_TEST_MODE=native vitest --run --config vitest.ported.config.ts",
3232
"test:unit:audit": "RXJS_NEXT_AUDIT_ONLY=1 RXJS_NEXT_TEST_MODE=cold vitest --run --config vitest.ported.config.ts",
33-
"test:unit:audit:check": "node --test test/ported/tools/mode-audit-baseline.test.mjs test/ported/tools/mode-audit-reporter.test.mjs && node test/ported/tools/check-mode-audits.mjs",
33+
"test:unit:audit:check": "node --test test/ported/tools/mode-audit-baseline.test.mjs test/ported/tools/mode-audit-repair.test.mjs test/ported/tools/mode-audit-reporter.test.mjs && node test/ported/tools/check-mode-audits.mjs",
3434
"test:unit:audit:polyfill": "RXJS_NEXT_AUDIT_ONLY=1 RXJS_NEXT_TEST_MODE=polyfill vitest --run --config vitest.ported.config.ts",
3535
"test:unit:audit:record": "node test/ported/tools/record-mode-audit.mjs",
3636
"test:unit:report": "node test/ported/report.mjs",

‎packages/rxjs/test/ported/tools/check-mode-audits.mjs‎

Lines changed: 42 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { tmpdir } from 'node:os';
66
import { dirname, join, resolve } from 'node:path';
77
import { fileURLToPath } from 'node:url';
88
import { assertAuditBaseline, baselineFromAuditReport } from './mode-audit-baseline.mjs';
9+
import { findIncompleteAuditFiles, repairAuditReport } from './mode-audit-repair.mjs';
910

1011
const toolDirectory = dirname(fileURLToPath(import.meta.url));
1112
const packageDirectory = resolve(toolDirectory, '../../..');
@@ -16,29 +17,27 @@ const temporaryDirectory = await mkdtemp(join(tmpdir(), 'rxjs-next-mode-audits-'
1617
try {
1718
for (const mode of ['cold', 'polyfill']) {
1819
const reportPath = join(temporaryDirectory, `${mode}.json`);
19-
const result = spawnSync(
20-
process.execPath,
21-
[
22-
resolve(packageDirectory, '../../node_modules/vitest/vitest.mjs'),
23-
'--run',
24-
'--config',
25-
'vitest.ported.config.ts',
26-
'--reporter=./test/ported/tools/mode-audit-reporter.mjs',
27-
],
28-
{
29-
cwd: packageDirectory,
30-
env: { ...process.env, RXJS_NEXT_AUDIT_ONLY: '1', RXJS_NEXT_AUDIT_REPORT: reportPath, RXJS_NEXT_TEST_MODE: mode },
31-
stdio: 'inherit',
20+
runAudit(mode, reportPath);
21+
22+
let [report, expected] = await Promise.all([readJson(reportPath), readJson(resolve(toolDirectory, `../verified-${mode}-passes.json`))]);
23+
const suiteMode = mode === 'cold' ? 'cold' : 'platform';
24+
const incompleteFiles = findIncompleteAuditFiles({
25+
migrationEntries: migrationReport.modes[suiteMode],
26+
packageDirectory,
27+
report,
28+
});
29+
if (incompleteFiles.length > 0) {
30+
process.stdout.write(
31+
`Re-running ${incompleteFiles.length} ${mode} audit file(s) whose complete task results were not retained by the full Vitest run.\n`
32+
);
33+
const replacementReports = [];
34+
for (const [index, file] of incompleteFiles.entries()) {
35+
const supplementalPath = join(temporaryDirectory, `${mode}-supplemental-${index}.json`);
36+
runAudit(mode, supplementalPath, file);
37+
replacementReports.push(await readJson(supplementalPath));
3238
}
33-
);
34-
if (result.signal || (result.status !== 0 && result.status !== 1)) {
35-
throw new Error(`${mode} audit process did not complete normally: status ${String(result.status)}, signal ${String(result.signal)}.`);
39+
report = repairAuditReport({ packageDirectory, report, replacementReports });
3640
}
37-
38-
const [report, expected] = await Promise.all([
39-
readJson(reportPath),
40-
readJson(resolve(toolDirectory, `../verified-${mode}-passes.json`)),
41-
]);
4241
const actual = baselineFromAuditReport({ manifest, migrationReport, mode, packageDirectory, report });
4342
assertAuditBaseline(actual, expected);
4443
process.stdout.write(
@@ -52,3 +51,25 @@ try {
5251
async function readJson(path) {
5352
return JSON.parse(await readFile(path, 'utf8'));
5453
}
54+
55+
function runAudit(mode, reportPath, file) {
56+
const result = spawnSync(
57+
process.execPath,
58+
[
59+
resolve(packageDirectory, '../../node_modules/vitest/vitest.mjs'),
60+
'--run',
61+
...(file ? [file] : []),
62+
'--config',
63+
'vitest.ported.config.ts',
64+
'--reporter=./test/ported/tools/mode-audit-reporter.mjs',
65+
],
66+
{
67+
cwd: packageDirectory,
68+
env: { ...process.env, RXJS_NEXT_AUDIT_ONLY: '1', RXJS_NEXT_AUDIT_REPORT: reportPath, RXJS_NEXT_TEST_MODE: mode },
69+
stdio: 'inherit',
70+
}
71+
);
72+
if (result.signal || (result.status !== 0 && result.status !== 1)) {
73+
throw new Error(`${mode} audit process did not complete normally: status ${String(result.status)}, signal ${String(result.signal)}.`);
74+
}
75+
}

‎packages/rxjs/test/ported/tools/mode-audit-repair.mjs‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import { relative } from 'node:path';
2+
3+
export function findIncompleteAuditFiles({ migrationEntries, packageDirectory, report }) {
4+
const resultsByFile = new Map();
5+
for (const result of report.testResults ?? []) {
6+
const file = normalizeFile(packageDirectory, result.name);
7+
const results = resultsByFile.get(file) ?? [];
8+
results.push(result);
9+
resultsByFile.set(file, results);
10+
}
11+
12+
return migrationEntries.flatMap((entry) => {
13+
const results = resultsByFile.get(entry.file) ?? [];
14+
if (results.length > 1) return [];
15+
const assertions = results[0]?.assertionResults ?? [];
16+
const incomplete = assertions.some((assertion) => assertion.status !== 'passed' && assertion.status !== 'failed');
17+
return results.length === 0 || assertions.length !== entry.caseIds.length || incomplete ? [entry.file] : [];
18+
});
19+
}
20+
21+
export function repairAuditReport({ packageDirectory, report, replacementReports }) {
22+
const replacements = new Map();
23+
let replacementUnhandledErrors = 0;
24+
for (const replacementReport of replacementReports) {
25+
replacementUnhandledErrors += replacementReport.unhandledErrors ?? 0;
26+
if (replacementReport.testResults?.length !== 1) {
27+
throw new Error(`A supplemental audit must report exactly one test file; received ${replacementReport.testResults?.length ?? 0}.`);
28+
}
29+
const [replacement] = replacementReport.testResults;
30+
const file = normalizeFile(packageDirectory, replacement.name);
31+
if (replacements.has(file)) {
32+
throw new Error(`Supplemental audit results contain a duplicate test file: ${file}`);
33+
}
34+
replacements.set(file, replacement);
35+
}
36+
37+
const replacedFiles = new Set();
38+
const testResults = (report.testResults ?? []).map((result) => {
39+
const file = normalizeFile(packageDirectory, result.name);
40+
const replacement = replacements.get(file);
41+
if (!replacement) return result;
42+
replacedFiles.add(file);
43+
return replacement;
44+
});
45+
for (const [file, replacement] of replacements) {
46+
if (!replacedFiles.has(file)) testResults.push(replacement);
47+
}
48+
49+
const assertions = testResults.flatMap((result) => result.assertionResults ?? []);
50+
return {
51+
...report,
52+
numTotalTests: assertions.length,
53+
numPassedTests: countStatus(assertions, 'passed'),
54+
numFailedTests: countStatus(assertions, 'failed'),
55+
numPendingTests: countStatus(assertions, 'pending'),
56+
numTodoTests: countStatus(assertions, 'todo'),
57+
unhandledErrors: (report.unhandledErrors ?? 0) + replacementUnhandledErrors,
58+
testResults,
59+
};
60+
}
61+
62+
function normalizeFile(packageDirectory, path) {
63+
return relative(packageDirectory, path).replaceAll('\\', '/');
64+
}
65+
66+
function countStatus(assertions, status) {
67+
return assertions.filter((assertion) => assertion.status === status).length;
68+
}

‎packages/rxjs/test/ported/tools/mode-audit-repair.test.mjs‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import assert from 'node:assert/strict';
2+
import test from 'node:test';
3+
import { findIncompleteAuditFiles, repairAuditReport } from './mode-audit-repair.mjs';
4+
5+
const packageDirectory = '/workspace/packages/rxjs';
6+
7+
test('finds missing, short, and incomplete files without masking duplicate reports', () => {
8+
const migrationEntries = [
9+
{ file: 'test/a.spec.ts', caseIds: ['A'] },
10+
{ file: 'test/b.spec.ts', caseIds: ['B'] },
11+
{ file: 'test/c.spec.ts', caseIds: ['C'] },
12+
{ file: 'test/duplicate.spec.ts', caseIds: ['D'] },
13+
];
14+
const duplicate = result('test/duplicate.spec.ts', ['passed']);
15+
const report = {
16+
testResults: [result('test/a.spec.ts', []), result('test/b.spec.ts', ['incomplete']), duplicate, duplicate],
17+
};
18+
19+
assert.deepEqual(findIncompleteAuditFiles({ migrationEntries, packageDirectory, report }), [
20+
'test/a.spec.ts',
21+
'test/b.spec.ts',
22+
'test/c.spec.ts',
23+
]);
24+
});
25+
26+
test('replaces incomplete files, appends missing files, and recalculates totals', () => {
27+
const report = {
28+
numTotalTests: 1,
29+
numPassedTests: 0,
30+
numFailedTests: 0,
31+
numPendingTests: 0,
32+
numTodoTests: 0,
33+
unhandledErrors: 0,
34+
testResults: [result('test/a.spec.ts', ['incomplete'])],
35+
};
36+
const repaired = repairAuditReport({
37+
packageDirectory,
38+
report,
39+
replacementReports: [
40+
{ unhandledErrors: 0, testResults: [result('test/a.spec.ts', ['passed'])] },
41+
{ unhandledErrors: 0, testResults: [result('test/b.spec.ts', ['failed'])] },
42+
],
43+
});
44+
45+
assert.deepEqual(
46+
{
47+
total: repaired.numTotalTests,
48+
passed: repaired.numPassedTests,
49+
failed: repaired.numFailedTests,
50+
pending: repaired.numPendingTests,
51+
todo: repaired.numTodoTests,
52+
},
53+
{ total: 2, passed: 1, failed: 1, pending: 0, todo: 0 }
54+
);
55+
assert.deepEqual(
56+
repaired.testResults.map((entry) => entry.assertionResults),
57+
[[{ status: 'passed' }], [{ status: 'failed' }]]
58+
);
59+
});
60+
61+
test('rejects ambiguous supplemental reports', () => {
62+
assert.throws(
63+
() => repairAuditReport({ packageDirectory, report: {}, replacementReports: [{ testResults: [] }] }),
64+
/exactly one test file/
65+
);
66+
});
67+
68+
function result(file, statuses) {
69+
return {
70+
name: `${packageDirectory}/${file}`,
71+
assertionResults: statuses.map((status) => ({ status })),
72+
};
73+
}

‎packages/rxjs/test/ported/tools/mode-audit-reporter.mjs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ export default class ModeAuditReporter {
77
onCollected(files = []) {
88
const snapshot = createCollectedSnapshot(files);
99
for (const file of snapshot.files) {
10-
this.#filesByPath.set(file.filepath, file);
10+
const existing = this.#filesByPath.get(file.filepath);
11+
this.#filesByPath.set(file.filepath, preferMoreCompleteTaskTree(existing, file));
1112
}
1213
for (const [id, task] of snapshot.tasksById) {
1314
this.#tasksById.set(id, task);
@@ -44,6 +45,10 @@ export function applyTaskUpdates(tasksById, packs) {
4445
}
4546
}
4647

48+
export function preferMoreCompleteTaskTree(existing, candidate) {
49+
return existing && collectTests(existing).length > collectTests(candidate).length ? existing : candidate;
50+
}
51+
4752
export function createAuditReport(files, errors = []) {
4853
const testResults = files.map((file) => {
4954
const tests = collectTests(file);

‎packages/rxjs/test/ported/tools/mode-audit-reporter.test.mjs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import assert from 'node:assert/strict';
22
import test from 'node:test';
3-
import { applyTaskUpdates, createAuditReport, createCollectedSnapshot } from './mode-audit-reporter.mjs';
3+
import { applyTaskUpdates, createAuditReport, createCollectedSnapshot, preferMoreCompleteTaskTree } from './mode-audit-reporter.mjs';
44

55
test('serializes nested Vitest task results in declaration order', () => {
66
const report = createAuditReport([
@@ -60,3 +60,8 @@ test('preserves collected tasks when Vitest only supplies later result packs', (
6060
applyTaskUpdates(snapshot.tasksById, [['test', { state: 'pass' }, {}]]);
6161
assert.deepEqual(createAuditReport(snapshot.files).testResults[0].assertionResults, [{ status: 'passed' }]);
6262
});
63+
64+
test('keeps a complete collection snapshot when Vitest later reports an empty file shell', () => {
65+
const complete = { type: 'suite', tasks: [{ type: 'test', result: { state: 'pass' } }] };
66+
assert.equal(preferMoreCompleteTaskTree(complete, { type: 'suite', tasks: [] }), complete);
67+
});

‎packages/rxjs/test/release/boot-ios-simulator.mjs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#!/usr/bin/env node
22

33
import { execFileSync } from 'node:child_process';
4+
import { appendFileSync } from 'node:fs';
45
import { pathToFileURL } from 'node:url';
56

67
export function selectIosSimulator(devicesByRuntime) {
@@ -25,7 +26,7 @@ export function selectIosSimulator(devicesByRuntime) {
2526
return candidates[0];
2627
}
2728

28-
export function bootIosSimulator(run = execFileSync) {
29+
export function bootIosSimulator(run = execFileSync, recordEnvironment = recordGitHubEnvironment) {
2930
const listing = JSON.parse(run('xcrun', ['simctl', 'list', '--json', 'devices', 'available'], { encoding: 'utf8' }));
3031
const simulator = selectIosSimulator(listing.devices ?? {});
3132

@@ -34,10 +35,16 @@ export function bootIosSimulator(run = execFileSync) {
3435
}
3536
run('open', ['-a', 'Simulator', '--args', '-CurrentDeviceUDID', simulator.udid], { stdio: 'inherit' });
3637
run('xcrun', ['simctl', 'bootstatus', simulator.udid, '-b'], { stdio: 'inherit' });
38+
run('xcrun', ['simctl', 'launch', simulator.udid, 'com.apple.mobilesafari'], { stdio: 'inherit' });
39+
recordEnvironment('RXJS_SAFARI_DEVICE_UDID', simulator.udid);
3740
process.stdout.write(`Booted ${simulator.name} (${simulator.runtime}, ${simulator.udid}).\n`);
3841
return simulator;
3942
}
4043

44+
function recordGitHubEnvironment(name, value) {
45+
if (process.env.GITHUB_ENV) appendFileSync(process.env.GITHUB_ENV, `${name}=${value}\n`);
46+
}
47+
4148
function compareRuntimeVersions(left, right) {
4249
const leftParts = runtimeVersion(left);
4350
const rightParts = runtimeVersion(right);

‎packages/rxjs/test/release/boot-ios-simulator.test.mjs‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import assert from 'node:assert/strict';
22
import test from 'node:test';
3-
import { selectIosSimulator } from './boot-ios-simulator.mjs';
3+
import { bootIosSimulator, selectIosSimulator } from './boot-ios-simulator.mjs';
44

55
test('selects a booted iPhone from the newest available iOS runtime', () => {
66
const selected = selectIosSimulator({
@@ -25,3 +25,24 @@ test('rejects a simulator inventory without an available iPhone', () => {
2525
/No available iPhone simulator/
2626
);
2727
});
28+
29+
test('launches Mobile Safari and records the selected simulator for the next workflow step', () => {
30+
const commands = [];
31+
const environment = [];
32+
const run = (command, args) => {
33+
commands.push([command, args]);
34+
if (args[0] === 'simctl' && args[1] === 'list') {
35+
return JSON.stringify({
36+
devices: {
37+
'com.apple.CoreSimulator.SimRuntime.iOS-18-5': [{ name: 'iPhone 16 Pro', udid: 'selected', state: 'Booted', isAvailable: true }],
38+
},
39+
});
40+
}
41+
return '';
42+
};
43+
44+
bootIosSimulator(run, (name, value) => environment.push([name, value]));
45+
46+
assert.ok(commands.some(([command, args]) => command === 'xcrun' && args.join(' ') === 'simctl launch selected com.apple.mobilesafari'));
47+
assert.deepEqual(environment, [['RXJS_SAFARI_DEVICE_UDID', 'selected']]);
48+
});

0 commit comments

Comments
 (0)