Skip to content

Commit edbcc92

Browse files
dsherretdsherret
authored
feat: add a testIsolation build option to run each test file in its own process (#514)
1 parent eaa5668 commit edbcc92

10 files changed

Lines changed: 242 additions & 53 deletions

File tree

‎README.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,27 @@ The module is transformed and type checked like the test files are and it is not
200200
included in the npm package. It is loaded once for each of the emitted script
201201
and ESM output, before any test file.
202202

203+
### Test File Isolation
204+
205+
All the test files run in the same process by default, so the module state of a
206+
test file can affect the next one. For example, the global hooks of
207+
`@std/testing/bdd` error with "Cannot add global hooks after a global test is
208+
registered" when a second test file uses them.
209+
210+
Set the `testIsolation` build option to `"process"` to run each test file in its
211+
own process, which is what `deno test` does by running each file in its own
212+
isolate:
213+
214+
```ts
215+
await build({
216+
// ...etc...
217+
testIsolation: "process",
218+
});
219+
```
220+
221+
This is slower because it starts a process per test file, and a preload module
222+
is loaded before each test file rather than once per output.
223+
203224
### Polyfills
204225

205226
dnt adds polyfills for language features that may not exist in the environment

‎deno.jsonc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
"tests/polyfill_disabled_project/npm",
5959
"tests/polyfill_project/npm",
6060
"tests/shim_project/npm",
61+
"tests/test_hooks_project/npm",
6162
"tests/test_project/npm",
6263
"tests/tla_project/npm",
6364
"tests/types_package_project/npm",

‎deno.lock‎

Lines changed: 9 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎lib/test_runner/get_test_runner_code.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,45 @@ main();
114114
);
115115
});
116116

117+
Deno.test("gets code when isolating the test files", () => {
118+
const code = getTestRunnerCode({
119+
testEntryPoints: ["./test.ts"],
120+
denoTestShimPackageName: undefined,
121+
includeEsModule: true,
122+
includeScriptModule: true,
123+
testIsolation: "process",
124+
});
125+
assertStringIncludes(
126+
code,
127+
` const fileIndexArg = process.argv[2];
128+
if (fileIndexArg == null) {
129+
const { spawnSync } = require("child_process");
130+
let failed = false;
131+
for (const i of filePaths.keys()) {
132+
if (i > 0) {
133+
console.log("");
134+
}
135+
const args = [...process.execArgv, __filename, String(i)];
136+
const result = spawnSync(process.execPath, args, { stdio: "inherit" });
137+
if (result.error != null) {
138+
console.error(result.error);
139+
}
140+
if (result.status !== 0) {
141+
failed = true;
142+
}
143+
}
144+
if (failed) {
145+
process.exitCode = 1;
146+
}
147+
return;
148+
}
149+
150+
const filePath = filePaths[Number(fileIndexArg)];`,
151+
);
152+
// there's no loop over the files because each one runs in its own process
153+
assertEquals(code.includes("for (const [i, filePath]"), false);
154+
});
155+
117156
Deno.test("gets code when a preload module is used", () => {
118157
const code = getTestRunnerCode({
119158
testEntryPoints: ["./test.ts"],

‎lib/test_runner/get_test_runner_code.ts‎

Lines changed: 106 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ export function getTestRunnerCode(options: {
1010
includeEsModule: boolean | undefined;
1111
includeScriptModule: boolean | undefined;
1212
preloadEntryPoint?: string;
13+
testIsolation?: "process" | "none";
1314
}) {
1415
const usesDenoTest = options.denoTestShimPackageName != null;
1516
const preloadPath = options.preloadEntryPoint == null
@@ -34,7 +35,11 @@ export function getTestRunnerCode(options: {
3435
});
3536
writer.writeLine("];").newLine();
3637

38+
const isolateTestFiles = options.testIsolation === "process";
3739
writer.write("async function main()").block(() => {
40+
if (isolateTestFiles) {
41+
writeSpawnPerFile();
42+
}
3843
if (preloadPath != null) {
3944
if (options.includeScriptModule) {
4045
writer.writeLine(`process.chdir(__dirname + "/script");`);
@@ -59,60 +64,18 @@ export function getTestRunnerCode(options: {
5964
writer.writeLine("pc,");
6065
}).write(";").newLine();
6166
}
62-
writer.write("for (const [i, filePath] of filePaths.entries())")
63-
.block(() => {
64-
writer.write("if (i > 0)").block(() => {
65-
writer.writeLine(`console.log("");`);
66-
}).blankLine();
67-
68-
if (options.includeScriptModule) {
69-
writer.writeLine(`const scriptPath = "./script/" + filePath;`);
70-
writer.writeLine(
71-
`console.log("Running tests in " + pc.underline(scriptPath) + "...\\n");`,
72-
);
73-
writer.writeLine(`process.chdir(__dirname + "/script");`);
74-
if (usesDenoTest) {
75-
writer.write(`const scriptTestContext = `).inlineBlock(() => {
76-
writer.writeLine("origin: pathToFileURL(filePath).toString(),");
77-
writer.writeLine("...testContext,");
78-
}).write(";").newLine();
79-
}
80-
writer.write("try ").inlineBlock(() => {
81-
writer.writeLine(`require(scriptPath);`);
82-
}).write(" catch(err)").block(() => {
83-
writer.writeLine("console.error(err);");
84-
writer.writeLine("process.exit(1);");
85-
});
86-
if (usesDenoTest) {
87-
writer.writeLine(
88-
"await runTestDefinitions(testDefinitions.splice(0, testDefinitions.length), scriptTestContext);",
89-
);
90-
}
91-
}
67+
if (isolateTestFiles) {
68+
writeTestFileRun();
69+
} else {
70+
writer.write("for (const [i, filePath] of filePaths.entries())")
71+
.block(() => {
72+
writer.write("if (i > 0)").block(() => {
73+
writer.writeLine(`console.log("");`);
74+
}).blankLine();
9275

93-
if (options.includeEsModule) {
94-
if (options.includeScriptModule) {
95-
writer.blankLine();
96-
}
97-
writer.writeLine(`const esmPath = "./esm/" + filePath;`);
98-
writer.writeLine(
99-
`console.log("\\nRunning tests in " + pc.underline(esmPath) + "...\\n");`,
100-
);
101-
writer.writeLine(`process.chdir(__dirname + "/esm");`);
102-
if (usesDenoTest) {
103-
writer.write(`const esmTestContext = `).inlineBlock(() => {
104-
writer.writeLine("origin: pathToFileURL(filePath).toString(),");
105-
writer.writeLine("...testContext,");
106-
}).write(";").newLine();
107-
}
108-
writer.writeLine(`await import(esmPath);`);
109-
if (usesDenoTest) {
110-
writer.writeLine(
111-
"await runTestDefinitions(testDefinitions.splice(0, testDefinitions.length), esmTestContext);",
112-
);
113-
}
114-
}
115-
});
76+
writeTestFileRun();
77+
});
78+
}
11679
});
11780
writer.blankLine();
11881

@@ -123,6 +86,96 @@ export function getTestRunnerCode(options: {
12386

12487
writer.writeLine("main();");
12588
return writer.toString();
89+
90+
function writeSpawnPerFile() {
91+
// run each test file in its own process so that the module state of a
92+
// test file doesn't leak into the next one, which is what `deno test`
93+
// does by running each file in its own isolate
94+
writer.writeLine("const fileIndexArg = process.argv[2];");
95+
writer.write("if (fileIndexArg == null)").block(() => {
96+
writer.writeLine(`const { spawnSync } = require("child_process");`);
97+
writer.writeLine("let failed = false;");
98+
writer.write("for (const i of filePaths.keys())").block(() => {
99+
writer.write("if (i > 0)").block(() => {
100+
writer.writeLine(`console.log("");`);
101+
});
102+
writer.writeLine(
103+
"const args = [...process.execArgv, __filename, String(i)];",
104+
);
105+
writer.writeLine(
106+
`const result = spawnSync(process.execPath, args, { stdio: "inherit" });`,
107+
);
108+
writer.write("if (result.error != null)").block(() => {
109+
writer.writeLine("console.error(result.error);");
110+
});
111+
writer.write("if (result.status !== 0)").block(() => {
112+
writer.writeLine("failed = true;");
113+
});
114+
});
115+
writer.write("if (failed)").block(() => {
116+
writer.writeLine("process.exitCode = 1;");
117+
});
118+
writer.writeLine("return;");
119+
}).blankLine();
120+
writer.writeLine("const filePath = filePaths[Number(fileIndexArg)];");
121+
writer.write("if (filePath == null)").block(() => {
122+
writer.writeLine(
123+
`console.error("Unknown test file index: " + fileIndexArg);`,
124+
);
125+
writer.writeLine("process.exitCode = 1;");
126+
writer.writeLine("return;");
127+
}).blankLine();
128+
}
129+
130+
function writeTestFileRun() {
131+
if (options.includeScriptModule) {
132+
writer.writeLine(`const scriptPath = "./script/" + filePath;`);
133+
writer.writeLine(
134+
`console.log("Running tests in " + pc.underline(scriptPath) + "...\\n");`,
135+
);
136+
writer.writeLine(`process.chdir(__dirname + "/script");`);
137+
if (usesDenoTest) {
138+
writer.write(`const scriptTestContext = `).inlineBlock(() => {
139+
writer.writeLine("origin: pathToFileURL(filePath).toString(),");
140+
writer.writeLine("...testContext,");
141+
}).write(";").newLine();
142+
}
143+
writer.write("try ").inlineBlock(() => {
144+
writer.writeLine(`require(scriptPath);`);
145+
}).write(" catch(err)").block(() => {
146+
writer.writeLine("console.error(err);");
147+
writer.writeLine("process.exit(1);");
148+
});
149+
if (usesDenoTest) {
150+
writer.writeLine(
151+
"await runTestDefinitions(testDefinitions.splice(0, testDefinitions.length), scriptTestContext);",
152+
);
153+
}
154+
}
155+
156+
if (options.includeEsModule) {
157+
if (options.includeScriptModule) {
158+
writer.blankLine();
159+
}
160+
writer.writeLine(`const esmPath = "./esm/" + filePath;`);
161+
writer.writeLine(
162+
`console.log("\\nRunning tests in " + pc.underline(esmPath) + "...\\n");`,
163+
);
164+
writer.writeLine(`process.chdir(__dirname + "/esm");`);
165+
if (usesDenoTest) {
166+
writer.write(`const esmTestContext = `).inlineBlock(() => {
167+
writer.writeLine("origin: pathToFileURL(filePath).toString(),");
168+
writer.writeLine("...testContext,");
169+
}).write(";").newLine();
170+
}
171+
writer.writeLine(`await import(esmPath);`);
172+
if (usesDenoTest) {
173+
writer.writeLine(
174+
"await runTestDefinitions(testDefinitions.splice(0, testDefinitions.length), esmTestContext);",
175+
);
176+
}
177+
}
178+
}
126179
}
127180

128181
function getRunTestDefinitionsCode() {

‎mod.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,17 @@ export interface BuildOptions {
141141
* Note that `node_modules` directories are never searched.
142142
*/
143143
testPattern?: string;
144+
/** How to isolate the test files from each other.
145+
*
146+
* * `"process"` - Run each test file in its own process, which is what
147+
* `deno test` does by running each file in its own isolate. Use this when
148+
* the module state of a test file affects the next one (ex. the global
149+
* hooks of `@std/testing/bdd`).
150+
* * `"none"` - Run all the test files in the same process, which is faster
151+
* because it doesn't start a process per test file.
152+
* @default "none"
153+
*/
154+
testIsolation?: "process" | "none";
144155
/** Path to a module to load before running the tests. Ex. `./scripts/test_preload.ts`
145156
*
146157
* This is useful for setting up the Node.js environment the tests run in
@@ -814,6 +825,7 @@ export async function build(options: BuildOptions): Promise<void> {
814825
path.join(options.outDir, "test_runner.cjs"),
815826
transformCodeToTarget(
816827
getTestRunnerCode({
828+
testIsolation: options.testIsolation,
817829
denoTestShimPackageName: denoTestShimPackage == null
818830
? undefined
819831
: denoTestShimPackage.name === "@deno/shim-deno"

‎tests/integration.test.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1516,6 +1516,22 @@ Deno.test("using declaration project", async () => {
15161516
});
15171517
});
15181518

1519+
Deno.test("should run each test file in its own process", async () => {
1520+
await runTest("test_hooks_project", {
1521+
entryPoints: ["mod.ts"],
1522+
outDir: "./npm",
1523+
testIsolation: "process",
1524+
typeCheck: false,
1525+
shims: {
1526+
deno: "dev",
1527+
},
1528+
package: {
1529+
name: "hooks",
1530+
version: "0.0.0",
1531+
},
1532+
});
1533+
});
1534+
15191535
Deno.test("should build jsr project", async () => {
15201536
await runTest("jsr_project", {
15211537
entryPoints: ["mod.ts"],
@@ -1758,6 +1774,7 @@ async function runTest(
17581774
| "undici_project"
17591775
| "shim_project"
17601776
| "test_preload_project"
1777+
| "test_hooks_project"
17611778
| "test_project"
17621779
| "tla_project"
17631780
| "types_package_project"

‎tests/test_hooks_project/a.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
// Copyright 2018-2024 the Deno authors. MIT license.
2+
3+
import { describe, it } from "jsr:@std/testing@^1.0.19/bdd";
4+
import { add } from "./mod.ts";
5+
6+
describe("a", () => {
7+
it("adds", () => {
8+
if (add(1, 1) !== 2) {
9+
throw new Error("Failed.");
10+
}
11+
});
12+
});

‎tests/test_hooks_project/b.test.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// Copyright 2018-2024 the Deno authors. MIT license.
2+
3+
import { beforeEach, describe, it } from "jsr:@std/testing@^1.0.19/bdd";
4+
import { add } from "./mod.ts";
5+
6+
let value = 0;
7+
8+
// a global hook errors when another test file already registered a global
9+
// test, so this only works when each test file runs in its own process
10+
beforeEach(() => {
11+
value = 1;
12+
});
13+
14+
describe("b", () => {
15+
it("adds", () => {
16+
if (add(value, 1) !== 2) {
17+
throw new Error("Failed.");
18+
}
19+
});
20+
});

‎tests/test_hooks_project/mod.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// Copyright 2018-2024 the Deno authors. MIT license.
2+
3+
export function add(a: number, b: number) {
4+
return a + b;
5+
}

0 commit comments

Comments
 (0)