Skip to content

Commit 2c077a5

Browse files
author
Shaw
committed
fix(ci): scope benchmark scenario discovery
1 parent f9ce440 commit 2c077a5

1 file changed

Lines changed: 18 additions & 10 deletions

File tree

scripts/run-scenario-benchmark.mjs

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
#!/usr/bin/env node
22

33
/**
4-
* Weekly benchmark harness for the 22 executive-assistant scenarios and 15
5-
* connector certification scenarios.
4+
* Weekly benchmark harness for the executive-assistant and connector
5+
* certification scenarios.
66
*
77
* Loads scenario ids from the filesystem (test/scenarios/executive-assistant/
88
* and test/scenarios/connector-certification/), invokes the
@@ -34,6 +34,10 @@ const CONNECTOR_DIR = path.join(
3434
"scenarios",
3535
"connector-certification",
3636
);
37+
const SCENARIO_FILE_GLOBS = [
38+
"test/scenarios/executive-assistant/*.scenario.ts",
39+
"test/scenarios/connector-certification/*.scenario.ts",
40+
];
3741
const REPORT_JSON = path.join(
3842
REPO_ROOT,
3943
"artifacts",
@@ -95,14 +99,18 @@ const runnerEnv = {
9599
};
96100

97101
console.log(
98-
`[benchmark] invoking scenario-runner for ${scenariosToRun.length} scenarios (threshold=${runnerEnv.LIFEOPS_JUDGE_THRESHOLD})`,
102+
`[benchmark] invoking scenario-runner for ${scenariosToRun.length} scenarios (threshold=${runnerEnv.LIFEOPS_JUDGE_THRESHOLD}, globs=${SCENARIO_FILE_GLOBS.join(",")})`,
99103
);
100104

101-
const result = spawnSync("node", ["scripts/run-live-scenarios.mjs"], {
102-
cwd: REPO_ROOT,
103-
env: runnerEnv,
104-
stdio: "inherit",
105-
});
105+
const result = spawnSync(
106+
"node",
107+
["scripts/run-live-scenarios.mjs", ...SCENARIO_FILE_GLOBS],
108+
{
109+
cwd: REPO_ROOT,
110+
env: runnerEnv,
111+
stdio: "inherit",
112+
},
113+
);
106114

107115
// Runner exits non-zero on failure; we still want to emit a report.
108116
const runnerExitCode = result.status ?? 1;
@@ -153,13 +161,13 @@ function renderMarkdown() {
153161
);
154162
}
155163
lines.push("");
156-
lines.push("## EA scenarios (22)");
164+
lines.push(`## EA scenarios (${eaScenarios.length})`);
157165
lines.push("");
158166
for (const entry of eaScenarios) {
159167
lines.push(`- \`${entry.id}\``);
160168
}
161169
lines.push("");
162-
lines.push("## Connector scenarios (15)");
170+
lines.push(`## Connector scenarios (${connectorScenarios.length})`);
163171
lines.push("");
164172
for (const entry of connectorScenarios) {
165173
lines.push(`- \`${entry.id}\``);

0 commit comments

Comments
 (0)