|
1 | 1 | #!/usr/bin/env node |
2 | 2 |
|
3 | 3 | /** |
4 | | - * Weekly benchmark harness for the 22 executive-assistant scenarios and 15 |
5 | | - * connector certification scenarios. |
| 4 | + * Weekly benchmark harness for the executive-assistant and connector |
| 5 | + * certification scenarios. |
6 | 6 | * |
7 | 7 | * Loads scenario ids from the filesystem (test/scenarios/executive-assistant/ |
8 | 8 | * and test/scenarios/connector-certification/), invokes the |
@@ -34,6 +34,10 @@ const CONNECTOR_DIR = path.join( |
34 | 34 | "scenarios", |
35 | 35 | "connector-certification", |
36 | 36 | ); |
| 37 | +const SCENARIO_FILE_GLOBS = [ |
| 38 | + "test/scenarios/executive-assistant/*.scenario.ts", |
| 39 | + "test/scenarios/connector-certification/*.scenario.ts", |
| 40 | +]; |
37 | 41 | const REPORT_JSON = path.join( |
38 | 42 | REPO_ROOT, |
39 | 43 | "artifacts", |
@@ -95,14 +99,18 @@ const runnerEnv = { |
95 | 99 | }; |
96 | 100 |
|
97 | 101 | console.log( |
98 | | - `[benchmark] invoking scenario-runner for ${scenariosToRun.length} scenarios (threshold=${runnerEnv.LIFEOPS_JUDGE_THRESHOLD})`, |
| 102 | + `[benchmark] invoking scenario-runner for ${scenariosToRun.length} scenarios (threshold=${runnerEnv.LIFEOPS_JUDGE_THRESHOLD}, globs=${SCENARIO_FILE_GLOBS.join(",")})`, |
99 | 103 | ); |
100 | 104 |
|
101 | | -const result = spawnSync("node", ["scripts/run-live-scenarios.mjs"], { |
102 | | - cwd: REPO_ROOT, |
103 | | - env: runnerEnv, |
104 | | - stdio: "inherit", |
105 | | -}); |
| 105 | +const result = spawnSync( |
| 106 | + "node", |
| 107 | + ["scripts/run-live-scenarios.mjs", ...SCENARIO_FILE_GLOBS], |
| 108 | + { |
| 109 | + cwd: REPO_ROOT, |
| 110 | + env: runnerEnv, |
| 111 | + stdio: "inherit", |
| 112 | + }, |
| 113 | +); |
106 | 114 |
|
107 | 115 | // Runner exits non-zero on failure; we still want to emit a report. |
108 | 116 | const runnerExitCode = result.status ?? 1; |
@@ -153,13 +161,13 @@ function renderMarkdown() { |
153 | 161 | ); |
154 | 162 | } |
155 | 163 | lines.push(""); |
156 | | - lines.push("## EA scenarios (22)"); |
| 164 | + lines.push(`## EA scenarios (${eaScenarios.length})`); |
157 | 165 | lines.push(""); |
158 | 166 | for (const entry of eaScenarios) { |
159 | 167 | lines.push(`- \`${entry.id}\``); |
160 | 168 | } |
161 | 169 | lines.push(""); |
162 | | - lines.push("## Connector scenarios (15)"); |
| 170 | + lines.push(`## Connector scenarios (${connectorScenarios.length})`); |
163 | 171 | lines.push(""); |
164 | 172 | for (const entry of connectorScenarios) { |
165 | 173 | lines.push(`- \`${entry.id}\``); |
|
0 commit comments