|
| 1 | +import assert from "node:assert/strict"; |
| 2 | +import { spawnSync } from "node:child_process"; |
| 3 | +import { existsSync, readFileSync } from "node:fs"; |
| 4 | +import path from "node:path"; |
| 5 | +import test from "node:test"; |
| 6 | + |
| 7 | +import { PROVIDER_COLLECTORS } from "../scripts/agent-customize/providers/index.mjs"; |
| 8 | +import { createAnalyzer, SESSION_ANALYSIS_HELP } from "../scripts/session-analysis/index.mjs"; |
| 9 | + |
| 10 | +// Canonical support declaration (roadmap A-06): CLI help, provider registry, |
| 11 | +// session platforms, report platforms, and docs must all agree on this set. |
| 12 | +const SUPPORTED_PLATFORMS = ["qoder", "codex", "claude", "cursor", "qwen"]; |
| 13 | + |
| 14 | +const cliPath = path.join(process.cwd(), "scripts", "better-harness.mjs"); |
| 15 | +const adapterMatrixPath = path.join(process.cwd(), "docs", "adapters", "README.md"); |
| 16 | + |
| 17 | +function runBetterHarness(args) { |
| 18 | + return spawnSync(process.execPath, [cliPath, ...args], { |
| 19 | + cwd: process.cwd(), |
| 20 | + encoding: "utf8", |
| 21 | + }); |
| 22 | +} |
| 23 | + |
| 24 | +function sortedSet(values) { |
| 25 | + return [...new Set(values)].sort(); |
| 26 | +} |
| 27 | + |
| 28 | +function assertSameSet(actual, label) { |
| 29 | + assert.deepEqual(sortedSet(actual), sortedSet(SUPPORTED_PLATFORMS), `${label} disagrees with the supported platform set`); |
| 30 | +} |
| 31 | + |
| 32 | +test("agent-customize provider registry declares exactly the supported platforms", () => { |
| 33 | + assertSameSet([...PROVIDER_COLLECTORS.keys()], "PROVIDER_COLLECTORS"); |
| 34 | + |
| 35 | + for (const platform of SUPPORTED_PLATFORMS) { |
| 36 | + const providerModule = path.join(process.cwd(), "scripts", "agent-customize", "providers", `${platform}.mjs`); |
| 37 | + assert.ok(existsSync(providerModule), `missing configured-asset provider module: ${providerModule}`); |
| 38 | + } |
| 39 | +}); |
| 40 | + |
| 41 | +test("session-analysis platform loader declares exactly the supported platforms", async () => { |
| 42 | + for (const platform of SUPPORTED_PLATFORMS) { |
| 43 | + const platformModule = path.join(process.cwd(), "scripts", "session-analysis", "platforms", `${platform}.mjs`); |
| 44 | + assert.ok(existsSync(platformModule), `missing session platform module: ${platformModule}`); |
| 45 | + } |
| 46 | + |
| 47 | + let message = ""; |
| 48 | + try { |
| 49 | + await createAnalyzer("__unsupported__"); |
| 50 | + } catch (error) { |
| 51 | + message = error.message; |
| 52 | + } |
| 53 | + const declared = message.match(/Supported platforms: ([a-z, ]+)\./u)?.[1]; |
| 54 | + assert.ok(declared, `platform loader did not fail closed with a supported list: ${message}`); |
| 55 | + assertSameSet(declared.split(", "), "session-analysis loadPlatform error"); |
| 56 | + |
| 57 | + const declaredHelp = SESSION_ANALYSIS_HELP.match(/--platform <([a-z|]+)>/u)?.[1]; |
| 58 | + assert.ok(declaredHelp, `exported session-analysis help does not declare a platform list:\n${SESSION_ANALYSIS_HELP}`); |
| 59 | + assertSameSet(declaredHelp.split("|"), "SESSION_ANALYSIS_HELP platform list"); |
| 60 | +}); |
| 61 | + |
| 62 | +test("session-analysis CLI help and platform gate agree with the supported platforms", () => { |
| 63 | + const result = runBetterHarness(["session-analysis", "--help"]); |
| 64 | + assert.equal(result.status, 0, result.stderr); |
| 65 | + |
| 66 | + const declared = result.stdout.match(/--platform <([a-z|]+)>/u)?.[1]; |
| 67 | + assert.ok(declared, `session-analysis help does not declare a platform list:\n${result.stdout}`); |
| 68 | + assertSameSet(declared.split("|"), "session-analysis --help platform list"); |
| 69 | + |
| 70 | + const gated = runBetterHarness(["session-analysis", "sources", "--platform", "__unsupported__", "--workspace", "."]); |
| 71 | + assert.notEqual(gated.status, 0, "session-analysis CLI accepted an unsupported platform"); |
| 72 | + const gateDeclared = `${gated.stderr}${gated.stdout}`.match(/Supported platforms: ([a-z, ]+)\./u)?.[1]; |
| 73 | + assert.ok(gateDeclared, `session-analysis CLI did not fail closed with a supported list:\n${gated.stderr}`); |
| 74 | + assertSameSet(gateDeclared.split(", "), "session-analysis CLI platform gate"); |
| 75 | +}); |
| 76 | + |
| 77 | +test("harness analyze help and platform gate agree with the supported platforms", () => { |
| 78 | + const help = runBetterHarness(["harness", "analyze", "--help"]); |
| 79 | + assert.equal(help.status, 0, help.stderr); |
| 80 | + |
| 81 | + const declared = help.stdout.match(/--platform <name>\s+([a-z, ]+or [a-z]+)/u)?.[1]; |
| 82 | + assert.ok(declared, `harness analyze help does not declare a platform list:\n${help.stdout}`); |
| 83 | + assertSameSet(declared.match(/[a-z]+/gu).filter((word) => word !== "or"), "harness analyze --help platform list"); |
| 84 | + |
| 85 | + const gated = runBetterHarness(["harness", "analyze", "--platform", "__unsupported__", "--workspace", ".", "--format", "json"]); |
| 86 | + assert.notEqual(gated.status, 0, "harness analyze accepted an unsupported platform"); |
| 87 | + assert.match(`${gated.stderr}${gated.stdout}`, /unsupported Harness report platform/u); |
| 88 | +}); |
| 89 | + |
| 90 | +test("asset-baseline provider gate lists exactly the supported platforms", () => { |
| 91 | + const result = runBetterHarness(["coding-agent-practices", "asset-baseline", "__unsupported__", "--workspace", "."]); |
| 92 | + assert.notEqual(result.status, 0, "asset-baseline accepted an unsupported provider"); |
| 93 | + |
| 94 | + const declared = `${result.stderr}${result.stdout}`.match(/Supported providers: ([a-z, ]+)\./u)?.[1]; |
| 95 | + assert.ok(declared, `asset-baseline did not fail closed with a supported list:\n${result.stderr}`); |
| 96 | + assertSameSet(declared.split(", "), "asset-baseline provider gate"); |
| 97 | +}); |
| 98 | + |
| 99 | +test("host adapter matrix documents exactly the supported platforms", () => { |
| 100 | + const matrix = readFileSync(adapterMatrixPath, "utf8"); |
| 101 | + |
| 102 | + for (const platform of SUPPORTED_PLATFORMS) { |
| 103 | + assert.ok( |
| 104 | + matrix.includes(`scripts/agent-customize/providers/${platform}.mjs`), |
| 105 | + `adapter matrix is missing the configured-asset provider for ${platform}`, |
| 106 | + ); |
| 107 | + assert.ok( |
| 108 | + matrix.includes(`scripts/session-analysis/platforms/${platform}.mjs`), |
| 109 | + `adapter matrix is missing the session platform for ${platform}`, |
| 110 | + ); |
| 111 | + } |
| 112 | + |
| 113 | + const documentedProviders = [...matrix.matchAll(/agent-customize\/providers\/([a-z-]+)\.mjs/gu)].map((match) => match[1]); |
| 114 | + const documentedPlatforms = [...matrix.matchAll(/session-analysis\/platforms\/([a-z-]+)\.mjs/gu)].map((match) => match[1]); |
| 115 | + assertSameSet(documentedProviders, "adapter matrix configured-asset providers"); |
| 116 | + assertSameSet(documentedPlatforms, "adapter matrix session platforms"); |
| 117 | +}); |
0 commit comments