diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ab24a2..5ef77fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,33 +3,84 @@ name: CI on: pull_request: push: - branches: - - master + branches: [master] + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true permissions: contents: read jobs: - test: - name: Build and test - runs-on: ubuntu-latest - + compatibility: + name: Test (Node 20.11.0) + runs-on: ubuntu-24.04 steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - - - name: Setup Node - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 with: - node-version-file: .nvmrc + node-version: 20.11.0 + package-manager-cache: false + - run: yarn install --immutable + - run: yarn build && yarn vitest run && yarn test:runner-cli-smoke - - name: Install dependencies - run: printf 'y\n' | NPQ_PKG_MGR=yarn npx --yes npq install - env: - GITHUB_TOKEN: ${{ github.token }} - - - name: Build - run: yarn build + package: + name: Test and package (Node 24) + runs-on: ubuntu-24.04 + outputs: + sha512: ${{ steps.pack.outputs.sha512 }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: 24 + package-manager-cache: false + - run: yarn install --immutable + - run: yarn quality + - run: yarn build && yarn vitest run && yarn test:runner-cli-smoke && yarn test:release-contracts + - id: pack + run: | + mkdir retained + filename="$(npm pack --pack-destination retained)" + mv "retained/$filename" retained/pathgrade.tgz + echo "sha512=$(sha512sum retained/pathgrade.tgz | cut -d ' ' -f 1)" >> "$GITHUB_OUTPUT" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: pathgrade-package + path: retained/pathgrade.tgz + if-no-files-found: error - - name: Test - run: yarn test + package-smoke: + name: Package smoke (${{ matrix.runner }}, Node ${{ matrix.node }}) + needs: [compatibility, package] + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - { runner: ubuntu-24.04, node: 20.11.0 } + - { runner: ubuntu-24.04, node: 24 } + - { runner: ubuntu-24.04-arm, node: 24 } + - { runner: macos-15, node: 24 } + - { runner: macos-15-intel, node: 24 } + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: ${{ matrix.node }} + package-manager-cache: false + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: pathgrade-package + path: retained + - run: node tests/standalone-package-smoke/run-smoke.mjs --tarball "$PWD/retained/pathgrade.tgz" --expected-sha512 "${{ needs.package.outputs.sha512 }}" + - run: | + prefix="$RUNNER_TEMP/pathgrade-prefix" + npm install --global --prefix "$prefix" --ignore-scripts retained/pathgrade.tgz + graph="$prefix/lib/node_modules/@wix/pathgrade/node_modules" + claude="$(find "$graph" -type f -name claude -perm -111 -print -quit)" + codex="$(find "$graph" -type f -name codex -perm -111 -print -quit)" + test -x "$claude" && test -x "$codex" + "$claude" --version + "$codex" --version diff --git a/README.md b/README.md index 58c6b87..1bd58df 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,27 @@ - Debug long conversations with preserved workspaces and run snapshots - Use the same evals locally and in CI +## Standalone—Node only + +Run a TypeScript eval without adding Pathgrade, Vitest, Claude, or Codex to the target project: + +```text +npx @wix/pathgrade standalone +npx @wix/pathgrade standalone run path/to/example.eval.ts +npm install --global @wix/pathgrade +pathgrade standalone +``` + +Standalone v0 supports Node.js 22 or 24 on macOS, Linux, or WSL (`x64` and `arm64`). It runs Claude through the bundled Claude Agent SDK and Claude Code binary, or Codex through the bundled Codex `app-server`; credentials and network access for the selected provider are still required. No local Pathgrade, Vitest, Claude, or Codex install is used, and there is no fallback to `PATH`, `npx`, or a target `node_modules` for those runtimes. + +Evals may import from root `vitest`, `@wix/pathgrade`, and the documented standalone Pathgrade eval subpaths. Vitest subpaths such as `vitest/config` are not supported in standalone v0. Application dependencies remain your responsibility and must be installed or otherwise resolvable by the target project. + +Standalone ignores project Vitest configuration and does not load `.env`. Its temporary HOME, cache, and workspace isolation prevents accidental project mutation, but local workspace isolation is not a security sandbox. Jest, Cursor, Codex `exec`, Docker, declarative specs, recording, and baselines are deferred beyond standalone v0. + +Standalone uses a compact, color-aware progress view in interactive terminals and stable line-oriented output in CI or redirected streams. Use `--verbose` for an agent-labeled live trace, `--quiet` for failures and the final status only, or `--diagnostics` for expanded final diagnostics. `--quiet` and `--verbose` are mutually exclusive. Color is semantic and never replaces status text; `NO_COLOR` or `FORCE_COLOR=0` disables it. + +Existing project-local `@wix/pathgrade` commands remain unchanged unless you select the explicit `standalone` namespace; project-local configuration, `.env`, adapters, and executable overrides retain their current behavior. This release installs and publishes only `@wix/pathgrade`: no unscoped `pathgrade` package is installed or published. + ## Quick Start **Prerequisites**: Node.js 20.11+, Vitest 4+ or Jest 30+, and at least one configured agent runtime. Claude uses the bundled `@anthropic-ai/claude-agent-sdk` binary by default; Codex requires the `codex` CLI; Cursor requires the `cursor-agent` CLI. diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 7ede21a..dab55ed 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -48,7 +48,7 @@ Pathgrade evaluates whether AI agents correctly discover and use your skills. Yo ## Installation -**Prerequisites**: Node.js 20+, Vitest 4+ or Jest 30+ +**Project-local prerequisites**: Node.js 20+, Vitest 4+ or Jest 30+. These prerequisites and the runtime overrides below apply to the existing project-local commands. ```bash npm i @wix/pathgrade vitest @@ -66,6 +66,23 @@ npm i @wix/pathgrade jest - **Codex**: install the Codex CLI per OpenAI's documentation; pathgrade shells out to it. - **Cursor**: install `cursor-agent` per Cursor's documentation; pathgrade shells out to it. +### Standalone—Node only + +Standalone v0 has a separate contract: Node.js 22 or 24 on macOS, Linux, or WSL (`x64` or `arm64`). It bundles Pathgrade, Vitest, the Claude Agent SDK and Claude Code binary, and the Codex `app-server`, so it needs no local Pathgrade, Vitest, Claude, or Codex install: + +```text +npx @wix/pathgrade standalone +npx @wix/pathgrade standalone run path/to/example.eval.ts +npm install --global @wix/pathgrade +pathgrade standalone +``` + +Credentials and network access remain required for live Claude or Codex requests. Standalone v0 supports imports from root `vitest`, not Vitest subpaths such as `vitest/config`; application dependencies remain your responsibility. The project Vitest configuration is ignored, and standalone does not load `.env`. + +Temporary HOME, cache, and workspace directories isolate local state, but local workspace isolation is not a security sandbox. Jest, Cursor, Codex `exec`, Docker, declarative specs, recording, and baselines are deferred. Existing project-local commands remain unchanged unless the explicit `standalone` namespace is selected, and this release does not install or publish an unscoped `pathgrade` package. + +Interactive standalone runs use the compact Calm Runner view. CI and redirected streams automatically receive stable line-oriented output. Pass `--verbose` for an agent-labeled Live Trace, `--quiet` to keep only failures and the authoritative final status, or `--diagnostics` to expand final diagnostics. `--quiet` and `--verbose` cannot be combined. Semantic color is optional: set `NO_COLOR` or `FORCE_COLOR=0` to disable it without changing the displayed status words. + ## Quick Start 1. Create a `vitest.config.ts` with the Pathgrade Vitest adapter plugin: @@ -1276,6 +1293,8 @@ The pathgrade vitest plugin uses `setRuntime({ onResult })` internally to captur ## Environment Variables +This section describes project-local execution. `pathgrade run` loads the project's `.env` as before. In contrast, `pathgrade standalone` does not load `.env`; provide selected-provider credentials in the invoking process environment. Standalone also ignores local runtime executable overrides and project Vitest configuration. + | Variable | Used By | |----------|---------| | `ANTHROPIC_API_KEY` | Claude agent, judge scorers, persona replies | diff --git a/knip.json b/knip.json index 48b3ec9..1459736 100644 --- a/knip.json +++ b/knip.json @@ -9,7 +9,8 @@ "vitest.config.ts", "evals/vitest.config.mts", "examples/**/vitest.config.mts", - "scripts/*.ts" + "scripts/*.ts", + "scripts/release/*.mjs" ], "project": [ "src/**/*.ts", @@ -17,12 +18,14 @@ "evals/**/*.ts", "examples/**/*.ts", "scripts/**/*.ts", + "scripts/**/*.mjs", "scorers/**/*.ts" ], "ignore": [ "tests/fixtures/**" ], "ignoreDependencies": [ - "jest" + "jest", + "@openai/codex" ] } diff --git a/package.json b/package.json index 747d161..f933da0 100644 --- a/package.json +++ b/package.json @@ -75,8 +75,10 @@ "quality:boundaries": "scripts/check-quality-boundaries.sh", "quality:max-lines": "scripts/check-max-lines.sh .", "quality": "yarn lint && yarn quality:deps && yarn quality:arch && yarn quality:boundaries && yarn quality:max-lines", - "test": "yarn build && vitest run && yarn test:runner-cli-smoke", + "test": "yarn build && vitest run && yarn test:runner-cli-smoke && yarn test:standalone-package-smoke && yarn test:release-contracts", "test:runner-cli-smoke": "node tests/runner-cli-smoke/run-smokes.mjs", + "test:standalone-package-smoke": "node tests/standalone-package-smoke/run-smoke.mjs", + "test:release-contracts": "node tests/release-platform-evidence.test.mjs && node tests/publish-workflow-contract.test.mjs", "test:evals": "vitest run --config evals/vitest.config.mts", "test:coverage": "vitest run --coverage", "dev": "tsx src/pathgrade.ts", @@ -118,18 +120,19 @@ "@types/jest": "^30.0.0", "@types/picomatch": "^4.0.2", "@vitest/coverage-v8": "4.1.7", - "jest": "^30.0.0", - "vitest": "4.1.7" + "jest": "^30.0.0" }, "dependencies": { "@anthropic-ai/claude-agent-sdk": "0.2.116", "@modelcontextprotocol/sdk": "1.29.0", + "@openai/codex": "0.144.0", "@types/node": "25.6.0", "fs-extra": "11.3.3", "jiti": "2.6.1", "picomatch": "^4.0.4", "tsx": "4.22.3", "typescript": "^5.9.3", + "vitest": "4.1.7", "zod": "4.3.6" }, "resolutions": { diff --git a/scripts/check-max-lines.sh b/scripts/check-max-lines.sh index d1bb51a..9b23d0b 100755 --- a/scripts/check-max-lines.sh +++ b/scripts/check-max-lines.sh @@ -15,12 +15,12 @@ esac allowlist_cap() { case "$1" in - src/agents/codex-app-server/agent.ts) echo 811 ;; + src/agents/codex-app-server/agent.ts) echo 865 ;; src/viewer.html) echo 1178 ;; tests/claude-ask-user-bridge.test.ts) echo 604 ;; tests/claude-sdk-driver.test.ts) echo 697 ;; tests/claude-sdk-projector.test.ts) echo 774 ;; - tests/codex-app-server-agent.test.ts) echo 1285 ;; + tests/codex-app-server-agent.test.ts) echo 1437 ;; tests/commands.run-changed.test.ts) echo 609 ;; tests/converse.test.ts) echo 608 ;; tests/grading-pipeline.test.ts) echo 691 ;; @@ -60,7 +60,7 @@ should_check() { emit_files() { if git -C "$root" rev-parse --is-inside-work-tree >/dev/null 2>&1; then - git -C "$root" ls-files -z + git -C "$root" ls-files --cached --others --exclude-standard -z else find "$root" \ \( -path "*/.git" -o -path "*/node_modules" -o -path "*/dist" -o -path "*/build" -o -path "*/coverage" -o -path "*/.next" -o -path "*/.nuxt" -o -path "*/.turbo" -o -path "*/.cache" \) -prune \ @@ -79,6 +79,7 @@ while IFS= read -r -d '' file; do path="$file" fi + [[ -f "$path" ]] || continue should_skip "$display" && continue should_check "$display" || continue diff --git a/scripts/check-quality-boundaries.sh b/scripts/check-quality-boundaries.sh index b011648..36ef03b 100755 --- a/scripts/check-quality-boundaries.sh +++ b/scripts/check-quality-boundaries.sh @@ -4,6 +4,22 @@ set -euo pipefail root="${1:-.}" cd "$root" +release_scripts=() +while IFS= read -r file; do + release_scripts+=("$file") +done < <(find scripts/release -type f -name '*.mjs' 2>/dev/null | sort) + +for file in "${release_scripts[@]}"; do + if ! node --check "$file"; then + echo "Release script failed syntax validation: $file" >&2 + exit 1 + fi +done + +if (( ${#release_scripts[@]} > 0 )); then + echo "Release-script syntax boundaries are clean (${#release_scripts[@]} files)." +fi + result_capture_files=() while IFS= read -r file; do result_capture_files+=("$file") diff --git a/scripts/release/verify-platform-evidence.mjs b/scripts/release/verify-platform-evidence.mjs new file mode 100644 index 0000000..67f1308 --- /dev/null +++ b/scripts/release/verify-platform-evidence.mjs @@ -0,0 +1,212 @@ +#!/usr/bin/env node +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const expectedTuples = new Map([ + ['darwin-arm64-node24', { platform: 'darwin', arch: 'arm64' }], + ['darwin-x64-node24', { platform: 'darwin', arch: 'x64' }], + ['linux-arm64-node24', { platform: 'linux', arch: 'arm64' }], + ['linux-x64-node24', { platform: 'linux', arch: 'x64' }], + ['wsl-x64-node24', { platform: 'linux', arch: 'x64' }], +]); +const expectedRuntimes = { + claude: { sdk_version: '0.2.116', claude_code_version: '2.1.116' }, + codex: { package_version: '0.144.0', native_version: '0.144.0' }, +}; + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + try { + const options = parseArgs(process.argv.slice(2)); + const result = verifyEvidence(options); + process.stdout.write(`${JSON.stringify(result)}\n`); + } catch (error) { + process.stderr.write(`platform evidence rejected: ${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + } +} + +export function verifyEvidence({ commit, tarballSha512, evidenceDir, liveEvidenceDir }) { + requireValue('commit', commit); + requireSha512(tarballSha512); + if (!evidenceDir || !path.isAbsolute(evidenceDir)) throw new Error('evidence-dir must be an absolute path'); + if (!fs.statSync(evidenceDir, { throwIfNoEntry: false })?.isDirectory()) { + throw new Error(`evidence-dir does not exist: ${evidenceDir}`); + } + + const records = new Map(); + for (const filename of jsonFiles(evidenceDir)) { + let record; + try { + record = JSON.parse(fs.readFileSync(filename, 'utf8')); + } catch { + throw new Error(`${path.basename(filename)} contains malformed JSON`); + } + if (!record || typeof record !== 'object' || Array.isArray(record)) { + throw new Error(`${path.basename(filename)} must contain one evidence object`); + } + const tupleId = record.tuple_id; + if (typeof tupleId !== 'string' || !expectedTuples.has(tupleId)) { + throw new Error(`unknown tuple_id ${String(tupleId)}`); + } + if (records.has(tupleId)) throw new Error(`duplicate ${tupleId}`); + records.set(tupleId, record); + } + + for (const tupleId of expectedTuples.keys()) { + if (!records.has(tupleId)) throw new Error(`missing ${tupleId}`); + } + for (const [tupleId, expected] of expectedTuples) { + verifyRecord(tupleId, records.get(tupleId), expected, { commit, tarballSha512 }); + } + if (liveEvidenceDir) verifyLiveEvidence(liveEvidenceDir, { commit, tarballSha512 }); + return { + status: 'pass', tuple_ids: [...expectedTuples.keys()], + ...(liveEvidenceDir ? { live_provider_ids: ['linux-claude', 'linux-codex', 'macos-codex'] } : {}), + }; +} + +function verifyRecord(tupleId, record, expected, release) { + equalField(tupleId, 'status', record.status, 'pass'); + equalField(tupleId, 'platform', record.platform, expected.platform); + equalField(tupleId, 'arch', record.arch, expected.arch); + equalField(tupleId, 'node_major', record.node_major, 24); + equalField(tupleId, 'package.name', record.package?.name, '@wix/pathgrade'); + equalField(tupleId, 'package.version', record.package?.version, packageVersion()); + equalField(tupleId, 'source_commit', record.source_commit, release.commit); + equalField(tupleId, 'tarball_sha512', record.tarball_sha512, release.tarballSha512); + equalField(tupleId, 'runtimes.claude.sdk_version', record.runtimes?.claude?.sdk_version, expectedRuntimes.claude.sdk_version); + equalField(tupleId, 'runtimes.claude.claude_code_version', record.runtimes?.claude?.claude_code_version, expectedRuntimes.claude.claude_code_version); + equalField(tupleId, 'runtimes.codex.package_version', record.runtimes?.codex?.package_version, expectedRuntimes.codex.package_version); + equalField(tupleId, 'runtimes.codex.native_version', record.runtimes?.codex?.native_version, expectedRuntimes.codex.native_version); + const wsl = tupleId === 'wsl-x64-node24'; + equalField(tupleId, 'runtime_environment.kind', record.runtime_environment?.kind, wsl ? 'wsl' : 'github-hosted'); + equalField(tupleId, 'runtime_environment.observed_platform', record.runtime_environment?.observed_platform, expected.platform); + equalField(tupleId, 'runtime_environment.observed_arch', record.runtime_environment?.observed_arch, expected.arch); + equalField(tupleId, 'runtime_environment.wsl', record.runtime_environment?.wsl, wsl); + if (wsl) { + if (!/microsoft|wsl/i.test(record.runtime_environment?.kernel_release ?? '')) { + throw new Error(`${tupleId} runtime_environment.kernel_release must identify WSL`); + } + if (typeof record.runtime_environment?.wsl_interop !== 'string' || !record.runtime_environment.wsl_interop) { + throw new Error(`${tupleId} runtime_environment.wsl_interop must be observed`); + } + } +} + +function verifyLiveEvidence(directory, release) { + if (!path.isAbsolute(directory) || !fs.statSync(directory, { throwIfNoEntry: false })?.isDirectory()) { + throw new Error('live-evidence-dir must be an existing absolute directory'); + } + const records = jsonFiles(directory).map(filename => JSON.parse(fs.readFileSync(filename, 'utf8'))); + if (records.length !== 2) throw new Error(`live evidence must contain exactly 2 records, got ${records.length}`); + const byPlatform = new Map(records.map(record => [record.runtime_environment?.platform, record])); + if (byPlatform.size !== 2 || !byPlatform.has('linux') || !byPlatform.has('darwin')) { + throw new Error('live evidence must contain one Linux and one macOS record'); + } + verifyLiveRecord(byPlatform.get('linux'), 'linux', ['claude', 'codex'], release); + verifyLiveRecord(byPlatform.get('darwin'), 'darwin', ['codex'], release); +} + +function verifyLiveRecord(record, platform, selected, release) { + const label = platform === 'darwin' ? 'macos' : 'linux'; + equalLive(label, 'schema', record.schema, 'pathgrade-live-evidence/v1'); + equalLive(label, 'status', record.status, 'pass'); + equalLive(label, 'source_commit', record.source_commit, release.commit); + equalLive(label, 'tarball_sha512', record.tarball_sha512, release.tarballSha512); + equalLive(label, 'runtime_environment.platform', record.runtime_environment?.platform, platform); + equalLive(label, 'runtime_environment.node_major', record.runtime_environment?.node_major, 24); + if (JSON.stringify(record.selected) !== JSON.stringify(selected)) throw new Error(`${label} selected providers differ`); + equalLive(label, 'passed', record.passed, selected.length); + equalLive(label, 'skipped', record.skipped, 2 - selected.length); + if (!Array.isArray(record.providers) || record.providers.length !== 2) { + throw new Error(`${label} providers must contain exactly 2 records`); + } + for (const provider of ['claude', 'codex']) { + const item = record.providers.find(value => value?.provider === provider); + if (!item) throw new Error(`${label}-${provider} record is missing`); + const active = selected.includes(provider); + equalLive(`${label}-${provider}`, 'id', item.id, `${label}-${provider}`); + equalLive(`${label}-${provider}`, 'status', item.status, active ? 'pass' : 'skipped'); + equalLive(`${label}-${provider}`, 'skipped', item.skipped, active ? 0 : 1); + if (active) verifyLiveReport(`${label}-${provider}`, item.report, provider); + else if ('report' in item) throw new Error(`${label}-${provider} skipped record must not contain a report`); + } +} + +function verifyLiveReport(label, report, provider) { + equalLive(label, 'report.version', report?.version, 1); + equalLive(label, 'report.status', report?.status, 'pass'); + equalLive(label, 'report.overall_pass_rate', report?.overall_pass_rate, 1); + equalLive(label, 'report.provenance.mode', report?.provenance?.mode, 'standalone'); + equalLive(label, 'report.provenance.runtimes.claude.sdk_version', report?.provenance?.runtimes?.claude?.sdk_version, '0.2.116'); + equalLive(label, 'report.provenance.runtimes.claude.claude_code_version', report?.provenance?.runtimes?.claude?.claude_code_version, '2.1.116'); + equalLive(label, 'report.provenance.runtimes.codex.package_version', report?.provenance?.runtimes?.codex?.package_version, '0.144.0'); + equalLive(label, 'report.provenance.runtimes.codex.native_version', report?.provenance?.runtimes?.codex?.native_version, '0.144.0'); + if (!Array.isArray(report?.groups) || report.groups.length !== 1 + || !Array.isArray(report.groups[0]?.trials) || report.groups[0].trials.length !== 1) { + throw new Error(`${label} report must contain exactly one group and trial`); + } + const trial = report.groups[0].trials[0]; + equalLive(label, 'reward', trial.reward, 1); + const provenance = trial.agent_provenance; + equalLive(label, 'agent', provenance?.agent, provider); + equalLive(label, 'transport', provenance?.transport, provider === 'claude' ? 'native' : 'app-server'); + equalLive(label, 'authentication', provenance?.authentication, 'api-key'); + equalLive(label, 'model.id', provenance?.model?.id, provider === 'claude' ? null : 'gpt-5.4'); + equalLive(label, 'model.source', provenance?.model?.source, provider === 'claude' ? 'provider-default' : 'pathgrade-default'); + equalLive(label, 'runtime.package', provenance?.runtime?.package, provider === 'claude' ? '@anthropic-ai/claude-agent-sdk' : '@openai/codex'); + equalLive(label, 'runtime.package_version', provenance?.runtime?.package_version, provider === 'claude' ? '0.2.116' : '0.144.0'); + equalLive(label, 'runtime.embedded_binary_version', provenance?.runtime?.embedded_binary_version, provider === 'claude' ? '2.1.116' : '0.144.0'); + equalLive(label, 'runtime.provenance', provenance?.runtime?.provenance, 'bundled'); +} + +function equalLive(label, field, actual, expected) { + if (actual !== expected) throw new Error(`${label} ${field} must be ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`); +} + +function equalField(tupleId, field, actual, expected) { + if (actual !== expected) { + throw new Error(`${tupleId} ${field} must be ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`); + } +} + +function parseArgs(args) { + const allowed = new Set(['--commit', '--tarball-sha512', '--evidence-dir', '--live-evidence-dir']); + const options = {}; + for (let index = 0; index < args.length; index += 2) { + const option = args[index]; + const value = args[index + 1]; + if (!allowed.has(option)) throw new Error(`unknown argument: ${option}`); + if (!value) throw new Error(`missing value for ${option}`); + options[option.slice(2).replaceAll('-', '_')] = value; + } + return { + commit: options.commit, + tarballSha512: options.tarball_sha512, + evidenceDir: options.evidence_dir, + liveEvidenceDir: options.live_evidence_dir, + }; +} + +function jsonFiles(directory) { + return fs.readdirSync(directory, { recursive: true, withFileTypes: true }) + .filter(entry => entry.isFile() && entry.name.endsWith('.json')) + .map(entry => path.join(entry.parentPath, entry.name)) + .sort(); +} + +function requireValue(field, value) { + if (typeof value !== 'string' || value.length === 0) throw new Error(`${field} is required`); +} + +function requireSha512(value) { + if (typeof value !== 'string' || !/^[a-f\d]{128}$/i.test(value)) { + throw new Error('tarball-sha512 must be a 128-character hexadecimal SHA-512'); + } +} + +function packageVersion() { + const packageJson = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../package.json'); + return JSON.parse(fs.readFileSync(packageJson, 'utf8')).version; +} diff --git a/src/adapters/vitest/index.ts b/src/adapters/vitest/index.ts index 6094021..3d7b95c 100644 --- a/src/adapters/vitest/index.ts +++ b/src/adapters/vitest/index.ts @@ -4,6 +4,7 @@ import type { PathgradePluginOptions } from '../../sdk/types.js'; import { PathgradeReporter } from './reporter.js'; import { discoverPathgradeEvalFiles } from '../../evals/discovery.js'; import { DEFAULT_EVAL_EXCLUDE, DEFAULT_EVAL_INCLUDE } from '../../config/pathgrade.js'; +import { isStandaloneMode } from '../../standalone/mode.js'; export type { PathgradePluginOptions }; @@ -50,7 +51,7 @@ export function pathgrade(opts?: PathgradePluginOptions): any { testTimeout: (timeoutSec + 30) * 1000, setupFiles: [resolveSetupFile()], reporters: [ - 'default', + isStandaloneMode(process.env) ? 'minimal' : 'default', new PathgradeReporter(opts), ], }, diff --git a/src/adapters/vitest/reporter.ts b/src/adapters/vitest/reporter.ts index 12aa5c5..f32246e 100644 --- a/src/adapters/vitest/reporter.ts +++ b/src/adapters/vitest/reporter.ts @@ -1,6 +1,6 @@ import * as path from 'path'; import { execSync } from 'child_process'; -import type { Reporter, TestModule } from 'vitest/node'; +import type { Reporter, TestCase, TestModule } from 'vitest/node'; import type { PathgradePluginOptions } from '../../sdk/types.js'; import { fmt } from '../../utils/cli.js'; import { getPathgradeDir } from '../../reporters/results-path.js'; @@ -9,6 +9,10 @@ import { printReportSummary } from '../../reporters/report-summary.js'; import { createVitestAdapter } from '../../runners/vitest-adapter.js'; import { runWithAdapter } from '../../runners/orchestrator.js'; import { resolvePathgradeConfig } from '../../config/pathgrade.js'; +import { isStandaloneMode } from '../../standalone/mode.js'; +import { readStandaloneRunProvenance } from '../../standalone/provenance.js'; +import { createStandaloneUiSink, type StandaloneUiSink } from '../../standalone/ui/protocol.js'; +import type { StandaloneCaseState } from '../../standalone/ui/events.js'; /** * Custom vitest reporter that layers pathgrade aggregate statistics @@ -16,19 +20,70 @@ import { resolvePathgradeConfig } from '../../config/pathgrade.js'; */ export class PathgradeReporter implements Reporter { private opts: PathgradePluginOptions; + private readonly ui: StandaloneUiSink; + private readonly startedAt = Date.now(); + private fileCount = 0; constructor(opts?: PathgradePluginOptions) { this.opts = opts ?? {}; + this.ui = createStandaloneUiSink( + isStandaloneMode(process.env) && (this.opts.reporter ?? 'cli') !== 'json', + ); + } + + onTestRunStart(specifications: ReadonlyArray<{ moduleId?: string }>): void { + const files = specifications.map((specification, index) => ({ + id: specification.moduleId ?? String(index), + name: specification.moduleId ?? `eval ${index + 1}`, + })); + this.fileCount = files.length; + this.ui.emit({ v: 1, type: 'run_start', files, startedAt: this.startedAt }); + } + + onTestCaseReady(testCase: TestCase): void { + this.ui.emit({ + v: 1, + type: 'case_start', + id: testCase.id, + file: testCase.module.relativeModuleId, + name: testCase.name, + startedAt: Date.now(), + }); + } + + onTestCaseResult(testCase: TestCase): void { + const evaluations = testCase.meta().pathgrade as Array<{ score: number }> | undefined; + const score = evaluations?.at(-1)?.score; + const state = normalizeUiState(testCase.result().state); + this.ui.emit({ + v: 1, + type: 'case_finish', + id: testCase.id, + file: testCase.module.relativeModuleId, + name: testCase.name, + state, + durationMs: testCase.diagnostic()?.duration ?? 0, + ...(score === undefined ? {} : { score }), + }); } async onTestRunEnd(testModules: ReadonlyArray): Promise { const cwd = process.cwd(); - const config = await resolvePathgradeConfig({ cwd }); + const config = await resolvePathgradeConfig({ + cwd, + standalone: isStandaloneMode(process.env), + }); const mode = this.opts.reporter ?? config.reporter ?? 'cli'; + const standalone = isStandaloneMode(process.env); const outputDir = getPathgradeDir(cwd); const adapter = createVitestAdapter({ testModules }); + const provenance = standalone + ? readStandaloneRunProvenance(process.env) + : undefined; - const exitCode = await runWithAdapter({ + let exitCode: number; + try { + exitCode = await runWithAdapter({ adapter, options: { cwd, @@ -38,17 +93,22 @@ export class PathgradeReporter implements Reporter { artifactRoot: outputDir, reporterMode: mode, threshold: this.opts.ci?.threshold ?? config.ci.threshold, + provenance, writeEmptyReport: false, warn: warning => console.warn(warning), - log: () => console.log(`\n ${fmt.dim('Results written to')} ${outputDir}\n`), + log: standalone + ? undefined + : () => console.log(`\n ${fmt.dim('Results written to')} ${outputDir}\n`), loadSelection: async () => (await readSidecar(cwd, msg => { console.warn(`[pathgrade] ${msg}`); })) ?? undefined, printSummary: summaries => { + const diagnostics = this.opts.diagnostics === true + || config.diagnostics + || process.env.PATHGRADE_DIAGNOSTICS === '1'; + if (standalone && !diagnostics) return; printReportSummary(summaries, { - forceVerbose: this.opts.diagnostics === true - || config.diagnostics - || process.env.PATHGRADE_DIAGNOSTICS === '1', + forceVerbose: diagnostics, currentTimeoutMs: this.opts.timeout != null ? this.opts.timeout * 1000 : undefined, }); }, @@ -57,12 +117,35 @@ export class PathgradeReporter implements Reporter { const avg = overallPassRate; const configuredThreshold = this.opts.ci?.threshold ?? threshold; process.exitCode = 1; - console.log( - `\n ${fmt.fail('CI THRESHOLD FAILED')} avg score ${fmt.bold(avg.toFixed(3))} < threshold ${fmt.bold(String(configuredThreshold))}\n`, - ); + if (!standalone || mode === 'json') { + console.log( + `\n ${fmt.fail('CI THRESHOLD FAILED')} avg score ${fmt.bold(avg.toFixed(3))} < threshold ${fmt.bold(String(configuredThreshold))}\n`, + ); + } + }, + onArtifactsWritten: ({ built, artifacts }) => { + const states = testModules.flatMap(module => [...module.children.allTests()]) + .map(testCase => normalizeUiState(testCase.result().state)); + this.ui.emit({ + v: 1, + type: 'run_finish', + status: built.report.status, + fileCount: this.fileCount || testModules.length, + passed: states.filter(state => state === 'passed').length, + failed: states.filter(state => state === 'failed').length, + skipped: states.filter(state => state === 'skipped' || state === 'pending').length, + durationMs: Date.now() - this.startedAt, + overallScore: built.report.overall_pass_rate, + ...(built.report.threshold === undefined ? {} : { threshold: built.report.threshold }), + resultsPath: path.relative(cwd, artifacts.resultsPath), + }); }, }, - }); + }); + } catch (error) { + this.ui.emit({ v: 1, type: 'run_error' }); + throw error; + } if (exitCode !== 0 && (process.exitCode === undefined || process.exitCode === 0)) { process.exitCode = exitCode; } @@ -82,3 +165,10 @@ export class PathgradeReporter implements Reporter { } } } + +function normalizeUiState(state: string): StandaloneCaseState { + if (state === 'passed' || state === 'failed' || state === 'skipped' || state === 'pending') { + return state; + } + return 'failed'; +} diff --git a/src/agents/claude-runtime.ts b/src/agents/claude-runtime.ts new file mode 100644 index 0000000..6119725 --- /dev/null +++ b/src/agents/claude-runtime.ts @@ -0,0 +1,104 @@ +import { spawn } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { dirname, join } from 'node:path'; + +const SDK_VERSION = '0.2.116' as const; +const CLAUDE_CODE_VERSION = '2.1.116' as const; +const pathgradeRequire = createRequire(import.meta.url); + +export interface VerifiedBundledClaudeRuntime { + sdkVersion: typeof SDK_VERSION; + embeddedBinaryVersion: typeof CLAUDE_CODE_VERSION; + provenance: 'bundled'; +} + +let verifiedRuntime: VerifiedBundledClaudeRuntime | undefined; + +function packagingDefect(message: string): Error { + return new Error(`Pathgrade packaging defect: bundled Claude ${message}`); +} + +function resolveSdkPackageJson(): string { + try { + return join(dirname(pathgradeRequire.resolve('@anthropic-ai/claude-agent-sdk')), 'package.json'); + } catch { + throw packagingDefect('Agent SDK is unavailable'); + } +} + +function resolveBundledClaudeExecutable(): string { + const suffix = process.platform === 'win32' ? '.exe' : ''; + const packageNames = process.platform === 'linux' + ? [ + `@anthropic-ai/claude-agent-sdk-linux-${process.arch}-musl`, + `@anthropic-ai/claude-agent-sdk-linux-${process.arch}`, + ] + : [`@anthropic-ai/claude-agent-sdk-${process.platform}-${process.arch}`]; + try { + const requireFromSdk = createRequire(resolveSdkPackageJson()); + for (const packageName of packageNames) { + try { + const packageJsonPath = requireFromSdk.resolve(`${packageName}/package.json`); + const executable = join(dirname(packageJsonPath), `claude${suffix}`); + if (existsSync(executable)) return executable; + } catch { + continue; + } + } + } catch { + // Resolve below so every missing artifact gets the same diagnostic. + } + throw packagingDefect(`native artifact for ${process.platform}/${process.arch} is unavailable`); +} + +export async function verifyBundledClaudeRuntime(): Promise { + if (verifiedRuntime) return verifiedRuntime; + + const packageJsonPath = resolveSdkPackageJson(); + let metadata: { version?: unknown; claudeCodeVersion?: unknown }; + try { + metadata = JSON.parse(readFileSync(packageJsonPath, 'utf8')) as { + version?: unknown; + claudeCodeVersion?: unknown; + }; + } catch { + throw packagingDefect('Agent SDK metadata is unreadable'); + } + if (metadata.version !== SDK_VERSION || metadata.claudeCodeVersion !== CLAUDE_CODE_VERSION) { + throw packagingDefect(`versions must be SDK ${SDK_VERSION} and Claude Code ${CLAUDE_CODE_VERSION}`); + } + + const executable = resolveBundledClaudeExecutable(); + const output = await new Promise((resolveOutput, reject) => { + const child = spawn(executable, ['--version'], { shell: false, stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', (chunk: string) => { stdout += chunk; }); + child.stderr.on('data', (chunk: string) => { stderr += chunk; }); + child.once('error', () => reject(packagingDefect('version probe could not start'))); + child.once('exit', (code, signal) => { + if (signal || code !== 0) { + reject(packagingDefect(`version probe failed${signal ? ` with ${signal}` : ` with exit ${code}`}`)); + return; + } + if (stderr.trim()) { + reject(packagingDefect('version probe wrote to stderr')); + return; + } + resolveOutput(stdout.trim()); + }); + }); + if (output !== `${CLAUDE_CODE_VERSION} (Claude Code)`) { + throw packagingDefect(`version probe returned '${output}'`); + } + + verifiedRuntime = { + sdkVersion: SDK_VERSION, + embeddedBinaryVersion: CLAUDE_CODE_VERSION, + provenance: 'bundled', + }; + return verifiedRuntime; +} diff --git a/src/agents/claude.ts b/src/agents/claude.ts index 02c9913..ba3a4e0 100644 --- a/src/agents/claude.ts +++ b/src/agents/claude.ts @@ -52,6 +52,7 @@ import { createAskUserAnswerStore } from './claude/ask-user-answer-store.js'; import { createClaudeToolPermissionBridge } from './claude/tool-permission-bridge.js'; import { createClaudeDeniedMcpEventStore } from './claude/denied-mcp-event-store.js'; import { requireAskBusForLiveBatches } from '../sdk/ask-bus/bus.js'; +import { isStandaloneMode } from '../standalone/mode.js'; /** Shape of the SDK `query()` callable, narrowed for orchestration use. */ export type ClaudeSdkQueryFn = (args: { @@ -81,6 +82,28 @@ export interface ClaudeAgentOptions { claudeCodeExecutable?: string; } +export async function collectClaudeSdkMessages( + queryFn: ClaudeSdkQueryFn, + args: Parameters[0], + secrets: Array = [], +): Promise { + try { + const messages: SDKMessage[] = []; + const stream = queryFn(args); + for await (const message of stream as unknown as AsyncIterable) messages.push(message); + return messages; + } catch (error) { + let message = error instanceof Error ? error.message : String(error); + for (const secret of secrets) { + if (secret && secret.length >= 6) message = message.replaceAll(secret, '[redacted]'); + } + message = message + .replace(/(?:Authorization\s*:\s*)?Bearer\s+\S+/gi, '[redacted]') + .replace(/(?:_authToken|_auth|ANTHROPIC_AUTH_TOKEN|CLAUDE_CODE_OAUTH_TOKEN)=\S+/gi, '[redacted]'); + throw new Error(message); + } +} + function createLinkedAbortController(signal: AbortSignal | undefined): AbortController { const controller = new AbortController(); if (!signal) return controller; @@ -127,6 +150,7 @@ export class ClaudeAgent extends BaseAgent { const claudeCodeExecutable = resolveClaudeCodeExecutable({ agentOptionsExecutable: this.opts.claudeCodeExecutable, envExecutable, + standalone: isStandaloneMode(hostEnv), }); const mcpMountOptions = { workspacePath, @@ -183,11 +207,15 @@ export class ClaudeAgent extends BaseAgent { abortController: createLinkedAbortController(getTurnAbortSignal(sessionOptions)), }); - const messages: SDKMessage[] = []; - const stream = queryFn({ prompt: message, options: sdkOptions }); - for await (const msg of stream as unknown as AsyncIterable) { - messages.push(msg); - } + const runtimeEnv = getRuntimeEnv(runtime); + const messages = await collectClaudeSdkMessages( + queryFn, + { prompt: message, options: sdkOptions }, + [ + hostEnv.ANTHROPIC_API_KEY, hostEnv.ANTHROPIC_AUTH_TOKEN, hostEnv.CLAUDE_CODE_OAUTH_TOKEN, + runtimeEnv.ANTHROPIC_API_KEY, runtimeEnv.ANTHROPIC_AUTH_TOKEN, runtimeEnv.CLAUDE_CODE_OAUTH_TOKEN, + ], + ); // The legacy NDJSON parser only synthesized the slash-command // `use_skill` event from the *opening* user message. The Claude // SDK emits a fresh `init` system message (carrying `skills`) on diff --git a/src/agents/claude/sdk-options.ts b/src/agents/claude/sdk-options.ts index 57bcb80..035419b 100644 --- a/src/agents/claude/sdk-options.ts +++ b/src/agents/claude/sdk-options.ts @@ -71,7 +71,11 @@ export interface ClaudeSdkOptionsInputs { export function resolveClaudeCodeExecutable(args: { agentOptionsExecutable?: string; envExecutable?: string; + standalone?: boolean; }): string | undefined { + if (args.standalone && args.envExecutable) { + throw new Error('PATHGRADE_CLAUDE_CODE_EXECUTABLE is unsupported in pathgrade standalone'); + } if (args.agentOptionsExecutable) return args.agentOptionsExecutable; if (args.envExecutable) return args.envExecutable; return undefined; diff --git a/src/agents/codex-app-server/agent.ts b/src/agents/codex-app-server/agent.ts index aa4cc52..3760811 100644 --- a/src/agents/codex-app-server/agent.ts +++ b/src/agents/codex-app-server/agent.ts @@ -40,6 +40,8 @@ import { } from './wire-translators.js'; import { extractTurnCompletionFailure } from './turn-completion.js'; import type { ToolRequestUserInputParams } from './protocol/index.js'; +import { resolveBundledCodexCommand, verifyBundledCodexRuntime } from '../codex-runtime.js'; +import { isStandaloneMode } from '../../standalone/mode.js'; const DEFAULT_MODEL = 'gpt-5.4'; const TURN_COMPLETED_METHOD = 'turn/completed'; @@ -73,6 +75,27 @@ export interface CodexAppServerAgentDeps { onPermissionGrant?: (entry: PermissionGrantLogEntry) => void; } +export async function loginCodexAppServerWithApiKey( + transport: AppServerTransport, + apiKey: string, +): Promise { + if (!apiKey) { + throw new Error('Codex app-server requires OPENAI_API_KEY for pathgrade standalone'); + } + try { + const response = await transport.sendRequest<{ type?: unknown }>('account/login/start', { + type: 'apiKey', + apiKey, + }); + if (response?.type !== 'apiKey') { + throw new Error('Codex app-server did not confirm API-key login'); + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(message.split(apiKey).join('[redacted]')); + } +} + interface ActiveTurnState { turnNumber: number; askBatchIds: string[]; @@ -368,6 +391,7 @@ export class CodexAppServerAgent extends BaseAgent { const askBus = requireAskBusForLiveBatches(options, 'CodexAppServerAgent'); const workspacePath = getWorkspacePath(runtime); const runtimeEnv = getRuntimeEnv(runtime) as NodeJS.ProcessEnv; + const standalone = isStandaloneMode(); const model = options?.model ?? DEFAULT_MODEL; const sandboxMode: SandboxMode = this.deps.sandboxMode ?? 'workspace-write'; @@ -383,20 +407,36 @@ export class CodexAppServerAgent extends BaseAgent { if (handle) return handle.transport; const factory = this.deps.createTransport ?? (async (ctx: { workspacePath: string; env: NodeJS.ProcessEnv }) => - spawnAppServerTransport({ cwd: ctx.workspacePath, env: ctx.env })); - handle = await factory({ workspacePath, env: runtimeEnv }); - const transport = handle.transport; - transport.onServerRequest((req) => this.dispatchServerRequest(req, { + { + if (!isStandaloneMode()) { + return spawnAppServerTransport({ cwd: ctx.workspacePath, env: ctx.env }); + } + const command = resolveBundledCodexCommand(); + const verifiedRuntime = await verifyBundledCodexRuntime(command); + if (verifiedRuntime.packageVersion !== command.packageVersion) { + throw new Error('Pathgrade packaging defect: bundled Codex version mismatch'); + } + return spawnAppServerTransport({ + binary: command.executable, + prefixArgs: command.argsPrefix, + cwd: ctx.workspacePath, + env: ctx.env, + }); + }); + const candidate = await factory({ workspacePath, env: runtimeEnv }); + const transport = candidate.transport; + closeInfo = null; + const serverRequestOff = transport.onServerRequest((req) => this.dispatchServerRequest(req, { transport, askBus, activeTurn: () => activeTurn, onPermissionGrant: this.deps.onPermissionGrant, mcpSafety: options?.mcpSafety, })); - transport.onClose((info) => { + const closeOff = transport.onClose((info) => { closeInfo = info; }); - transport.onNotification((n) => { + const notificationOff = transport.onNotification((n) => { if (process.env.PATHGRADE_CODEX_DEBUG) { console.error(`[codex app-server] notification method=${n.method} params=${JSON.stringify(n.params).slice(0, 300)}`); } @@ -417,15 +457,32 @@ export class CodexAppServerAgent extends BaseAgent { if (!params?.item) return; projectItemIntoTurn(params.item, turn); }); - await transport.sendRequest('initialize', { - clientInfo: { name: 'pathgrade', version: '0.5.0', title: null }, - capabilities: { experimentalApi: true, optOutNotificationMethods: null }, - }); - // Upstream ClientNotification = { method: "initialized" }: send it - // before any thread/start so the handshake matches the v0.124 - // contract and is forward-compatible with servers that enforce it. - transport.sendNotification('initialized', null); - return transport; + try { + await transport.sendRequest('initialize', { + clientInfo: { name: 'pathgrade', version: '0.5.0', title: null }, + capabilities: { experimentalApi: true, optOutNotificationMethods: null }, + }); + // Upstream ClientNotification = { method: "initialized" }: send it + // before any thread/start so the handshake matches the v0.144 + // contract and is forward-compatible with servers that enforce it. + transport.sendNotification('initialized', null); + if (standalone) { + await loginCodexAppServerWithApiKey(transport, runtimeEnv.OPENAI_API_KEY ?? ''); + } + handle = candidate; + return transport; + } catch (error) { + serverRequestOff(); + closeOff(); + notificationOff(); + closeInfo = null; + try { + await candidate.close(); + } catch { + // Preserve the handshake failure as the actionable error. + } + throw error; + } }; const runTurn = async (message: string): Promise => { @@ -756,8 +813,6 @@ function buildThreadStartParams(opts: { approvalPolicy: 'never', sandbox: opts.sandboxMode, ephemeral: true, - experimentalRawEvents: false, - persistExtendedHistory: false, model: opts.model, ...(opts.mcpConfig ? { config: opts.mcpConfig } : {}), }; diff --git a/src/agents/codex-app-server/protocol/ClientRequest.ts b/src/agents/codex-app-server/protocol/ClientRequest.ts index fea20b4..6b0ec66 100644 --- a/src/agents/codex-app-server/protocol/ClientRequest.ts +++ b/src/agents/codex-app-server/protocol/ClientRequest.ts @@ -3,10 +3,10 @@ // `codex-rs/app-server-protocol/schema/typescript/ClientRequest.ts` // (top-level, NOT `typescript/v2/`). This module enumerates the subset of // method names the pathgrade Codex app-server driver actually sends. Refresh -// from openai/codex@rust-v0.124.0 when bumping the vendored version. +// from openai/codex@rust-v0.144.0 when bumping the vendored version. /** - * The 8 client-request methods the pathgrade driver may send to the Codex + * The 9 client-request methods the pathgrade driver may send to the Codex * app-server. Scoped to this driver's surface rather than the full upstream * protocol; phantom response-shaped entries are deliberately excluded because * JSON-RPC responses to server-initiated requests flow through @@ -20,6 +20,7 @@ * - thread/read * - thread/list * - review/start + * - account/login/start */ export type ClientRequestMethod = | 'initialize' @@ -29,7 +30,8 @@ export type ClientRequestMethod = | 'thread/inject_items' | 'thread/read' | 'thread/list' - | 'review/start'; + | 'review/start' + | 'account/login/start'; /** * JSON-RPC 2.0 client→server request envelope the driver constructs. diff --git a/src/agents/codex-app-server/protocol/DynamicToolCallParams.ts b/src/agents/codex-app-server/protocol/DynamicToolCallParams.ts index 3b4e14b..dab4eb4 100644 --- a/src/agents/codex-app-server/protocol/DynamicToolCallParams.ts +++ b/src/agents/codex-app-server/protocol/DynamicToolCallParams.ts @@ -1,4 +1,4 @@ -// Vendored from openai/codex@rust-v0.124.0 +// Vendored from openai/codex@rust-v0.144.0 // Source: codex-rs/app-server-protocol/schema/typescript/v2/DynamicToolCallParams.ts // GENERATED CODE in upstream; do not modify locally either. diff --git a/src/agents/codex-app-server/protocol/GrantedPermissionProfile.ts b/src/agents/codex-app-server/protocol/GrantedPermissionProfile.ts index d35421a..01a808b 100644 --- a/src/agents/codex-app-server/protocol/GrantedPermissionProfile.ts +++ b/src/agents/codex-app-server/protocol/GrantedPermissionProfile.ts @@ -1,4 +1,4 @@ -// Vendored from openai/codex@rust-v0.124.0 +// Vendored from openai/codex@rust-v0.144.0 // Source: codex-rs/app-server-protocol/schema/typescript/v2/GrantedPermissionProfile.ts // GENERATED CODE in upstream; do not modify locally either. // diff --git a/src/agents/codex-app-server/protocol/McpElicitationRequestParams.ts b/src/agents/codex-app-server/protocol/McpElicitationRequestParams.ts index 122e7fa..e5f18bf 100644 --- a/src/agents/codex-app-server/protocol/McpElicitationRequestParams.ts +++ b/src/agents/codex-app-server/protocol/McpElicitationRequestParams.ts @@ -1,4 +1,4 @@ -// Vendored from openai/codex@rust-v0.124.0 +// Vendored from openai/codex@rust-v0.144.0 // Source: codex-rs/app-server-protocol/schema/typescript/v2/McpServerElicitationRequestParams.ts // GENERATED CODE in upstream; do not modify locally either. // @@ -20,5 +20,6 @@ export type McpElicitationRequestParams = { serverName: string; } & ( | { mode: 'form'; _meta: JsonValue | null; message: string; requestedSchema: McpElicitationSchema } + | { mode: 'openai/form'; _meta: JsonValue | null; message: string; requestedSchema: JsonValue } | { mode: 'url'; _meta: JsonValue | null; message: string; url: string; elicitationId: string } ); diff --git a/src/agents/codex-app-server/protocol/PermissionsRequestApprovalParams.ts b/src/agents/codex-app-server/protocol/PermissionsRequestApprovalParams.ts index 53d39fd..23b5b68 100644 --- a/src/agents/codex-app-server/protocol/PermissionsRequestApprovalParams.ts +++ b/src/agents/codex-app-server/protocol/PermissionsRequestApprovalParams.ts @@ -1,4 +1,4 @@ -// Vendored from openai/codex@rust-v0.124.0 +// Vendored from openai/codex@rust-v0.144.0 // Source: codex-rs/app-server-protocol/schema/typescript/v2/PermissionsRequestApprovalParams.ts // GENERATED CODE in upstream; do not modify locally either. // @@ -11,6 +11,9 @@ export type PermissionsRequestApprovalParams = { threadId: string; turnId: string; itemId: string; + environmentId: string | null; + /** Unix timestamp (in milliseconds) when this approval request started. */ + startedAtMs: number; cwd: AbsolutePathBuf; reason: string | null; permissions: RequestPermissionProfile; diff --git a/src/agents/codex-app-server/protocol/PermissionsRequestApprovalResponse.ts b/src/agents/codex-app-server/protocol/PermissionsRequestApprovalResponse.ts index c4ffd89..775bc76 100644 --- a/src/agents/codex-app-server/protocol/PermissionsRequestApprovalResponse.ts +++ b/src/agents/codex-app-server/protocol/PermissionsRequestApprovalResponse.ts @@ -1,4 +1,4 @@ -// Vendored from openai/codex@rust-v0.124.0 +// Vendored from openai/codex@rust-v0.144.0 // Source: codex-rs/app-server-protocol/schema/typescript/v2/PermissionsRequestApprovalResponse.ts // GENERATED CODE in upstream; do not modify locally either. diff --git a/src/agents/codex-app-server/protocol/README.md b/src/agents/codex-app-server/protocol/README.md index eeac57c..dd721b6 100644 --- a/src/agents/codex-app-server/protocol/README.md +++ b/src/agents/codex-app-server/protocol/README.md @@ -2,7 +2,7 @@ The `.ts` files in this directory are a curated Pathgrade protocol surface based on [`openai/codex`](https://github.com/openai/codex) at tag -`rust-v0.124.0`. Most vendored shapes live upstream under +`rust-v0.144.0`. Most vendored shapes live upstream under `codex-rs/app-server-protocol/schema/typescript/v2/`. The union files `ClientRequest.ts` and `ServerRequest.ts` live upstream directly under `codex-rs/app-server-protocol/schema/typescript/` (no `v2/` segment), but @@ -40,7 +40,7 @@ does not currently inspect the nested shape. ## Runtime consumption -Pathgrade does not depend on the `@openai/codex` npm package at build or run -time; the driver spawns the PATH-installed `codex` binary and talks JSON-RPC -over stdio, using these vendored types for compile-time shape checking. The -protocol tag above is the single source of truth for protocol alignment. +Pathgrade's standalone runtime depends on the pinned `@openai/codex` npm +package and invokes its JavaScript launcher through Node. Project-local mode +continues to spawn the PATH-installed `codex` binary. Both modes talk JSON-RPC +over stdio using these vendored types for compile-time shape checking. diff --git a/src/agents/codex-app-server/protocol/SandboxMode.ts b/src/agents/codex-app-server/protocol/SandboxMode.ts index 8a597e9..e96f1b1 100644 --- a/src/agents/codex-app-server/protocol/SandboxMode.ts +++ b/src/agents/codex-app-server/protocol/SandboxMode.ts @@ -1,4 +1,4 @@ -// Vendored from openai/codex@rust-v0.124.0 +// Vendored from openai/codex@rust-v0.144.0 // Source: codex-rs/app-server-protocol/schema/typescript/v2/SandboxMode.ts // GENERATED CODE in upstream; do not modify locally either. diff --git a/src/agents/codex-app-server/protocol/ServerRequest.ts b/src/agents/codex-app-server/protocol/ServerRequest.ts index 86ca3bb..fdef045 100644 --- a/src/agents/codex-app-server/protocol/ServerRequest.ts +++ b/src/agents/codex-app-server/protocol/ServerRequest.ts @@ -1,6 +1,6 @@ // Pathgrade-local composition; not a single upstream file. -// Enumerates the 9 server-request variants the driver observes over the wire -// under rust-v0.124.0. Upstream ships the discriminated union at +// Enumerates the 10 server-request variants the driver observes over the wire +// under rust-v0.144.0. Upstream ships the discriminated union at // `codex-rs/app-server-protocol/schema/typescript/ServerRequest.ts` // (top-level, NOT `typescript/v2/`); the per-variant params files this module // imports live under the `v2/` subdirectory. @@ -19,7 +19,8 @@ export type ServerRequestMethod = | 'item/fileChange/requestApproval' | 'applyPatchApproval' | 'execCommandApproval' - | 'account/chatgptAuthTokens/refresh'; + | 'account/chatgptAuthTokens/refresh' + | 'attestation/generate'; export type ServerRequest = | { method: 'item/tool/requestUserInput'; id: number | string; params: ToolRequestUserInputParams } @@ -30,4 +31,5 @@ export type ServerRequest = | { method: 'item/fileChange/requestApproval'; id: number | string; params: unknown } | { method: 'applyPatchApproval'; id: number | string; params: unknown } | { method: 'execCommandApproval'; id: number | string; params: unknown } - | { method: 'account/chatgptAuthTokens/refresh'; id: number | string; params: unknown }; + | { method: 'account/chatgptAuthTokens/refresh'; id: number | string; params: unknown } + | { method: 'attestation/generate'; id: number | string; params: unknown }; diff --git a/src/agents/codex-app-server/protocol/ThreadStartParams.ts b/src/agents/codex-app-server/protocol/ThreadStartParams.ts index 8c19542..c2c9014 100644 --- a/src/agents/codex-app-server/protocol/ThreadStartParams.ts +++ b/src/agents/codex-app-server/protocol/ThreadStartParams.ts @@ -1,4 +1,4 @@ -// Vendored from openai/codex@rust-v0.124.0 +// Vendored from openai/codex@rust-v0.144.0 // Source: codex-rs/app-server-protocol/schema/typescript/v2/ThreadStartParams.ts // GENERATED CODE in upstream; do not modify locally either. // @@ -12,12 +12,12 @@ import type { SandboxMode } from './SandboxMode.js'; // Opaque aliases — shapes the driver does not yet destructure. export type Personality = unknown; -export type ServiceTier = unknown; +export type ServiceTier = string; export type JsonValue = unknown; export type ApprovalsReviewer = unknown; export type AskForApproval = unknown; -export type PermissionProfile = unknown; export type ThreadStartSource = unknown; +export type ThreadSource = string; export type ThreadStartParams = { model?: string | null; @@ -31,11 +31,6 @@ export type ThreadStartParams = { */ approvalsReviewer?: ApprovalsReviewer | null; sandbox?: SandboxMode | null; - /** - * Full permissions override for this thread. Cannot be combined with - * `sandbox`. - */ - permissionProfile?: PermissionProfile | null; config?: { [key: string]: JsonValue | undefined } | null; serviceName?: string | null; baseInstructions?: string | null; @@ -43,14 +38,6 @@ export type ThreadStartParams = { personality?: Personality | null; ephemeral?: boolean | null; sessionStartSource?: ThreadStartSource | null; - /** - * If true, opt into emitting raw Responses API items on the event stream. - * This is for internal use only (e.g. Codex Cloud). - */ - experimentalRawEvents: boolean; - /** - * If true, persist additional rollout EventMsg variants required to - * reconstruct a richer thread history on resume/fork/read. - */ - persistExtendedHistory: boolean; + /** Optional client-supplied analytics source classification for this thread. */ + threadSource?: ThreadSource | null; }; diff --git a/src/agents/codex-app-server/protocol/ToolRequestUserInputAnswer.ts b/src/agents/codex-app-server/protocol/ToolRequestUserInputAnswer.ts index 84675f9..b3af56b 100644 --- a/src/agents/codex-app-server/protocol/ToolRequestUserInputAnswer.ts +++ b/src/agents/codex-app-server/protocol/ToolRequestUserInputAnswer.ts @@ -1,4 +1,4 @@ -// Vendored from openai/codex@rust-v0.124.0 +// Vendored from openai/codex@rust-v0.144.0 // Source: codex-rs/app-server-protocol/schema/typescript/v2/ToolRequestUserInputAnswer.ts // GENERATED CODE in upstream; do not modify locally either. diff --git a/src/agents/codex-app-server/protocol/ToolRequestUserInputOption.ts b/src/agents/codex-app-server/protocol/ToolRequestUserInputOption.ts index a67e0d9..29460b1 100644 --- a/src/agents/codex-app-server/protocol/ToolRequestUserInputOption.ts +++ b/src/agents/codex-app-server/protocol/ToolRequestUserInputOption.ts @@ -1,4 +1,4 @@ -// Vendored from openai/codex@rust-v0.124.0 +// Vendored from openai/codex@rust-v0.144.0 // Source: codex-rs/app-server-protocol/schema/typescript/v2/ToolRequestUserInputOption.ts // GENERATED CODE in upstream; do not modify locally either. diff --git a/src/agents/codex-app-server/protocol/ToolRequestUserInputParams.ts b/src/agents/codex-app-server/protocol/ToolRequestUserInputParams.ts index 08d80cf..a208942 100644 --- a/src/agents/codex-app-server/protocol/ToolRequestUserInputParams.ts +++ b/src/agents/codex-app-server/protocol/ToolRequestUserInputParams.ts @@ -1,4 +1,4 @@ -// Vendored from openai/codex@rust-v0.124.0 +// Vendored from openai/codex@rust-v0.144.0 // Source: codex-rs/app-server-protocol/schema/typescript/v2/ToolRequestUserInputParams.ts // GENERATED CODE in upstream; do not modify locally either. @@ -12,4 +12,5 @@ export type ToolRequestUserInputParams = { turnId: string; itemId: string; questions: Array; + autoResolutionMs: number | null; }; diff --git a/src/agents/codex-app-server/protocol/ToolRequestUserInputQuestion.ts b/src/agents/codex-app-server/protocol/ToolRequestUserInputQuestion.ts index 4dcf841..64fc65e 100644 --- a/src/agents/codex-app-server/protocol/ToolRequestUserInputQuestion.ts +++ b/src/agents/codex-app-server/protocol/ToolRequestUserInputQuestion.ts @@ -1,4 +1,4 @@ -// Vendored from openai/codex@rust-v0.124.0 +// Vendored from openai/codex@rust-v0.144.0 // Source: codex-rs/app-server-protocol/schema/typescript/v2/ToolRequestUserInputQuestion.ts // GENERATED CODE in upstream; do not modify locally either. diff --git a/src/agents/codex-app-server/protocol/ToolRequestUserInputResponse.ts b/src/agents/codex-app-server/protocol/ToolRequestUserInputResponse.ts index 85931af..6051964 100644 --- a/src/agents/codex-app-server/protocol/ToolRequestUserInputResponse.ts +++ b/src/agents/codex-app-server/protocol/ToolRequestUserInputResponse.ts @@ -1,4 +1,4 @@ -// Vendored from openai/codex@rust-v0.124.0 +// Vendored from openai/codex@rust-v0.144.0 // Source: codex-rs/app-server-protocol/schema/typescript/v2/ToolRequestUserInputResponse.ts // GENERATED CODE in upstream; do not modify locally either. diff --git a/src/agents/codex-app-server/protocol/TurnCompletedNotification.ts b/src/agents/codex-app-server/protocol/TurnCompletedNotification.ts index 92dc040..d307a9d 100644 --- a/src/agents/codex-app-server/protocol/TurnCompletedNotification.ts +++ b/src/agents/codex-app-server/protocol/TurnCompletedNotification.ts @@ -1,4 +1,4 @@ -// Vendored from openai/codex@rust-v0.124.0 +// Vendored from openai/codex@rust-v0.144.0 // Source: codex-rs/app-server-protocol/schema/typescript/v2/TurnCompletedNotification.ts // GENERATED CODE in upstream; do not modify locally either. diff --git a/src/agents/codex-app-server/protocol/index.ts b/src/agents/codex-app-server/protocol/index.ts index 9151b51..24bfc8d 100644 --- a/src/agents/codex-app-server/protocol/index.ts +++ b/src/agents/codex-app-server/protocol/index.ts @@ -1,6 +1,6 @@ // Curated re-exports for the pathgrade Codex app-server driver and fixture // tests. Every re-exported symbol is either a curated upstream shape from -// openai/codex@rust-v0.124.0 (with an upstream-citation header in its file) or +// openai/codex@rust-v0.144.0 (with an upstream-citation header in its file) or // a pathgrade-local composition (ClientRequest, ServerRequest, Op) whose header // spells out the composition rationale. // diff --git a/src/agents/codex-app-server/transport.ts b/src/agents/codex-app-server/transport.ts index 6c43dbc..0f0b89b 100644 --- a/src/agents/codex-app-server/transport.ts +++ b/src/agents/codex-app-server/transport.ts @@ -202,6 +202,7 @@ function createNdjsonTransportInternal( export interface SpawnAppServerTransportInput { binary?: string; + prefixArgs?: readonly string[]; args?: readonly string[]; env?: NodeJS.ProcessEnv; cwd?: string; @@ -239,6 +240,14 @@ export function buildAppServerSpawnArgs( ]; } +export function buildAppServerProcessArgs( + prefixArgs: readonly string[] = [], + args: readonly string[] = [], + env: NodeJS.ProcessEnv = process.env, +): string[] { + return [...prefixArgs, ...buildAppServerSpawnArgs(args, env)]; +} + /** * Minimal child-process surface the session handle needs. Real spawns satisfy * this via `ChildProcessWithoutNullStreams`; tests can substitute a fake that @@ -328,7 +337,7 @@ export function spawnAppServerTransport( const binary = cfg.binary ?? 'codex'; const args = cfg.args ?? []; const env = cfg.env ?? process.env; - const child: ChildProcessWithoutNullStreams = spawn(binary, buildAppServerSpawnArgs(args, env), { + const child: ChildProcessWithoutNullStreams = spawn(binary, buildAppServerProcessArgs(cfg.prefixArgs, args, env), { stdio: ['pipe', 'pipe', 'pipe'], env, cwd: cfg.cwd, @@ -338,12 +347,41 @@ export function spawnAppServerTransport( // this, a codex auth or connectivity failure just shows up as an empty // turn with no explanation — especially painful in CI logs. let stderrBuf = ''; + let stderrPending = ''; const STDERR_CAP_BYTES = 8_192; - child.stderr.setEncoding('utf8'); - child.stderr.on('data', (chunk: string) => { + const apiKey = env.OPENAI_API_KEY; + const appendStderr = (value: string): void => { if (stderrBuf.length >= STDERR_CAP_BYTES) return; - stderrBuf += chunk; + stderrBuf += value; if (stderrBuf.length > STDERR_CAP_BYTES) stderrBuf = stderrBuf.slice(0, STDERR_CAP_BYTES); + }; + const consumeStderr = (flush: boolean): void => { + if (!apiKey) { + appendStderr(stderrPending); + stderrPending = ''; + return; + } + let secretIndex = stderrPending.indexOf(apiKey); + while (secretIndex !== -1) { + appendStderr(stderrPending.slice(0, secretIndex)); + appendStderr('[redacted]'); + stderrPending = stderrPending.slice(secretIndex + apiKey.length); + secretIndex = stderrPending.indexOf(apiKey); + } + if (flush) { + appendStderr(stderrPending); + stderrPending = ''; + return; + } + const retainedLength = Math.min(stderrPending.length, apiKey.length - 1); + const emittedLength = stderrPending.length - retainedLength; + appendStderr(stderrPending.slice(0, emittedLength)); + stderrPending = stderrPending.slice(emittedLength); + }; + child.stderr.setEncoding('utf8'); + child.stderr.on('data', (chunk: string) => { + stderrPending += chunk; + consumeStderr(false); }); const transport = createNdjsonTransportInternal({ @@ -353,6 +391,7 @@ export function spawnAppServerTransport( }); child.on('exit', (exitCode, signal) => { + consumeStderr(true); if (stderrBuf.trim().length > 0) { console.error( `[codex app-server pid=${child.pid}] exited with code=${exitCode} signal=${signal}. stderr:\n${stderrBuf}`, diff --git a/src/agents/codex-runtime.ts b/src/agents/codex-runtime.ts new file mode 100644 index 0000000..cd489a3 --- /dev/null +++ b/src/agents/codex-runtime.ts @@ -0,0 +1,188 @@ +import { spawn } from 'node:child_process'; +import { + chmodSync, + existsSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; + +const CODEX_VERSION = '0.144.0' as const; +const pathgradeRequire = createRequire(import.meta.url); + +export interface BundledCodexCommand { + executable: string; + argsPrefix: string[]; + packageVersion: typeof CODEX_VERSION; + nativeVersion: typeof CODEX_VERSION; + provenance: 'bundled'; +} + +export interface VerifiedBundledCodexRuntime { + packageVersion: typeof CODEX_VERSION; + nativeVersion: typeof CODEX_VERSION; + provenance: 'bundled'; +} + +let verifiedRuntime: VerifiedBundledCodexRuntime | undefined; + +function packagingDefect(message: string): Error { + return new Error(`Pathgrade packaging defect: bundled Codex ${message}`); +} + +/** @internal Exported for package-layout regression coverage. */ +export function resolveBundledCodexNativeArtifact(packageJsonPath: string): string { + return resolveBundledCodexNativeArtifactMetadata(packageJsonPath).executable; +} + +function resolveBundledCodexNativeArtifactMetadata(packageJsonPath: string): { + executable: string; + version: typeof CODEX_VERSION; +} { + const target = `${process.platform}:${process.arch}`; + const platformPackageByTarget: Record = { + 'darwin:x64': '@openai/codex-darwin-x64', + 'darwin:arm64': '@openai/codex-darwin-arm64', + 'linux:x64': '@openai/codex-linux-x64', + 'linux:arm64': '@openai/codex-linux-arm64', + }; + const platformPackage = platformPackageByTarget[target]; + if (!platformPackage) { + throw packagingDefect(`does not support ${process.platform}/${process.arch}`); + } + let artifactPackageJson: string; + try { + const codexRequire = createRequire(packageJsonPath); + artifactPackageJson = codexRequire.resolve(`${platformPackage}/package.json`); + } catch { + throw packagingDefect(`native artifact ${platformPackage} is unavailable`); + } + let artifactMetadata: { version?: unknown }; + try { + artifactMetadata = JSON.parse(readFileSync(artifactPackageJson, 'utf8')) as { version?: unknown }; + } catch { + throw packagingDefect('native artifact metadata is unreadable'); + } + const artifactVersionSuffixByTarget: Record = { + 'darwin:x64': 'darwin-x64', + 'darwin:arm64': 'darwin-arm64', + 'linux:x64': 'linux-x64', + 'linux:arm64': 'linux-arm64', + }; + const expectedArtifactVersion = `${CODEX_VERSION}-${artifactVersionSuffixByTarget[target]}`; + if (artifactMetadata.version !== expectedArtifactVersion) { + throw packagingDefect(`native artifact version must be ${expectedArtifactVersion}`); + } + const targetTripleByTarget: Record = { + 'darwin:x64': 'x86_64-apple-darwin', + 'darwin:arm64': 'aarch64-apple-darwin', + 'linux:x64': 'x86_64-unknown-linux-musl', + 'linux:arm64': 'aarch64-unknown-linux-musl', + }; + const targetTriple = targetTripleByTarget[target]!; + const executable = join( + dirname(artifactPackageJson), + 'vendor', + targetTriple, + 'bin', + process.platform === 'win32' ? 'codex.exe' : 'codex', + ); + if (!existsSync(executable)) { + throw packagingDefect(`native artifact ${platformPackage} is unavailable`); + } + return { executable, version: CODEX_VERSION }; +} + +export function resolveBundledCodexCommand(): BundledCodexCommand { + if (process.platform === 'win32') { + throw new Error( + 'pathgrade standalone: native Windows is unsupported; use WSL with x64 or arm64', + ); + } + + let packageJsonPath: string; + try { + packageJsonPath = pathgradeRequire.resolve('@openai/codex/package.json'); + } catch { + throw packagingDefect('@openai/codex is unavailable'); + } + + let metadata: { version?: unknown; bin?: unknown }; + try { + metadata = JSON.parse(readFileSync(packageJsonPath, 'utf8')) as { version?: unknown; bin?: unknown }; + } catch { + throw packagingDefect('package metadata is unreadable'); + } + const launcher = typeof metadata.bin === 'object' && metadata.bin !== null + ? (metadata.bin as { codex?: unknown }).codex + : undefined; + if (typeof launcher !== 'string') throw packagingDefect('launcher is unavailable'); + if (metadata.version !== CODEX_VERSION) { + throw packagingDefect(`version must be ${CODEX_VERSION}`); + } + const launcherPath = resolve(dirname(packageJsonPath), launcher); + if (!existsSync(launcherPath)) throw packagingDefect('launcher is unavailable'); + const nativeArtifact = resolveBundledCodexNativeArtifactMetadata(packageJsonPath); + + return { + executable: process.execPath, + argsPrefix: [launcherPath], + packageVersion: CODEX_VERSION, + nativeVersion: nativeArtifact.version, + provenance: 'bundled', + }; +} + +export async function verifyBundledCodexRuntime( + command: BundledCodexCommand, +): Promise { + if (verifiedRuntime) return verifiedRuntime; + const shimDir = mkdtempSync(join(tmpdir(), 'pathgrade-codex-path-')); + const shimPath = join(shimDir, process.platform === 'win32' ? 'codex.cmd' : 'codex'); + writeFileSync( + shimPath, + process.platform === 'win32' ? '@exit /b 127\r\n' : '#!/bin/sh\nexit 127\n', + ); + if (process.platform !== 'win32') chmodSync(shimPath, 0o755); + + const output = await new Promise((resolveOutput, reject) => { + const child = spawn(command.executable, [...command.argsPrefix, '--version'], { + shell: false, + env: { ...process.env, PATH: shimDir }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', (chunk: string) => { stdout += chunk; }); + child.stderr.on('data', (chunk: string) => { stderr += chunk; }); + child.once('error', () => reject(packagingDefect('version probe could not start'))); + child.once('exit', (code, signal) => { + if (signal || code !== 0) { + reject(packagingDefect(`version probe failed${signal ? ` with ${signal}` : ` with exit ${code}`}`)); + return; + } + if (stderr.trim()) { + reject(packagingDefect('version probe wrote to stderr')); + return; + } + resolveOutput(stdout.trim()); + }); + }).finally(() => { + rmSync(shimDir, { recursive: true, force: true }); + }); + if (output !== `codex-cli ${CODEX_VERSION}`) { + throw packagingDefect(`version probe returned '${output}'`); + } + verifiedRuntime = { + packageVersion: CODEX_VERSION, + nativeVersion: CODEX_VERSION, + provenance: 'bundled', + }; + return verifiedRuntime; +} diff --git a/src/commands/affected.ts b/src/commands/affected.ts index 3ff4351..7458e81 100644 --- a/src/commands/affected.ts +++ b/src/commands/affected.ts @@ -28,6 +28,7 @@ import { export interface RunAffectedOptions { /** Absolute path to the repo root (CLI passes `process.cwd()`). */ cwd: string; + standalone?: boolean; /** Newline-delimited list of repo-relative changed files (overrides git). */ changedFilesPath?: string; /** Git ref to diff against (`...HEAD`). Overrides auto-detection. */ @@ -70,6 +71,7 @@ export async function runAffected(opts: RunAffectedOptions): Promise { try { config = await resolvePathgradeConfig({ cwd, + standalone: opts.standalone, warn: w => process.stderr.write(`${w}\n`), }); } catch (err) { diff --git a/src/commands/run-changed.ts b/src/commands/run-changed.ts index c2934e0..a1cde92 100644 --- a/src/commands/run-changed.ts +++ b/src/commands/run-changed.ts @@ -24,11 +24,13 @@ import type { RunnerInvocationAdapter } from '../runners/invocation.js'; import type { SpawnVitest } from '../runners/vitest-invocation.js'; import type { SelectionResult } from '../affected/types.js'; import type { PathgradeRunArgs } from './run-args.js'; +import { validateStandaloneInvocation } from '../standalone/validation.js'; export type { SpawnVitest }; export interface RunChangedOptions { cwd: string; + standalone?: boolean; parsed: PathgradeRunArgs; /** Override for tests; default spawns `npx vitest` inheriting stdio. */ spawnVitest?: SpawnVitest; @@ -41,6 +43,7 @@ export async function runChanged(opts: RunChangedOptions): Promise { ...process.env, ...(parsed.forceDiagnostics ? { PATHGRADE_DIAGNOSTICS: '1' } : {}), ...(parsed.forceVerbose ? { PATHGRADE_VERBOSE: '1' } : {}), + ...(parsed.quiet ? { PATHGRADE_QUIET: '1' } : {}), }; const configPath = findVitestConfigArg(parsed.runnerArgs); @@ -49,16 +52,24 @@ export async function runChanged(opts: RunChangedOptions): Promise { try { config = await resolvePathgradeConfig({ cwd, + standalone: opts.standalone, legacyVitestConfigPath: configPath, warn: w => { if (!parsed.quiet) process.stderr.write(`${w}\n`); }, }); + if (opts.standalone) { + validateStandaloneInvocation({ + adapterName: parsed.adapterName ?? config.runner.adapter, + runnerArgs: [...config.runner.args, ...parsed.runnerArgs], + }); + } runnerInvocation = opts.runnerInvocation ?? await loadRunnerInvocationAdapter({ adapterName: parsed.adapterName ?? config.runner.adapter, cwd, config, spawnVitest: opts.spawnVitest, + standalone: opts.standalone, }); } catch (err) { process.stderr.write(`${errMsg(err)}\n`); diff --git a/src/config/pathgrade.ts b/src/config/pathgrade.ts index 59db7bd..1f63415 100644 --- a/src/config/pathgrade.ts +++ b/src/config/pathgrade.ts @@ -80,17 +80,20 @@ export function defaultPathgradeConfig(): ResolvedPathgradeConfig { export async function resolvePathgradeConfig(input: { cwd: string; + standalone?: boolean; cli?: PathgradeConfig; configPath?: string; legacyVitestConfigPath?: string; warn?: (message: string) => void; }): Promise { const fileConfig = await loadPathgradeConfigFile(input.cwd, input.configPath); - const legacyConfig = await loadLegacyVitestConfig( - input.cwd, - input.legacyVitestConfigPath, - input.warn, - ); + const legacyConfig = input.standalone + ? undefined + : await loadLegacyVitestConfig( + input.cwd, + input.legacyVitestConfigPath, + input.warn, + ); return mergePathgradeConfig( mergePathgradeConfig( mergePathgradeConfig(defaultPathgradeConfig(), legacyConfig), diff --git a/src/pathgrade.ts b/src/pathgrade.ts index da45540..6ee61bb 100644 --- a/src/pathgrade.ts +++ b/src/pathgrade.ts @@ -24,6 +24,14 @@ import { runChanged } from './commands/run-changed.js'; import { clearSidecar } from './affected/sidecar.js'; import { resolvePathgradeConfig } from './config/pathgrade.js'; import { loadRunnerInvocationAdapter } from './runners/adapter-loader.js'; +import { + parseStandaloneCommand, + PATHGRADE_STANDALONE_ENV, +} from './standalone/mode.js'; +import { + assertStandalonePlatform, + validateStandaloneInvocation, +} from './standalone/validation.js'; import { fmt } from './utils/cli.js'; import { shutdown } from './utils/shutdown.js'; @@ -50,36 +58,80 @@ function loadDotenv(): void { } } -function validateApiKeys(): void { +function validateApiKeys(standalone = false): void { const hasAnthropic = !!process.env.ANTHROPIC_API_KEY; const hasOpenAI = !!process.env.OPENAI_API_KEY; - const hasClaude = !!process.env.HOME; // Claude CLI uses OS keychain, just check it exists + + // Standalone validates the selected provider at invocation time. A generic + // warning here is both noisy for deterministic evals and unable to account + // for Claude keychain or Codex cached authentication. + if (standalone) return; if (!hasAnthropic && !hasOpenAI) { + const location = '.env or environment'; + const fallback = 'Claude CLI auth (keychain) and Codex exec cached login (~/.codex/auth.json) may still work if installed.'; console.log( - `\n ${fmt.dim('warning:')} No API keys found. Set ANTHROPIC_API_KEY or OPENAI_API_KEY in .env or environment.\n` + - ` ${fmt.dim(' Claude CLI auth (keychain) and Codex exec cached login (~/.codex/auth.json) may still work if installed.')}\n`, + `\n ${fmt.dim('warning:')} No API keys found. Set ANTHROPIC_API_KEY or OPENAI_API_KEY in ${location}.\n` + + ` ${fmt.dim(` ${fallback}`)}\n`, ); } } async function main() { shutdown.install(); - const args = process.argv.slice(2); + const parsedCommand = parseStandaloneCommand(process.argv.slice(2)); + if (parsedCommand.standalone) { + process.env[PATHGRADE_STANDALONE_ENV] = '1'; + } else { + delete process.env[PATHGRADE_STANDALONE_ENV]; + } + + const args = parsedCommand.args; const command = args[0]; + if (parsedCommand.standalone && (command === '--help' || command === '-h')) { + printStandaloneHelp(); + return; + } + + if (parsedCommand.standalone && command === '--version') { + console.log(await readPackageVersion()); + return; + } + + if ( + parsedCommand.standalone + && command !== 'run' + && command !== 'affected' + && !command.startsWith('-') + ) { + console.error(`pathgrade standalone: unsupported command "${command}"`); + console.error('Run "pathgrade standalone --help" for supported commands.'); + process.exitCode = 1; + return; + } + + if (parsedCommand.standalone) { + try { + assertStandalonePlatform({ + nodeMajor: Number(process.versions.node.split('.')[0]), + platform: process.platform, + arch: process.arch, + }); + } catch (err) { + console.error(err instanceof Error ? err.message : String(err)); + process.exitCode = 1; + return; + } + } + if (command === '--help' || command === '-h') { printHelp(); return; } - if (command === '--version' || command === '-v') { - const pkg = JSON.parse( - await import('fs').then(fs => fs.promises.readFile( - new URL('../package.json', import.meta.url), 'utf-8' - )) - ); - console.log(pkg.version); + if (command === '--version' || (!parsedCommand.standalone && command === '-v')) { + console.log(await readPackageVersion()); return; } @@ -145,6 +197,7 @@ async function main() { const json = affectedArgs.includes('--json'); const exitCode = await runAffected({ cwd: process.cwd(), + standalone: parsedCommand.standalone, changedFilesPath, since, explain, @@ -167,11 +220,19 @@ async function main() { if (command === 'run' || !command || command.startsWith('-')) { // pathgrade run [--changed [--since=…|--changed-files=…]] [--] [runner-args] - loadDotenv(); - validateApiKeys(); + if (!parsedCommand.standalone) { + loadDotenv(); + } + validateApiKeys(parsedCommand.standalone); const parsed = parsePathgradeRunArgs(command === 'run' ? args.slice(1) : args); + if (parsedCommand.standalone && parsed.forceVerbose && parsed.quiet) { + console.error('pathgrade standalone: --quiet and --verbose are mutually exclusive'); + process.exitCode = 1; + return; + } + for (const warning of parsed.warnings ?? []) { console.error(`pathgrade: ${warning}`); } @@ -180,6 +241,7 @@ async function main() { const exitCode = await runChanged({ cwd: process.cwd(), parsed, + standalone: parsedCommand.standalone, }); process.exitCode = exitCode; return; @@ -194,13 +256,24 @@ async function main() { ...process.env, ...(parsed.forceDiagnostics ? { PATHGRADE_DIAGNOSTICS: '1' } : {}), ...(parsed.forceVerbose ? { PATHGRADE_VERBOSE: '1' } : {}), + ...(parsed.quiet ? { PATHGRADE_QUIET: '1' } : {}), }; try { - const config = await resolvePathgradeConfig({ cwd: process.cwd() }); + const config = await resolvePathgradeConfig({ + cwd: process.cwd(), + standalone: parsedCommand.standalone, + }); + if (parsedCommand.standalone) { + validateStandaloneInvocation({ + adapterName: parsed.adapterName ?? config.runner.adapter, + runnerArgs: [...config.runner.args, ...parsed.runnerArgs], + }); + } const runner = await loadRunnerInvocationAdapter({ adapterName: parsed.adapterName ?? config.runner.adapter, cwd: process.cwd(), config, + standalone: parsedCommand.standalone, }); process.exitCode = await runner.run({ cwd: process.cwd(), @@ -232,6 +305,8 @@ function printHelp() { [--diagnostics] Print full diagnostics for passing evals too [--quiet] Suppress the run-start summary [--verbose|-v] Stream live per-turn events to stderr during the run + pathgrade standalone [run|affected] + Embedded Vitest; Claude and Codex app-server only pathgrade init [--force] Generate eval scaffolding pathgrade analyze [--skill=X] Analyze skills and output JSON pathgrade validate Validate an .eval.ts file @@ -266,6 +341,36 @@ function printHelp() { `); } +function printStandaloneHelp(): void { + console.log(` + pathgrade standalone - Run evals with the embedded Vitest runtime + + Usage: + pathgrade standalone [run] [eval-files...] [options] + pathgrade standalone affected [options] + + Run options: + --verbose, -v Stream live agent events + --quiet Suppress progress and passing case details + --diagnostics Print expanded final diagnostics + --changed Run only affected evals + --since= Override the affected base ref + --changed-files= Read changed files from a newline-delimited file + -t, --testNamePattern= Filter Vitest cases by name + + Affected options: + --since= Diff ...HEAD + --changed-files= Read an explicit changed-file list + --explain Explain each selection decision on stderr + --json Emit structured JSON on stdout +`); +} + +async function readPackageVersion(): Promise { + const content = await fs.promises.readFile(new URL('../package.json', import.meta.url), 'utf-8'); + return (JSON.parse(content) as { version: string }).version; +} + main().catch(err => { console.error(err); process.exit(1); diff --git a/src/providers/credentials.ts b/src/providers/credentials.ts index 3a5558f..6286d9e 100644 --- a/src/providers/credentials.ts +++ b/src/providers/credentials.ts @@ -16,7 +16,7 @@ import { execSync, execFileSync } from 'child_process'; import * as os from 'os'; import * as path from 'path'; import fs from 'fs-extra'; -import type { AgentName } from '../sdk/types.js'; +import type { AgentName, AgentTransport } from '../sdk/types.js'; export interface CredentialPorts { /** Read a host environment variable. */ @@ -52,6 +52,11 @@ export interface CredentialResult { linkFromHome?: string[]; } +export interface CredentialResolutionOptions { + mode: 'project' | 'standalone'; + transport?: AgentTransport; +} + const EMPTY: CredentialResult = { env: {}, setupCommands: [], copyFromHome: [] }; /** Default ports using real process.env, Keychain, and filesystem. */ @@ -97,14 +102,19 @@ export async function resolveCredentials( agent: AgentName, userEnv: Record, ports?: CredentialPorts, + options: CredentialResolutionOptions = { mode: 'project' }, ): Promise { const p = ports ?? defaultPorts(); switch (agent) { case 'claude': - return resolveClaude(userEnv, p); + return options.mode === 'standalone' + ? resolveStandaloneClaude(userEnv, p) + : resolveClaude(userEnv, p); case 'codex': - return resolveCodex(userEnv, p); + return options.mode === 'standalone' + ? resolveStandaloneCodex(userEnv, p, options.transport) + : resolveCodex(userEnv, p); case 'cursor': return resolveCursor(userEnv, p); default: @@ -112,6 +122,52 @@ export async function resolveCredentials( } } +async function resolveStandaloneClaude( + userEnv: Record, + ports: CredentialPorts, +): Promise { + if (userEnv.PATHGRADE_CLAUDE_LOCAL_OAUTH === '1') { + if (userEnv.ANTHROPIC_API_KEY || userEnv.ANTHROPIC_BASE_URL) { + throw new Error('PATHGRADE_CLAUDE_LOCAL_OAUTH cannot be combined with Anthropic API-key credentials'); + } + return EMPTY; + } + + const result = await resolveClaude(userEnv, ports); + if ( + userEnv.ANTHROPIC_API_KEY + || result.env.ANTHROPIC_API_KEY + || result.env.PATHGRADE_CLAUDE_LOCAL_OAUTH === '1' + ) { + return result; + } + throw new Error('Claude authentication required for pathgrade standalone'); +} + +function resolveStandaloneCodex( + userEnv: Record, + ports: CredentialPorts, + transport: AgentTransport | undefined, +): CredentialResult { + if (transport !== undefined && transport !== 'app-server') { + throw new Error('pathgrade standalone supports Codex app-server only'); + } + + const userKey = userEnv.OPENAI_API_KEY; + const hostKey = ports.hostEnv('OPENAI_API_KEY'); + if (!userKey && !hostKey) { + throw new Error('Codex authentication required for pathgrade standalone; set OPENAI_API_KEY'); + } + + const env: Record = {}; + if (!userKey && hostKey) env.OPENAI_API_KEY = hostKey; + if (!userEnv.OPENAI_BASE_URL) { + const hostBaseUrl = ports.hostEnv('OPENAI_BASE_URL'); + if (hostBaseUrl) env.OPENAI_BASE_URL = hostBaseUrl; + } + return { env, setupCommands: [], copyFromHome: [] }; +} + async function resolveClaude( userEnv: Record, ports: CredentialPorts, diff --git a/src/providers/sandbox.ts b/src/providers/sandbox.ts index 5cd28db..c79a2bd 100644 --- a/src/providers/sandbox.ts +++ b/src/providers/sandbox.ts @@ -18,6 +18,10 @@ export interface SandboxConfig { * When omitted, DEFAULT_COPY_IGNORE is used. */ copyIgnore?: string[]; + /** Internal credential policy propagated by standalone mode. */ + credentialMode?: 'project' | 'standalone'; + /** Internal agent transport used during credential resolution. */ + transport?: import('../sdk/types.js').AgentTransport; } export interface Sandbox { diff --git a/src/providers/workspace.ts b/src/providers/workspace.ts index bc7c1c0..48ced90 100644 --- a/src/providers/workspace.ts +++ b/src/providers/workspace.ts @@ -43,14 +43,24 @@ export async function linkPathsFromHostHome(pathsToLink: string[], sandboxHomePa } export async function prepareWorkspace(spec: SandboxConfig): Promise { - const { mcp, ...sandboxSpec } = spec; + const { + mcp, + credentialMode = 'project', + transport, + ...sandboxSpec + } = spec; const sandbox = await createSandbox(sandboxSpec); const { workspacePath, homePath, env: sandboxEnv, rootDir } = sandbox; try { // Resolve credentials: pass user's original env (not sandboxEnv) so // the resolver can distinguish explicit user intent from auto-resolved values. - const creds = await resolveCredentials(spec.agent, spec.env ?? {}); + const creds = await resolveCredentials( + spec.agent, + spec.env ?? {}, + undefined, + { mode: credentialMode, transport }, + ); Object.assign(sandboxEnv, creds.env); await copyPathsFromHostHome(creds.copyFromHome, homePath); await linkPathsFromHostHome(creds.linkFromHome ?? [], homePath); diff --git a/src/reporters/verbose-emitter.ts b/src/reporters/verbose-emitter.ts index 33230d5..8bb8a6f 100644 --- a/src/reporters/verbose-emitter.ts +++ b/src/reporters/verbose-emitter.ts @@ -35,6 +35,7 @@ export interface VerboseEmitterOptions { * Helps the reader locate the beginning of each test block. */ testName?: string; + agentName?: string; } export interface TurnStartArgs { @@ -129,6 +130,11 @@ export function createVerboseEmitter(opts: VerboseEmitterOptions): VerboseEmitte } const sink = opts.sink ?? stderrSink(); const testName = opts.testName; + const agentLabel = opts.agentName?.toUpperCase().padEnd(7); + const traceColor = process.env.PATHGRADE_UI_COLOR === '1'; + const tracePaint = (code: number, value: string) => traceColor + ? `\x1b[${code}m${value}\x1b[0m` + : value; let headerPrinted = false; const emitHeaderOnce = () => { @@ -149,39 +155,47 @@ export function createVerboseEmitter(opts: VerboseEmitterOptions): VerboseEmitte turnStart({ turn, kind, message }) { const p = preview(message, PREVIEW_MAX_CHARS); - writeLine(`${fmt.cyan('→')} Turn ${fmt.bold(String(turn))} ${fmt.dim(`[${kind}]`)} "${p}"`); + writeLine(agentLabel + ? `${agentLabel} ${tracePaint(36, 'TURN')} ${tracePaint(1, String(turn))} ${tracePaint(2, `[${kind}]`)} "${p}"` + : `${fmt.cyan('→')} Turn ${fmt.bold(String(turn))} ${fmt.dim(`[${kind}]`)} "${p}"`); }, toolEvent({ action, summary }) { - writeLine(` ${fmt.dim('·')} ${fmt.cyan(action)} ${summary}`); + writeLine(agentLabel + ? `${agentLabel} ${tracePaint(36, 'TOOL')} ${action} ${summary}` + : ` ${fmt.dim('·')} ${fmt.cyan(action)} ${summary}`); }, turnEnd({ turn, durationMs, outputLines, messagePreview }) { const p = preview(messagePreview, PREVIEW_MAX_CHARS); const duration = formatDurationSeconds(durationMs); - writeLine( - `${fmt.green('←')} Turn ${fmt.bold(String(turn))} ${fmt.dim(duration)} ${fmt.dim(`${outputLines}l`)} "${p}"`, - ); + writeLine(agentLabel + ? `${agentLabel} ${tracePaint(32, 'TURN')} ${tracePaint(1, String(turn))} ${tracePaint(2, duration)} ${tracePaint(2, `${outputLines}l`)} "${p}"` + : `${fmt.green('←')} Turn ${fmt.bold(String(turn))} ${fmt.dim(duration)} ${fmt.dim(`${outputLines}l`)} "${p}"`); }, retry({ attempt, maxAttempts, errorMessage }) { const msg = preview(errorMessage, RETRY_ERROR_MAX_CHARS); - writeLine(` ${fmt.red('⟲')} retry ${attempt}/${maxAttempts}: ${msg}`); + writeLine(agentLabel + ? `${agentLabel} ${tracePaint(33, 'RETRY')} ${attempt}/${maxAttempts}: ${msg}` + : ` ${fmt.red('⟲')} retry ${attempt}/${maxAttempts}: ${msg}`); }, reactionFired({ reactionIndex, pattern, reply }) { const p = preview(reply, PREVIEW_MAX_CHARS); - writeLine(` ${fmt.cyan('⚡')} reaction ${fmt.bold(`#${reactionIndex}`)} ${fmt.dim(pattern)} → "${p}"`); + writeLine(agentLabel + ? `${agentLabel} ${tracePaint(35, 'REACTION')} ${tracePaint(1, `#${reactionIndex}`)} ${tracePaint(2, pattern)} → "${p}"` + : ` ${fmt.cyan('⚡')} reaction ${fmt.bold(`#${reactionIndex}`)} ${fmt.dim(pattern)} → "${p}"`); }, conversationEnd({ reason, turns, durationMs, detail }) { const duration = formatDurationSeconds(durationMs); const detailSuffix = detail - ? ` ${fmt.red('detail=')}${preview(detail, RETRY_ERROR_MAX_CHARS)}` + ? ` ${agentLabel ? tracePaint(31, 'detail=') : fmt.red('detail=')}${preview(detail, RETRY_ERROR_MAX_CHARS)}` : ''; - writeLine( - `${fmt.bold('■')} end ${fmt.dim('reason=')}${reason} ${fmt.dim('turns=')}${turns} ${fmt.dim(duration)}${detailSuffix}`, - ); + writeLine(agentLabel + ? `${agentLabel} ${tracePaint(1, 'END')} ${tracePaint(2, 'reason=')}${reason} ${tracePaint(2, 'turns=')}${turns} ${tracePaint(2, duration)}${detailSuffix}` + : `${fmt.bold('■')} end ${fmt.dim('reason=')}${reason} ${fmt.dim('turns=')}${turns} ${fmt.dim(duration)}${detailSuffix}`); }, }; } diff --git a/src/reporting/core.ts b/src/reporting/core.ts index cf19c83..3f5eac8 100644 --- a/src/reporting/core.ts +++ b/src/reporting/core.ts @@ -70,6 +70,7 @@ export function buildPathgradeReport(input: ReportRunInput): PathgradeReportBuil status, groups: consolidatedGroups, ...(input.selection ? { selection: input.selection } : {}), + ...(input.provenance ? { provenance: input.provenance } : {}), }, traces, summaries, diff --git a/src/reporting/types.ts b/src/reporting/types.ts index 7a4d14f..8e365eb 100644 --- a/src/reporting/types.ts +++ b/src/reporting/types.ts @@ -1,11 +1,13 @@ import type { DiagnosticsReport } from '../sdk/diagnostics.js'; import type { PathgradeReport, PathgradeSelectionReport, TrialResult } from '../types.js'; +import type { StandaloneRunProvenance } from '../standalone/provenance.js'; export type ReportCaseState = 'passed' | 'failed' | 'skipped' | 'pending'; export interface ReportRunInput { threshold?: number; selection?: PathgradeSelectionReport; + provenance?: StandaloneRunProvenance; groups: ReportGroupInput[]; } diff --git a/src/runners/adapter-loader.ts b/src/runners/adapter-loader.ts index c22c494..5d6e1d8 100644 --- a/src/runners/adapter-loader.ts +++ b/src/runners/adapter-loader.ts @@ -6,6 +6,7 @@ import { createJestInvocationAdapter } from '../adapters/jest/invocation-adapter import { createJestAdapter } from '../adapters/jest/runner-adapter.js'; import { createNodeTestInvocationAdapter } from '../adapters/node-test/invocation-adapter.js'; import { createVitestInvocationAdapter, type SpawnVitest } from './vitest-invocation.js'; +import { createStandaloneVitestInvocationAdapter } from '../standalone/vitest-invocation.js'; import type { RunnerAdapter } from './adapter.js'; import type { RunnerInvocationAdapter } from './invocation.js'; import { resolveRunnerAdapter } from './selection.js'; @@ -44,8 +45,12 @@ export async function loadRunnerInvocationAdapter(input: { cwd?: string; config: ResolvedPathgradeConfig; spawnVitest?: SpawnVitest; + standalone?: boolean; }): Promise { const name = input.adapterName ?? 'vitest'; + if (name === 'vitest' && input.standalone) { + return createStandaloneVitestInvocationAdapter({ config: input.config }); + } if (name === 'vitest') return createVitestInvocationAdapter({ spawnVitest: input.spawnVitest }); if (name === 'node-test') return createNodeTestInvocationAdapter({ config: input.config }); if (name === 'jest') return createJestInvocationAdapter({ config: input.config }); diff --git a/src/runners/orchestrator.ts b/src/runners/orchestrator.ts index 7876e73..a339101 100644 --- a/src/runners/orchestrator.ts +++ b/src/runners/orchestrator.ts @@ -3,6 +3,8 @@ import { writePathgradeArtifacts } from '../reporting/artifacts.js'; import { projectNormalizedRunSnapshotToReportInput } from './report-projection.js'; import type { PathgradeSelectionReport } from '../types.js'; import type { ReportSummaryGroup } from '../reporting/types.js'; +import type { ArtifactWriteResult, PathgradeReportBuildResult } from '../reporting/types.js'; +import type { StandaloneRunProvenance } from '../standalone/provenance.js'; import type { AdapterDiscoveryInput, AdapterLifecycleHooks, @@ -21,6 +23,7 @@ export interface PathgradeRunOptions { reporterMode?: AdapterReporterMode; threshold?: number; selection?: PathgradeSelectionReport; + provenance?: StandaloneRunProvenance; lifecycle?: AdapterLifecycleHooks; signal?: AbortSignal; printSummary?: (summaries: ReportSummaryGroup[]) => void; @@ -30,6 +33,10 @@ export interface PathgradeRunOptions { writeEmptyReport?: boolean; log?: (message: string) => void; warn?: (message: string) => void; + onArtifactsWritten?: (input: { + built: PathgradeReportBuildResult; + artifacts: ArtifactWriteResult; + }) => void | Promise; } export async function runWithAdapter(input: { @@ -59,6 +66,7 @@ export async function runWithAdapter(input: { ); let built = buildPathgradeReport({ threshold: options.threshold, + provenance: options.provenance, ...reportInput, }); @@ -75,6 +83,7 @@ export async function runWithAdapter(input: { built = buildPathgradeReport({ threshold: options.threshold, selection: loadedSelection, + provenance: options.provenance, groups: reportInput.groups, }); } @@ -84,7 +93,8 @@ export async function runWithAdapter(input: { options.printSummary?.(built.summaries); } - await writePathgradeArtifacts(options.artifactRoot, built); + const artifacts = await writePathgradeArtifacts(options.artifactRoot, built); + await options.onArtifactsWritten?.({ built, artifacts }); options.log?.(`results:${options.artifactRoot}`); if (mode === 'browser') { diff --git a/src/runners/vitest-lifecycle.ts b/src/runners/vitest-lifecycle.ts index 53c3532..dbc3a90 100644 --- a/src/runners/vitest-lifecycle.ts +++ b/src/runners/vitest-lifecycle.ts @@ -188,7 +188,16 @@ function currentFileContext(): CaseContext | null { function recordVitestResult(result: RecordedEvalResult, agent: Agent): void { const current = getCurrentCaseContext(); const taskId = currentTaskId(); - const owner = lifecycleCore.getAgentOwner(agent); + let owner = lifecycleCore.getAgentOwner(agent); + if (!owner && current.status === 'active') { + owner = current.context.scope === 'runner-case' + ? { type: 'runner-case', caseId: current.context.caseId } + : { type: 'runner-suite-shared', caseId: current.context.caseId }; + lifecycleCore.registerAgent(agent, owner); + } else if (!owner && taskId) { + owner = { type: 'runner-case', caseId: taskId }; + lifecycleCore.registerAgent(agent, owner); + } if (owner?.type !== 'runner-case' && (current.status !== 'active' || current.context.scope !== 'runner-case')) { if (!taskId) { lifecycleCore.recordResult(result, agent); diff --git a/src/sdk/agent-resolution.ts b/src/sdk/agent-resolution.ts index 4496375..56cf105 100644 --- a/src/sdk/agent-resolution.ts +++ b/src/sdk/agent-resolution.ts @@ -9,6 +9,13 @@ export class InvalidTransportEnvError extends Error { } } +export class StandaloneCodexTransportError extends Error { + constructor() { + super("pathgrade standalone supports Codex app-server only; remove transport: 'exec' or use project-local @wix/pathgrade"); + this.name = 'StandaloneCodexTransportError'; + } +} + export function resolveAgentName( opts: Pick, env: { PATHGRADE_AGENT?: string }, @@ -18,15 +25,35 @@ export function resolveAgentName( export function resolveCodexTransport( opts: { transport?: AgentTransport }, - env: { PATHGRADE_CODEX_TRANSPORT?: string }, + env: { PATHGRADE_CODEX_TRANSPORT?: string; PATHGRADE_STANDALONE?: string }, ): AgentTransport { - if (opts.transport) return opts.transport; + if (opts.transport) { + if (opts.transport === 'exec' && env.PATHGRADE_STANDALONE === '1') { + throw new StandaloneCodexTransportError(); + } + return opts.transport; + } const envValue = env.PATHGRADE_CODEX_TRANSPORT; if (envValue) { if (envValue !== 'exec' && envValue !== 'app-server') { throw new InvalidTransportEnvError(envValue); } + if (envValue === 'exec' && env.PATHGRADE_STANDALONE === '1') { + throw new StandaloneCodexTransportError(); + } return envValue; } return 'app-server'; } + +export function assertStandaloneAgent( + agent: AgentName, + transport?: AgentTransport, +): void { + if (agent === 'cursor') { + throw new Error('Cursor is unsupported in pathgrade standalone'); + } + if (agent === 'codex' && transport !== 'app-server') { + throw new Error('pathgrade standalone supports Codex app-server only'); + } +} diff --git a/src/sdk/agent.ts b/src/sdk/agent.ts index e87a15e..69ec022 100644 --- a/src/sdk/agent.ts +++ b/src/sdk/agent.ts @@ -13,10 +13,15 @@ import type { ConverseOptions, Message, Agent, + AgentInvocationProvenance, AgentOptions, } from './types.js'; import type { McpSafetyOptions } from './mcp-safety.js'; -import { resolveAgentName, resolveCodexTransport } from './agent-resolution.js'; +import { + assertStandaloneAgent, + resolveAgentName, + resolveCodexTransport, +} from './agent-resolution.js'; import { lifecycleCore } from './lifecycle.js'; import { ChatSessionImpl } from './chat.js'; import { runConversation } from './converse.js'; @@ -32,6 +37,9 @@ import { getCurrentCaseContext } from './case-context.js'; import { createVerboseEmitter, type VerboseEmitter, type VerboseSink } from '../reporters/verbose-emitter.js'; import fs from 'fs-extra'; import * as path from 'path'; +import { isStandaloneMode } from '../standalone/mode.js'; +import { verifyBundledClaudeRuntime } from '../agents/claude-runtime.js'; +import { resolveBundledCodexCommand, verifyBundledCodexRuntime } from '../agents/codex-runtime.js'; /** * Test-only injection point: override the sink used by the next emitter @@ -63,8 +71,9 @@ class AgentImpl implements Agent { readonly verbose: VerboseEmitter; private transport?: AgentTransport; private mcpSafety?: McpSafetyOptions; + readonly provenance?: AgentInvocationProvenance; - constructor(ws: Workspace, agentName: AgentName, llm: LLMPort, timeoutSetting: number | 'auto', conversationWindow: ConversationWindowConfig | false | undefined, modelOpt: string | undefined, debugOpt: boolean | string | undefined, debugName: string, debugBaseDir: string, verbose: VerboseEmitter, transport?: AgentTransport, mcpSafety?: McpSafetyOptions) { + constructor(ws: Workspace, agentName: AgentName, llm: LLMPort, timeoutSetting: number | 'auto', conversationWindow: ConversationWindowConfig | false | undefined, modelOpt: string | undefined, debugOpt: boolean | string | undefined, debugName: string, debugBaseDir: string, verbose: VerboseEmitter, transport?: AgentTransport, mcpSafety?: McpSafetyOptions, provenance?: AgentInvocationProvenance) { this.ws = ws; this.agentName = agentName; this.llm = llm; @@ -77,6 +86,7 @@ class AgentImpl implements Agent { this.verbose = verbose; this.transport = transport; this.mcpSafety = mcpSafety; + this.provenance = provenance; } get messages(): Message[] { @@ -371,6 +381,7 @@ class AgentImpl implements Agent { if (this.interactionMode === 'runConversation' && this.lastConversationResult) { const snapshot = buildRunSnapshot({ agent: this.agentName, + ...(this.provenance ? { agent_provenance: this.provenance } : {}), messages: this._messages, log: this._log, conversationResult: this.lastConversationResult, @@ -398,12 +409,13 @@ function slugify(s: string): string { .toLowerCase(); } -function resolveCaseDebugContext(): { name: string; dir: string } { +function resolveCaseContext(): { displayName: string; debugName: string; dir: string } { const current = getCurrentCaseContext(); - if (current.status !== 'active') return { name: '', dir: '' }; + if (current.status !== 'active') return { displayName: '', debugName: '', dir: '' }; return { - name: current.context.caseName ? slugify(current.context.caseName) : '', + displayName: current.context.caseName ?? '', + debugName: current.context.caseName ? slugify(current.context.caseName) : '', dir: current.context.filePath ? path.dirname(current.context.filePath) : '', }; } @@ -413,34 +425,100 @@ export async function createAgent(opts: AgentOptions): Promise { const transport: AgentTransport | undefined = agentName === 'codex' ? resolveCodexTransport(opts, process.env) : undefined; + const standalone = isStandaloneMode(process.env); + if (standalone) { + assertStandaloneAgent(agentName, transport); + } const timeoutSetting = opts.timeout ?? 300; // Capture runner context now; adapters own installation and restoration. - const testCtx = opts.debug ? resolveCaseDebugContext() : { name: '', dir: '' }; + const caseContext = resolveCaseContext(); + const testCtx = opts.debug + ? { name: caseContext.debugName, dir: caseContext.dir } + : { name: '', dir: '' }; const { timeout: _, mcpMock, mcpConfigFile, agent: __, debug: ___, model: ____, transport: _____, mcpSafety: ______, ...rest } = opts; const workspace = await prepareWorkspace({ ...rest, agent: agentName, + credentialMode: standalone ? 'standalone' : 'project', + transport, mcp: mcpConfigFile ? { configFile: mcpConfigFile } : mcpMock ? { mock: mcpMock } : undefined, }); - // Create agent LLM once, using the fully-resolved sandbox env (includes - // keychain OAuth tokens, API keys, safe host vars). - const llm = createAgentLLM(agentName, workspace.env); + try { + // Create agent LLM once, using the fully-resolved sandbox env (includes + // keychain OAuth tokens, API keys, safe host vars). + const llm = createAgentLLM(agentName, workspace.env); + + // Fall back to sandbox dir name if no test name resolved + const debugName = testCtx.name || path.basename(path.dirname(workspace.path)); + // Default debug dir is next to the eval file, fallback to cwd + const debugBaseDir = testCtx.dir || process.cwd(); + + const verbose = createVerboseEmitter({ + enabled: process.env.PATHGRADE_VERBOSE === '1', + sink: verboseSinkOverride ?? undefined, + testName: caseContext.displayName || undefined, + ...(standalone ? { agentName } : {}), + }); - // Fall back to sandbox dir name if no test name resolved - const debugName = testCtx.name || path.basename(path.dirname(workspace.path)); - // Default debug dir is next to the eval file, fallback to cwd - const debugBaseDir = testCtx.dir || process.cwd(); + const provenance = standalone + ? await buildStandaloneAgentInvocationProvenance({ + agentName, + transport, + model: opts.model, + workspaceEnv: workspace.env, + }) + : undefined; + const agent = new AgentImpl(workspace, agentName, llm, timeoutSetting, opts.conversationWindow, opts.model, opts.debug, debugName, debugBaseDir, verbose, transport, opts.mcpSafety, provenance); + lifecycleCore.registerAgent(agent); + return agent; + } catch (error) { + await workspace.dispose().catch(() => {}); + throw error; + } +} - const verbose = createVerboseEmitter({ - enabled: process.env.PATHGRADE_VERBOSE === '1', - sink: verboseSinkOverride ?? undefined, - testName: testCtx.name || undefined, - }); +async function buildStandaloneAgentInvocationProvenance(input: { + agentName: AgentName; + transport?: AgentTransport; + model?: string; + workspaceEnv: Record; +}): Promise { + if (input.agentName === 'claude') { + const runtime = await verifyBundledClaudeRuntime(); + return { + agent: 'claude', + transport: input.transport ?? 'native', + model: input.model === undefined + ? { id: null, source: 'provider-default' } + : { id: input.model, source: 'user' }, + authentication: input.workspaceEnv.PATHGRADE_CLAUDE_LOCAL_OAUTH === '1' + ? 'claude-oauth' + : 'api-key', + runtime: { + package: '@anthropic-ai/claude-agent-sdk', + package_version: runtime.sdkVersion, + embedded_binary_version: runtime.embeddedBinaryVersion, + provenance: runtime.provenance, + }, + }; + } - const agent = new AgentImpl(workspace, agentName, llm, timeoutSetting, opts.conversationWindow, opts.model, opts.debug, debugName, debugBaseDir, verbose, transport, opts.mcpSafety); - lifecycleCore.registerAgent(agent); - return agent; + const runtime = await verifyBundledCodexRuntime(resolveBundledCodexCommand()); + return { + agent: 'codex', + transport: input.transport ?? 'native', + model: input.model === undefined + ? { id: 'gpt-5.4', source: 'pathgrade-default' } + : { id: input.model, source: 'user' }, + authentication: 'api-key', + runtime: { + package: '@openai/codex', + package_version: runtime.packageVersion, + embedded_binary_version: runtime.nativeVersion, + provenance: runtime.provenance, + }, + }; } diff --git a/src/sdk/evaluate.ts b/src/sdk/evaluate.ts index 758b0f1..741dbc3 100644 --- a/src/sdk/evaluate.ts +++ b/src/sdk/evaluate.ts @@ -120,6 +120,7 @@ function makeEvaluateAgent() { { ...evalResult, tokenUsage: deltaTokenUsage }, conversationTokens, conversationCost, + agent.provenance, ), }; emitEvalResult({ result: recordedResult, agent }); @@ -161,7 +162,7 @@ async function fromSnapshot( const evalResult = await evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM }); const recordedResult: RecordedEvalResult = { ...evalResult, - trial: buildTrialResult(snapshot.log, evalResult), + trial: buildTrialResult(snapshot.log, evalResult, undefined, undefined, snapshot.agent_provenance), }; maybeThrowOnScorerErrors(recordedResult, opts?.onScorerError ?? 'skip'); return recordedResult; @@ -329,6 +330,7 @@ function buildTrialResult( result: EvalResult, conversationTokens?: { conversation_input_tokens: number; conversation_output_tokens: number }, conversationCost?: { conversation_cost_usd: number }, + agentProvenance?: import('./types.js').AgentInvocationProvenance, ): TrialResult { const nCommands = log.filter((entry) => entry.type === 'command').length; const skills = extractSkillsFromLog(log); @@ -349,6 +351,7 @@ function buildTrialResult( // guaranteed cost surface; future judge-cost work unlocks the total // field. ...conversationCost, + ...(agentProvenance ? { agent_provenance: agentProvenance } : {}), session_log: [...log], ...(skills.length > 0 ? { skills_used: skills } : {}), }; diff --git a/src/sdk/index.ts b/src/sdk/index.ts index ee1d54e..79ed06e 100644 --- a/src/sdk/index.ts +++ b/src/sdk/index.ts @@ -5,6 +5,7 @@ export { resolveAgentName, resolveCodexTransport, InvalidTransportEnvError, + StandaloneCodexTransportError, } from './agent-resolution.js'; export { AgentCrashError } from './agent-crash.js'; export { check, score, judge, toolUsage } from './scorers.js'; @@ -52,6 +53,7 @@ export { export { getAgentCapabilities } from './types.js'; export type { AgentTransport, + AgentInvocationProvenance, AgentCapabilities, AgentName, McpRunMode, diff --git a/src/sdk/snapshots.ts b/src/sdk/snapshots.ts index 2bc45bd..983f892 100644 --- a/src/sdk/snapshots.ts +++ b/src/sdk/snapshots.ts @@ -1,6 +1,6 @@ import type { LogEntry } from '../types.js'; import type { ToolEvent } from '../tool-events.js'; -import type { AgentName, ConversationResult, Message, TurnTiming } from './types.js'; +import type { AgentInvocationProvenance, AgentName, ConversationResult, Message, TurnTiming } from './types.js'; import fs from 'fs-extra'; export const RUN_SNAPSHOT_VERSION = 1; @@ -9,6 +9,7 @@ export interface RunSnapshot { version: 1; timestamp: string; agent: AgentName; + agent_provenance?: AgentInvocationProvenance; messages: Message[]; log: LogEntry[]; toolEvents: ToolEvent[]; @@ -24,13 +25,14 @@ export interface RunSnapshot { export function buildRunSnapshot(params: { agent: AgentName; + agent_provenance?: AgentInvocationProvenance; messages: Message[]; log: LogEntry[]; conversationResult: ConversationResult; workspace: string | null; timestamp?: string; }): RunSnapshot { - const { agent, messages, log, conversationResult, workspace, timestamp } = params; + const { agent, agent_provenance, messages, log, conversationResult, workspace, timestamp } = params; const toolEvents = log .filter((entry) => entry.type === 'tool_event' && entry.tool_event) .map((entry) => entry.tool_event as ToolEvent); @@ -39,6 +41,7 @@ export function buildRunSnapshot(params: { version: RUN_SNAPSHOT_VERSION, timestamp: timestamp ?? new Date().toISOString(), agent, + ...(agent_provenance ? { agent_provenance } : {}), messages: [...messages], log: [...log], toolEvents, @@ -163,6 +166,9 @@ function validateRunSnapshot(input: unknown): RunSnapshot { if (typeof snapshot.conversationResult.completionReason !== 'string') { throw new SnapshotParseError('Snapshot conversationResult is missing required field: completionReason'); } + if (snapshot.agent_provenance !== undefined && !isAgentInvocationProvenance(snapshot.agent_provenance)) { + throw new SnapshotParseError('Snapshot agent_provenance is invalid'); + } const turnTimings = Array.isArray(snapshot.turnTimings) ? snapshot.turnTimings @@ -172,6 +178,7 @@ function validateRunSnapshot(input: unknown): RunSnapshot { version: RUN_SNAPSHOT_VERSION, timestamp: typeof snapshot.timestamp === 'string' ? snapshot.timestamp : new Date(0).toISOString(), agent: snapshot.agent === 'claude' || snapshot.agent === 'codex' || snapshot.agent === 'cursor' ? snapshot.agent : 'claude', + ...(snapshot.agent_provenance ? { agent_provenance: snapshot.agent_provenance } : {}), messages: snapshot.messages, log: snapshot.log, toolEvents: snapshot.toolEvents, @@ -187,3 +194,21 @@ function validateRunSnapshot(input: unknown): RunSnapshot { workspace: typeof snapshot.workspace === 'string' ? snapshot.workspace : null, }; } + +function isAgentInvocationProvenance(value: unknown): value is AgentInvocationProvenance { + if (!value || typeof value !== 'object') return false; + const record = value as Record; + const model = record.model as Record | undefined; + const runtime = record.runtime as Record | undefined; + const hasValidModel = (typeof model?.id === 'string' + && (model.source === 'user' || model.source === 'pathgrade-default')) + || (model?.id === null && model.source === 'provider-default'); + return (record.agent === 'claude' || record.agent === 'codex' || record.agent === 'cursor') + && (record.transport === 'native' || record.transport === 'exec' || record.transport === 'app-server') + && hasValidModel + && (record.authentication === 'api-key' || record.authentication === 'claude-oauth') + && typeof runtime?.package === 'string' + && typeof runtime?.package_version === 'string' + && (runtime?.embedded_binary_version === undefined || typeof runtime.embedded_binary_version === 'string') + && (runtime?.provenance === 'bundled' || runtime?.provenance === 'project'); +} diff --git a/src/sdk/types.ts b/src/sdk/types.ts index 890d335..59c0b01 100644 --- a/src/sdk/types.ts +++ b/src/sdk/types.ts @@ -10,6 +10,21 @@ import type { McpSafetyOptions } from './mcp-safety.js'; export type AgentName = 'claude' | 'codex' | 'cursor'; +export interface AgentInvocationProvenance { + agent: AgentName; + transport: 'native' | AgentTransport; + model: + | { id: string; source: 'user' | 'pathgrade-default' } + | { id: null; source: 'provider-default' }; + authentication: 'api-key' | 'claude-oauth'; + runtime: { + package: string; + package_version: string; + embedded_binary_version?: string; + provenance: 'bundled' | 'project'; + }; +} + // --- Agent --- export interface AgentOptions { @@ -76,6 +91,7 @@ export interface Agent { readonly messages: Message[]; readonly log: LogEntry[]; readonly workspace: string; + readonly provenance?: AgentInvocationProvenance; dispose(): Promise; } diff --git a/src/standalone/diagnostics.ts b/src/standalone/diagnostics.ts new file mode 100644 index 0000000..e3efcf1 --- /dev/null +++ b/src/standalone/diagnostics.ts @@ -0,0 +1,70 @@ +export interface StandaloneVitestFailure { + kind: 'packaging' | 'project-dependency'; + message: string; +} + +export function classifyStandaloneVitestFailure( + stderr: string, +): StandaloneVitestFailure | undefined { + const unresolved = parseUnresolvedImport(stderr); + if (!unresolved) return undefined; + + const { specifier, importer } = unresolved; + if (isBundledSpecifier(specifier) || isInstalledToolImporter(importer)) { + return { + kind: 'packaging', + message: + `pathgrade standalone: bundled dependency "${specifier}" could not be resolved; ` + + 'this is a Pathgrade packaging defect', + }; + } + if (!isBareSpecifier(specifier)) return undefined; + + return { + kind: 'project-dependency', + message: + `pathgrade standalone: project dependency "${specifier}" is unavailable in ${importer}; ` + + 'install the application dependencies or remove that import', + }; +} + +function parseUnresolvedImport( + stderr: string, +): { specifier: string; importer: string } | undefined { + const vite = stderr.match( + /Failed to resolve import\s+["']([^"']+)["']\s+from\s+["']([^"']+)["']/, + ); + if (vite) return { specifier: vite[1], importer: vite[2] }; + + const node = stderr.match( + /Cannot find package\s+["']([^"']+)["']\s+imported from\s+([^\s\n]+)/, + ); + if (!node) return undefined; + return { + specifier: node[1], + importer: node[2].replace(/^["']|["']$/g, ''), + }; +} + +function isBundledSpecifier(specifier: string): boolean { + return specifier === 'vitest' + || specifier.startsWith('vitest/') + || specifier === '@wix/pathgrade' + || specifier.startsWith('@wix/pathgrade/'); +} + +function isInstalledToolImporter(importer: string): boolean { + const normalized = importer.replaceAll('\\', '/'); + return [ + '/node_modules/@wix/pathgrade/', + '/node_modules/vitest/', + '/node_modules/@anthropic-ai/claude-agent-sdk/', + '/node_modules/@openai/codex/', + ].some(segment => normalized.includes(segment)); +} + +function isBareSpecifier(specifier: string): boolean { + return !specifier.startsWith('.') + && !specifier.startsWith('/') + && !specifier.startsWith('file:'); +} diff --git a/src/standalone/mode.ts b/src/standalone/mode.ts new file mode 100644 index 0000000..ab6ff45 --- /dev/null +++ b/src/standalone/mode.ts @@ -0,0 +1,19 @@ +export const PATHGRADE_STANDALONE_ENV = 'PATHGRADE_STANDALONE' as const; + +export function isStandaloneMode(env: NodeJS.ProcessEnv = process.env): boolean { + return env[PATHGRADE_STANDALONE_ENV] === '1'; +} + +export function parseStandaloneCommand( + args: readonly string[], +): { standalone: boolean; args: string[] } { + if (args[0] !== 'standalone') { + return { standalone: false, args: [...args] }; + } + + const nested = args.slice(1); + return { + standalone: true, + args: nested.length === 0 ? ['run'] : [...nested], + }; +} diff --git a/src/standalone/module-aliases.ts b/src/standalone/module-aliases.ts new file mode 100644 index 0000000..3d9a7f8 --- /dev/null +++ b/src/standalone/module-aliases.ts @@ -0,0 +1,136 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +export const STANDALONE_VITEST_PAYLOAD_ENV = + 'PATHGRADE_STANDALONE_VITEST_PAYLOAD' as const; + +export interface StandaloneVitestPayload { + root: string; + include: string[]; + exclude: string[]; + diagnostics: boolean; + reporter?: 'cli' | 'browser' | 'json'; + threshold?: number; + cacheDir: string; +} + +export function encodeStandaloneVitestPayload( + payload: StandaloneVitestPayload, +): string { + return Buffer.from(JSON.stringify(payload), 'utf8').toString('base64url'); +} + +export function decodeStandaloneVitestPayload( + value: string | undefined, +): StandaloneVitestPayload { + if (!value) throw new Error('pathgrade standalone: missing internal Vitest payload'); + + let parsed: unknown; + try { + parsed = JSON.parse(Buffer.from(value, 'base64url').toString('utf8')); + } catch { + throw new Error('pathgrade standalone: invalid internal Vitest payload'); + } + if (!isRecord(parsed)) { + throw new Error('pathgrade standalone: invalid internal Vitest payload'); + } + + const root = requireAbsolutePath(parsed.root, 'root'); + const cacheDir = requireAbsolutePath(parsed.cacheDir, 'cacheDir'); + const include = requireStringArray(parsed.include, 'include'); + const exclude = requireStringArray(parsed.exclude, 'exclude'); + if (typeof parsed.diagnostics !== 'boolean') { + throw new Error('pathgrade standalone: payload diagnostics must be a boolean'); + } + if ( + parsed.reporter !== undefined + && parsed.reporter !== 'cli' + && parsed.reporter !== 'browser' + && parsed.reporter !== 'json' + ) { + throw new Error('pathgrade standalone: payload reporter is unsupported'); + } + if ( + parsed.threshold !== undefined + && (typeof parsed.threshold !== 'number' || !Number.isFinite(parsed.threshold)) + ) { + throw new Error('pathgrade standalone: payload threshold must be finite'); + } + + return { + root, + include, + exclude, + diagnostics: parsed.diagnostics, + ...(parsed.reporter === undefined ? {} : { reporter: parsed.reporter }), + ...(parsed.threshold === undefined ? {} : { threshold: parsed.threshold }), + cacheDir, + }; +} + +function requireAbsolutePath(value: unknown, field: string): string { + if (typeof value !== 'string' || !path.isAbsolute(value)) { + throw new Error(`pathgrade standalone: payload ${field} must be an absolute path`); + } + return value; +} + +function requireStringArray(value: unknown, field: string): string[] { + if (!Array.isArray(value) || value.some(item => typeof item !== 'string')) { + throw new Error(`pathgrade standalone: payload ${field} must be an array of strings`); + } + return [...value] as string[]; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +const SUPPORTED_ESM_CONDITIONS = new Set(['node', 'import', 'default']); + +export function selectStandaloneEsmExportTarget(value: unknown): string | undefined { + if (typeof value === 'string') return value; + if (!isRecord(value)) return undefined; + + for (const [condition, target] of Object.entries(value)) { + if (!SUPPORTED_ESM_CONDITIONS.has(condition)) continue; + const selected = selectStandaloneEsmExportTarget(target); + if (selected) return selected; + } + return undefined; +} + +const STANDALONE_EVAL_EXPORTS = [ + ['.', '@wix/pathgrade'], + ['./mcp-mock', '@wix/pathgrade/mcp-mock'], +] as const; + +export function resolveStandaloneModuleAliases( + packageRoot: string, + vitestEntry: string, +): Array<{ find: RegExp; replacement: string }> { + const packageJson = JSON.parse( + fs.readFileSync(path.join(packageRoot, 'package.json'), 'utf8'), + ) as { exports?: Record }; + const exportsMap = packageJson.exports ?? {}; + const aliases: Array<{ find: RegExp; replacement: string }> = []; + + for (const [exportKey, specifier] of STANDALONE_EVAL_EXPORTS) { + const target = selectStandaloneEsmExportTarget(exportsMap[exportKey]); + if (!target) continue; + aliases.push({ + find: new RegExp(`^${escapeRegExp(specifier)}$`), + replacement: path.resolve(packageRoot, target), + }); + } + + aliases.push({ + find: /^vitest$/, + replacement: vitestEntry, + }); + return aliases; +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/src/standalone/provenance.ts b/src/standalone/provenance.ts new file mode 100644 index 0000000..0f19bc4 --- /dev/null +++ b/src/standalone/provenance.ts @@ -0,0 +1,107 @@ +import { readFile } from 'node:fs/promises'; +import { verifyBundledClaudeRuntime } from '../agents/claude-runtime.js'; +import { resolveBundledCodexCommand } from '../agents/codex-runtime.js'; +import { resolveBundledVitestCli } from './vitest-runtime.js'; + +export const STANDALONE_PROVENANCE_ENV = 'PATHGRADE_STANDALONE_PROVENANCE' as const; + +export interface StandaloneRunProvenance { + mode: 'standalone'; + package_version: string; + vitest_version: string; + runtimes: { + claude: { + sdk_version: string; + claude_code_version: string; + }; + codex: { + package_version: string; + native_version: string; + }; + }; + platform: { + os: NodeJS.Platform; + arch: string; + node: string; + }; +} + +export async function buildStandaloneRunProvenance(): Promise { + const [packageVersion, vitest, claude, codex] = await Promise.all([ + readPathgradePackageVersion(), + Promise.resolve(resolveBundledVitestCli()), + verifyBundledClaudeRuntime(), + Promise.resolve(resolveBundledCodexCommand()), + ]); + + return { + mode: 'standalone', + package_version: packageVersion, + vitest_version: vitest.version, + runtimes: { + claude: { + sdk_version: claude.sdkVersion, + claude_code_version: claude.embeddedBinaryVersion, + }, + codex: { + package_version: codex.packageVersion, + native_version: codex.nativeVersion, + }, + }, + platform: { os: process.platform, arch: process.arch, node: process.version }, + }; +} + +export function encodeStandaloneRunProvenance(provenance: StandaloneRunProvenance): string { + return Buffer.from(JSON.stringify(provenance), 'utf8').toString('base64url'); +} + +export function readStandaloneRunProvenance( + env: NodeJS.ProcessEnv = process.env, +): StandaloneRunProvenance { + const encoded = env[STANDALONE_PROVENANCE_ENV]; + if (!encoded) throw packagingDefect('provenance payload is missing'); + try { + if (!/^[A-Za-z0-9_-]+$/.test(encoded)) throw new Error('invalid base64url alphabet'); + const decoded = Buffer.from(encoded, 'base64url'); + if (decoded.toString('base64url') !== encoded) throw new Error('noncanonical base64url payload'); + const parsed = JSON.parse(decoded.toString('utf8')) as unknown; + if (!isStandaloneRunProvenance(parsed)) throw new Error('invalid shape'); + return parsed; + } catch { + throw packagingDefect('provenance payload is invalid'); + } +} + +async function readPathgradePackageVersion(): Promise { + try { + const packageJson = JSON.parse(await readFile(new URL('../../package.json', import.meta.url), 'utf8')) as { version?: unknown }; + if (typeof packageJson.version !== 'string') throw new Error('missing version'); + return packageJson.version; + } catch { + throw packagingDefect('package metadata is unreadable'); + } +} + +function isStandaloneRunProvenance(value: unknown): value is StandaloneRunProvenance { + if (!value || typeof value !== 'object') return false; + const record = value as Record; + const runtimes = record.runtimes as Record | undefined; + const platform = record.platform as Record | undefined; + const claude = runtimes?.claude as Record | undefined; + const codex = runtimes?.codex as Record | undefined; + return record.mode === 'standalone' + && typeof record.package_version === 'string' + && typeof record.vitest_version === 'string' + && typeof claude?.sdk_version === 'string' + && typeof claude?.claude_code_version === 'string' + && typeof codex?.package_version === 'string' + && typeof codex?.native_version === 'string' + && typeof platform?.os === 'string' + && typeof platform?.arch === 'string' + && typeof platform?.node === 'string'; +} + +function packagingDefect(message: string): Error { + return new Error(`pathgrade standalone: invalid provenance payload; this is a Pathgrade packaging defect (${message})`); +} diff --git a/src/standalone/ui/calm-renderer.ts b/src/standalone/ui/calm-renderer.ts new file mode 100644 index 0000000..7cfda16 --- /dev/null +++ b/src/standalone/ui/calm-renderer.ts @@ -0,0 +1,193 @@ +import type { Writable } from 'node:stream'; +import type { StandaloneUiEvent } from './events.js'; +import { displayWidth, formatDuration, truncateEnd, truncateMiddle } from './format.js'; +import { createStandaloneTheme, type StandaloneTheme } from './theme.js'; + +interface ActiveCase { + id: string; + label: string; + startedAt: number; +} + +export interface CalmRendererOptions { + stream?: Writable & { columns?: number; isTTY?: boolean; getColorDepth?: (env?: NodeJS.ProcessEnv) => number }; + env?: NodeJS.ProcessEnv; + now?: () => number; +} + +export class CalmRenderer { + private readonly stream; + private readonly env; + private readonly theme: StandaloneTheme; + private readonly now; + private readonly active = new Map(); + private dynamicLines = 0; + private frame = 0; + private timer?: ReturnType; + private started = false; + private finished = false; + private readonly traceMode: boolean; + private readonly quiet: boolean; + private readonly onSignal = () => this.restoreTerminal(); + + constructor(options: CalmRendererOptions = {}) { + this.stream = options.stream ?? process.stdout; + this.env = options.env ?? process.env; + this.theme = createStandaloneTheme(this.stream, this.env); + this.now = options.now ?? Date.now; + this.traceMode = this.env.PATHGRADE_VERBOSE === '1'; + this.quiet = this.env.PATHGRADE_QUIET === '1'; + if (this.theme.interactive && !this.traceMode && !this.quiet) { + process.once('SIGINT', this.onSignal); + process.once('SIGTERM', this.onSignal); + } + } + + handle(event: StandaloneUiEvent): void { + if (this.finished) return; + if (event.type === 'run_start') return this.start(event.files.length); + if (event.type === 'case_start') return this.startCase(event); + if (event.type === 'case_finish') return this.finishCase(event); + if (event.type === 'run_finish') return this.finishRun(event); + if (event.type === 'run_error') this.fail(); + } + + externalWrite(chunk: Buffer | string, target: Writable = process.stderr): void { + this.clearDynamic(); + target.write(chunk); + this.renderDynamic(); + } + + fail(): void { + this.clearDynamic(); + this.stopTimer(); + this.restoreTerminal(); + this.finished = true; + } + + dispose(): void { + this.clearDynamic(); + this.stopTimer(); + this.restoreTerminal(); + } + + private start(fileCount: number): void { + if (this.started) return; + this.started = true; + if (!this.traceMode && !this.quiet) { + this.write(`${this.theme.green(this.theme.bold('pathgrade'))} ${this.theme.bold('standalone')}\n`); + this.write(`${fileCount} eval ${fileCount === 1 ? 'file' : 'files'}\n\n`); + } else if (this.traceMode) { + this.write(`${this.theme.green(this.theme.bold('pathgrade'))} ${this.theme.bold('trace')}\n`); + } + } + + private startCase(event: Extract): void { + this.active.set(event.id, { + id: event.id, + label: `${event.file} › ${event.name}`, + startedAt: event.startedAt, + }); + if (this.theme.interactive && !this.traceMode && !this.quiet && !this.timer) { + this.stream.write('\x1b[?25l'); + this.timer = setInterval(() => this.renderDynamic(), 125); + this.timer.unref?.(); + } + this.renderDynamic(); + } + + private finishCase(event: Extract): void { + this.active.delete(event.id); + if (!this.traceMode && (!this.quiet || event.state === 'failed')) { + this.clearDynamic(); + this.writeCompleted(event); + } + this.renderDynamic(); + } + + private finishRun(event: Extract): void { + this.clearDynamic(); + this.stopTimer(); + this.restoreTerminal(); + const status = event.status === 'pass' + ? this.theme.green(this.theme.bold('PASS')) + : this.theme.red(this.theme.bold('FAIL')); + const counts = [ + `${event.fileCount} ${event.fileCount === 1 ? 'file' : 'files'}`, + event.passed ? `${event.passed} passed` : '', + event.failed ? `${event.failed} failed` : '', + event.skipped ? `${event.skipped} skipped` : '', + formatDuration(event.durationMs), + ].filter(Boolean).join(' · '); + this.write(`${this.traceMode ? '\n' : ''}${status} ${counts}\n`); + const threshold = event.threshold === undefined ? '' : ` · threshold ${event.threshold.toFixed(2)}`; + this.write(` overall score ${event.overallScore.toFixed(2)}${threshold}\n`); + this.write(` results ${this.theme.cyan(event.resultsPath)}\n`); + this.finished = true; + } + + private writeCompleted(event: Extract): void { + const icon = this.theme.interactive + ? (event.state === 'passed' ? '✓' : event.state === 'failed' ? '✗' : '–') + : (event.state === 'passed' ? 'PASS' : event.state === 'failed' ? 'FAIL' : 'SKIP'); + const coloredIcon = event.state === 'passed' + ? this.theme.green(icon) + : event.state === 'failed' ? this.theme.red(icon) : this.theme.yellow(icon); + const width = this.columns(); + this.write(`${coloredIcon} ${truncateEnd(`${event.file} › ${event.name}`, Math.max(8, width - displayWidth(icon) - 1))}\n`); + const status = event.state === 'passed' + ? this.theme.green('PASS') + : event.state === 'failed' ? this.theme.red('FAIL') : this.theme.yellow('SKIP'); + const score = event.score === undefined ? '' : ` score ${event.score.toFixed(2)}`; + this.write(` ${status}${score} ${this.theme.dim(formatDuration(event.durationMs))}\n\n`); + } + + private renderDynamic(): void { + if (!this.theme.interactive || this.traceMode || this.quiet || this.finished) return; + this.clearDynamic(); + const visible = [...this.active.values()].slice(0, 6); + const lines = visible.map(item => { + const frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']; + const marker = this.theme.cyan(frames[this.frame % frames.length]); + const elapsed = formatDuration(this.now() - item.startedAt); + const suffix = ` RUN ${elapsed}`; + const available = Math.max(8, this.columns() - displayWidth(suffix) - 2); + return `${marker} ${truncateMiddle(item.label, available)}${suffix}`; + }); + if (this.active.size > visible.length) lines.push(this.theme.dim(` and ${this.active.size - visible.length} more`)); + if (lines.length) this.write(`${lines.join('\n')}\n`); + this.dynamicLines = lines.length; + this.frame++; + } + + private clearDynamic(): void { + if (!this.dynamicLines) return; + this.stream.write(`\x1b[${this.dynamicLines}A`); + for (let index = 0; index < this.dynamicLines; index++) { + this.stream.write('\x1b[2K'); + if (index < this.dynamicLines - 1) this.stream.write('\x1b[1B'); + } + if (this.dynamicLines > 1) this.stream.write(`\x1b[${this.dynamicLines - 1}A`); + this.stream.write('\r'); + this.dynamicLines = 0; + } + + private stopTimer(): void { + if (this.timer) clearInterval(this.timer); + this.timer = undefined; + } + + private restoreTerminal(): void { + process.removeListener('SIGINT', this.onSignal); + process.removeListener('SIGTERM', this.onSignal); + if (this.theme.interactive && !this.traceMode && !this.quiet) this.stream.write('\x1b[?25h'); + } + + private columns(): number { + return Math.max(20, this.stream.columns ?? 80); + } + + private write(value: string): void { + this.stream.write(value); + } +} diff --git a/src/standalone/ui/events.ts b/src/standalone/ui/events.ts new file mode 100644 index 0000000..158f360 --- /dev/null +++ b/src/standalone/ui/events.ts @@ -0,0 +1,90 @@ +export const STANDALONE_UI_PROTOCOL_VERSION = 1 as const; +export const STANDALONE_UI_FD = 3 as const; + +export type StandaloneCaseState = 'passed' | 'failed' | 'skipped' | 'pending'; + +export type StandaloneUiEvent = + | { + v: 1; + type: 'run_start'; + files: Array<{ id: string; name: string }>; + startedAt: number; + } + | { + v: 1; + type: 'case_start'; + id: string; + file: string; + name: string; + startedAt: number; + } + | { + v: 1; + type: 'case_finish'; + id: string; + file: string; + name: string; + state: StandaloneCaseState; + durationMs: number; + score?: number; + } + | { + v: 1; + type: 'run_finish'; + status: 'pass' | 'fail'; + fileCount: number; + passed: number; + failed: number; + skipped: number; + durationMs: number; + overallScore: number; + threshold?: number; + resultsPath: string; + } + | { + v: 1; + type: 'run_error'; + }; + +export function isStandaloneUiEvent(value: unknown): value is StandaloneUiEvent { + if (!isRecord(value) || value.v !== STANDALONE_UI_PROTOCOL_VERSION || typeof value.type !== 'string') { + return false; + } + if (value.type === 'run_error') return true; + if (value.type === 'run_start') { + return Array.isArray(value.files) + && value.files.every(file => isRecord(file) && typeof file.id === 'string' && typeof file.name === 'string') + && typeof value.startedAt === 'number'; + } + if (value.type === 'case_start') { + return hasCaseIdentity(value) && typeof value.startedAt === 'number'; + } + if (value.type === 'case_finish') { + return hasCaseIdentity(value) + && isCaseState(value.state) + && typeof value.durationMs === 'number' + && (value.score === undefined || typeof value.score === 'number'); + } + if (value.type === 'run_finish') { + return (value.status === 'pass' || value.status === 'fail') + && ['fileCount', 'passed', 'failed', 'skipped', 'durationMs', 'overallScore'] + .every(key => typeof value[key] === 'number') + && (value.threshold === undefined || typeof value.threshold === 'number') + && typeof value.resultsPath === 'string'; + } + return false; +} + +function hasCaseIdentity(value: Record): boolean { + return typeof value.id === 'string' + && typeof value.file === 'string' + && typeof value.name === 'string'; +} + +function isCaseState(value: unknown): value is StandaloneCaseState { + return value === 'passed' || value === 'failed' || value === 'skipped' || value === 'pending'; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} diff --git a/src/standalone/ui/format.ts b/src/standalone/ui/format.ts new file mode 100644 index 0000000..8d117e0 --- /dev/null +++ b/src/standalone/ui/format.ts @@ -0,0 +1,75 @@ +// eslint-disable-next-line no-control-regex -- ANSI escape-sequence parser +const ANSI_PATTERN = /\u001B\[[0-?]*[ -/]*[@-~]/g; + +export function stripAnsi(value: string): string { + return value.replace(ANSI_PATTERN, ''); +} + +export function displayWidth(value: string): number { + let width = 0; + for (const char of stripAnsi(value)) width += isWide(char) ? 2 : 1; + return width; +} + +export function truncateEnd(value: string, width: number): string { + value = stripAnsi(value); + if (width <= 0) return ''; + if (displayWidth(value) <= width) return value; + if (width === 1) return '…'; + return takeWidth(value, width - 1) + '…'; +} + +export function truncateMiddle(value: string, width: number): string { + value = stripAnsi(value); + if (width <= 0) return ''; + if (displayWidth(value) <= width) return value; + if (width === 1) return '…'; + const leftWidth = Math.ceil((width - 1) / 2); + const rightWidth = Math.floor((width - 1) / 2); + return takeWidth(value, leftWidth) + '…' + takeWidthFromEnd(value, rightWidth); +} + +export function formatDuration(durationMs: number): string { + const seconds = Math.max(0, Math.round(durationMs / 1000)); + if (seconds < 60) return `${seconds}s`; + return `${Math.floor(seconds / 60)}m ${String(seconds % 60).padStart(2, '0')}s`; +} + +function takeWidth(value: string, maxWidth: number): string { + let result = ''; + let width = 0; + for (const char of value) { + const next = isWide(char) ? 2 : 1; + if (width + next > maxWidth) break; + result += char; + width += next; + } + return result; +} + +function takeWidthFromEnd(value: string, maxWidth: number): string { + let result = ''; + let width = 0; + for (const char of Array.from(value).reverse()) { + const next = isWide(char) ? 2 : 1; + if (width + next > maxWidth) break; + result = char + result; + width += next; + } + return result; +} + +function isWide(char: string): boolean { + const code = char.codePointAt(0) ?? 0; + return /\p{Extended_Pictographic}/u.test(char) + || (code >= 0x1100 && ( + code <= 0x115f || code === 0x2329 || code === 0x232a + || (code >= 0x2e80 && code <= 0xa4cf) + || (code >= 0xac00 && code <= 0xd7a3) + || (code >= 0xf900 && code <= 0xfaff) + || (code >= 0xfe10 && code <= 0xfe6f) + || (code >= 0xff00 && code <= 0xff60) + || (code >= 0x1f300 && code <= 0x1faff) + || (code >= 0x20000 && code <= 0x3fffd) + )); +} diff --git a/src/standalone/ui/output-controller.ts b/src/standalone/ui/output-controller.ts new file mode 100644 index 0000000..00bfbeb --- /dev/null +++ b/src/standalone/ui/output-controller.ts @@ -0,0 +1,109 @@ +import { CalmRenderer } from './calm-renderer.js'; +import { StandaloneUiDecoder } from './protocol.js'; +import type { Writable } from 'node:stream'; + +const RAW_OUTPUT_LIMIT = 2 * 1024 * 1024; + +interface OutputStreams { + stdout: Writable & { + columns?: number; + isTTY?: boolean; + getColorDepth?: (env?: NodeJS.ProcessEnv) => number; + }; + stderr: Writable; +} + +export class StandaloneOutputController { + private readonly renderer: CalmRenderer; + private readonly decoder: StandaloneUiDecoder; + private rawStdout: Buffer[] = []; + private rawBytes = 0; + private invalid = false; + private warned = false; + private protocolSeen = false; + private finalReceived = false; + private runnerFailed = false; + private readonly passthrough: boolean; + + constructor( + private readonly env: NodeJS.ProcessEnv = process.env, + private readonly streams: OutputStreams = { stdout: process.stdout, stderr: process.stderr }, + ) { + this.passthrough = env.PATHGRADE_REPORTER_MODE === 'json'; + this.renderer = new CalmRenderer({ env, stream: streams.stdout }); + this.decoder = new StandaloneUiDecoder( + event => { + if (event.type === 'run_finish') { + this.finalReceived = true; + this.runnerFailed = event.failed > 0; + if (this.runnerFailed) this.flushRaw(); + } + this.renderer.handle(event); + }, + () => this.protocolFailure(), + ); + } + + stdout(chunk: Buffer | string): void { + if (this.passthrough) { + this.streams.stdout.write(chunk); + return; + } + if (this.invalid) { + this.streams.stdout.write(chunk); + return; + } + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + this.rawStdout.push(buffer); + this.rawBytes += buffer.length; + while (this.rawBytes > RAW_OUTPUT_LIMIT && this.rawStdout.length > 1) { + this.rawBytes -= this.rawStdout.shift()!.length; + } + } + + stderr(chunk: Buffer | string): void { + this.renderer.externalWrite(chunk, this.streams.stderr); + } + + protocol(chunk: Buffer | string): void { + if (this.passthrough) return; + this.protocolSeen = true; + this.decoder.push(chunk); + } + + finish(exitCode: number): void { + if (this.passthrough) { + this.renderer.dispose(); + return; + } + this.decoder.end(); + if (!this.protocolSeen) { + this.protocolFailure(); + } + this.renderer.dispose(); + if (this.invalid || (exitCode !== 0 && (!this.finalReceived || this.runnerFailed))) { + this.flushRaw(); + } + } + + dispose(): void { + this.renderer.dispose(); + } + + private protocolFailure(): void { + if (this.invalid) return; + this.invalid = true; + this.renderer.fail(); + if (!this.warned) { + this.warned = true; + this.streams.stderr.write('pathgrade standalone: rich output unavailable; using raw Vitest output\n'); + } + this.flushRaw(); + } + + private flushRaw(): void { + for (const chunk of this.rawStdout) this.streams.stdout.write(chunk); + this.rawStdout = []; + this.rawBytes = 0; + } +} diff --git a/src/standalone/ui/protocol.ts b/src/standalone/ui/protocol.ts new file mode 100644 index 0000000..e139ddc --- /dev/null +++ b/src/standalone/ui/protocol.ts @@ -0,0 +1,68 @@ +import fs from 'node:fs'; +import { + isStandaloneUiEvent, + STANDALONE_UI_FD, + type StandaloneUiEvent, +} from './events.js'; + +export interface StandaloneUiSink { + emit(event: StandaloneUiEvent): void; +} + +export function createStandaloneUiSink( + enabled: boolean, + write: (line: string) => void = defaultWrite, +): StandaloneUiSink { + let available = enabled; + return { + emit(event) { + if (!available) return; + try { + write(`${JSON.stringify(event)}\n`); + } catch { + available = false; + } + }, + }; +} + +function defaultWrite(line: string): void { + fs.writeSync(STANDALONE_UI_FD, line); +} + +export class StandaloneUiDecoder { + private pending = ''; + private failed = false; + + constructor( + private readonly onEvent: (event: StandaloneUiEvent) => void, + private readonly onInvalid: () => void, + ) {} + + push(chunk: Buffer | string): void { + if (this.failed) return; + this.pending += chunk.toString(); + const lines = this.pending.split('\n'); + this.pending = lines.pop() ?? ''; + for (const line of lines) { + if (!line) continue; + this.decodeLine(line); + } + } + + end(): void { + if (!this.failed && this.pending.trim()) this.decodeLine(this.pending); + this.pending = ''; + } + + private decodeLine(line: string): void { + try { + const parsed: unknown = JSON.parse(line); + if (!isStandaloneUiEvent(parsed)) throw new Error('unsupported event'); + this.onEvent(parsed); + } catch { + this.failed = true; + this.onInvalid(); + } + } +} diff --git a/src/standalone/ui/theme.ts b/src/standalone/ui/theme.ts new file mode 100644 index 0000000..3b745a8 --- /dev/null +++ b/src/standalone/ui/theme.ts @@ -0,0 +1,51 @@ +import type { Writable } from 'node:stream'; + +export interface StandaloneTheme { + color: boolean; + interactive: boolean; + bold(value: string): string; + dim(value: string): string; + green(value: string): string; + red(value: string): string; + cyan(value: string): string; + yellow(value: string): string; + magenta(value: string): string; +} + +type ColorStream = Writable & { + isTTY?: boolean; + getColorDepth?: (env?: NodeJS.ProcessEnv) => number; +}; + +export function createStandaloneTheme( + stream: ColorStream = process.stdout, + env: NodeJS.ProcessEnv = process.env, +): StandaloneTheme { + const color = colorEnabled(stream, env); + const interactive = stream.isTTY === true + && env.TERM !== 'dumb' + && !env.CI + && env.PATHGRADE_QUIET !== '1'; + const apply = (ansi: number) => (value: string) => color + ? `\x1b[${ansi}m${value}\x1b[0m` + : value; + return { + color, + interactive, + bold: apply(1), + dim: apply(2), + red: apply(31), + green: apply(32), + yellow: apply(33), + cyan: apply(36), + magenta: apply(35), + }; +} + +function colorEnabled(stream: ColorStream, env: NodeJS.ProcessEnv): boolean { + if (env.FORCE_COLOR === '0') return false; + if (env.FORCE_COLOR !== undefined) return true; + if (env.NO_COLOR !== undefined || env.NODE_DISABLE_COLORS !== undefined) return false; + if (stream.getColorDepth) return stream.getColorDepth(env) > 1; + return stream.isTTY === true && env.TERM !== 'dumb'; +} diff --git a/src/standalone/validation.ts b/src/standalone/validation.ts new file mode 100644 index 0000000..2bf842f --- /dev/null +++ b/src/standalone/validation.ts @@ -0,0 +1,72 @@ +export class StandaloneConfigurationError extends Error { + constructor(message: string) { + super(message); + this.name = 'StandaloneConfigurationError'; + } +} + +const STANDALONE_VALUE_FLAGS = new Set([ + '-t', + '--testNamePattern', +]); + +function unsupported(value: string): StandaloneConfigurationError { + return new StandaloneConfigurationError( + `pathgrade standalone: ${value} is unsupported; ` + + 'the standalone runner allows only eval-file filters and -t/--testNamePattern; ' + + 'use project-local @wix/pathgrade for project runner compatibility', + ); +} + +function isSupportedEvalFilter(value: string): boolean { + return value.endsWith('.eval.ts'); +} + +export function validateStandaloneInvocation(input: { + adapterName: string; + runnerArgs: string[]; +}): void { + if (input.adapterName !== 'vitest') { + throw unsupported(input.adapterName); + } + + for (let index = 0; index < input.runnerArgs.length; index += 1) { + const arg = input.runnerArgs[index]; + const [flag, inlineValue] = splitInlineValue(arg); + + if (STANDALONE_VALUE_FLAGS.has(flag)) { + const value = inlineValue ?? input.runnerArgs[++index]; + if (!value || value.startsWith('-')) { + throw unsupported(arg); + } + continue; + } + + if (arg.startsWith('-') || !isSupportedEvalFilter(arg)) { + throw unsupported(arg); + } + } +} + +function splitInlineValue(arg: string): [string, string | undefined] { + const equalsIndex = arg.indexOf('='); + if (equalsIndex === -1) return [arg, undefined]; + return [arg.slice(0, equalsIndex), arg.slice(equalsIndex + 1)]; +} + +export function assertStandalonePlatform(input: { + nodeMajor: number; + platform: NodeJS.Platform; + arch: string; +}): void { + const supported = (input.nodeMajor === 22 || input.nodeMajor === 24) + && (input.platform === 'darwin' || input.platform === 'linux') + && (input.arch === 'x64' || input.arch === 'arm64'); + if (supported) return; + + throw new StandaloneConfigurationError( + `pathgrade standalone: unsupported runtime Node ${input.nodeMajor} ` + + `on ${input.platform}/${input.arch}; supported runtimes are Node 22 or 24 ` + + 'on macOS, Linux, or WSL with x64 or arm64', + ); +} diff --git a/src/standalone/vitest-config.ts b/src/standalone/vitest-config.ts new file mode 100644 index 0000000..9a2ce1c --- /dev/null +++ b/src/standalone/vitest-config.ts @@ -0,0 +1,106 @@ +import path from 'node:path'; +import fs from 'node:fs'; +import { createRequire } from 'node:module'; +import type { ViteUserConfig } from 'vitest/config'; +import { pathgrade } from '../adapters/vitest/index.js'; +import { + decodeStandaloneVitestPayload, + resolveStandaloneModuleAliases, + selectStandaloneEsmExportTarget, + STANDALONE_VITEST_PAYLOAD_ENV, + type StandaloneVitestPayload, +} from './module-aliases.js'; + +export interface StandaloneVitestRuntimePaths { + packageRoot: string; + vitestEntry: string; +} + +export type StandaloneVitestConfig = ViteUserConfig & { envFile: false }; + +export function buildStandaloneVitestConfig( + payload: StandaloneVitestPayload, + runtimePaths: StandaloneVitestRuntimePaths, +): StandaloneVitestConfig { + const generated = { + vite: path.join(payload.cacheDir, 'vite'), + coverage: path.join(payload.cacheDir, 'coverage'), + attachments: path.join(payload.cacheDir, 'attachments'), + json: path.join(payload.cacheDir, 'reports', 'pathgrade.json'), + blob: path.join(payload.cacheDir, 'reports', 'vitest.blob'), + }; + for (const outputPath of Object.values(generated)) { + assertOwnedOutputPath(payload.cacheDir, outputPath); + } + + return { + root: payload.root, + envFile: false, + envDir: payload.cacheDir, + cacheDir: generated.vite, + resolve: { + alias: resolveStandaloneModuleAliases( + runtimePaths.packageRoot, + runtimePaths.vitestEntry, + ), + }, + plugins: [ + pathgrade({ + include: payload.include, + exclude: payload.exclude, + diagnostics: payload.diagnostics, + reporter: payload.reporter, + ci: { threshold: payload.threshold }, + }), + ], + test: { + include: payload.include, + exclude: payload.exclude, + coverage: { + reportsDirectory: generated.coverage, + }, + attachmentsDir: generated.attachments, + outputFile: { + json: generated.json, + blob: generated.blob, + }, + }, + } as StandaloneVitestConfig; +} + +function assertOwnedOutputPath(cacheDir: string, outputPath: string): void { + const relative = path.relative(cacheDir, outputPath); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) { + throw new Error( + `pathgrade standalone: generated output escapes cache directory: ${outputPath}`, + ); + } +} + +function buildDefaultConfig(): ViteUserConfig { + const encoded = process.env[STANDALONE_VITEST_PAYLOAD_ENV]; + if (!encoded) return {}; + + const packageRoot = path.resolve(import.meta.dirname, '..', '..'); + const require = createRequire(import.meta.url); + const vitestPackageJsonPath = require.resolve('vitest/package.json'); + const vitestPackageJson = JSON.parse( + fs.readFileSync(vitestPackageJsonPath, 'utf8'), + ) as { exports?: Record }; + const vitestTarget = selectStandaloneEsmExportTarget( + vitestPackageJson.exports?.['.'], + ); + if (!vitestTarget) { + throw new Error('pathgrade standalone: bundled Vitest ESM entry is unavailable'); + } + const vitestEntry = path.resolve( + path.dirname(vitestPackageJsonPath), + vitestTarget, + ); + return buildStandaloneVitestConfig( + decodeStandaloneVitestPayload(encoded), + { packageRoot, vitestEntry }, + ); +} + +export default buildDefaultConfig(); diff --git a/src/standalone/vitest-invocation.ts b/src/standalone/vitest-invocation.ts new file mode 100644 index 0000000..d7203c7 --- /dev/null +++ b/src/standalone/vitest-invocation.ts @@ -0,0 +1,164 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawn as spawnChild } from 'node:child_process'; +import type { ResolvedPathgradeConfig } from '../config/pathgrade.js'; +import type { RunnerInvocationAdapter } from '../runners/invocation.js'; +import { + encodeStandaloneVitestPayload, + STANDALONE_VITEST_PAYLOAD_ENV, + type StandaloneVitestPayload, +} from './module-aliases.js'; +import { classifyStandaloneVitestFailure } from './diagnostics.js'; +import { buildStandaloneRunProvenance, encodeStandaloneRunProvenance, STANDALONE_PROVENANCE_ENV } from './provenance.js'; +import { resolveBundledVitestCli, type BundledVitestRuntime } from './vitest-runtime.js'; +import { StandaloneOutputController } from './ui/output-controller.js'; +import { createStandaloneTheme } from './ui/theme.js'; + +export { resolveBundledVitestCli, type BundledVitestRuntime } from './vitest-runtime.js'; + +const STDERR_LIMIT_BYTES = 64 * 1024; + +export interface SpawnStandaloneVitestRequest { + command: string; + argv: string[]; + env: NodeJS.ProcessEnv; + cwd: string; +} + +export interface SpawnStandaloneVitestResult { + exitCode: number; + stderr?: string; +} + +export type SpawnStandaloneVitest = ( + request: SpawnStandaloneVitestRequest, +) => Promise | number | SpawnStandaloneVitestResult; + +export function createStandaloneVitestInvocationAdapter(input: { + config: ResolvedPathgradeConfig; + spawn?: SpawnStandaloneVitest; + resolveRuntime?: () => BundledVitestRuntime; + internalConfigPath?: string; + createTempDir?: () => Promise | string; + removeTempDir?: (directory: string) => Promise | void; +}): RunnerInvocationAdapter { + const spawn = input.spawn ?? defaultSpawnStandaloneVitest; + const resolveRuntime = input.resolveRuntime ?? resolveBundledVitestCli; + const internalConfigPath = input.internalConfigPath + ?? path.resolve(import.meta.dirname, 'vitest-config.js'); + const createTempDir = input.createTempDir ?? defaultCreateTempDir; + const removeTempDir = input.removeTempDir ?? defaultRemoveTempDir; + + return { + name: 'vitest', + async run(runInput): Promise { + const cacheDir = path.resolve(await createTempDir()); + try { + const runtime = resolveRuntime(); + const payload = buildPayload(input.config, runInput, cacheDir); + const outerTheme = createStandaloneTheme(process.stdout, runInput.env); + const provenance = await buildStandaloneRunProvenance(); + const result = await spawn({ + command: process.execPath, + argv: [ + runtime.cliPath, + 'run', + ...(runInput.selectedFiles ?? []), + ...runInput.runnerArgs, + '--config', + internalConfigPath, + ], + cwd: runInput.cwd, + env: { + ...runInput.env, + [STANDALONE_VITEST_PAYLOAD_ENV]: + encodeStandaloneVitestPayload(payload), + [STANDALONE_PROVENANCE_ENV]: encodeStandaloneRunProvenance(provenance), + PATHGRADE_REPORTER_MODE: payload.reporter ?? 'cli', + PATHGRADE_UI_COLOR: outerTheme.color ? '1' : '0', + }, + }); + const normalized = typeof result === 'number' + ? { exitCode: result, stderr: undefined } + : result; + if (normalized.exitCode !== 0 && normalized.stderr) { + const classified = classifyStandaloneVitestFailure(normalized.stderr); + if (classified) process.stderr.write(`${classified.message}\n`); + } + return normalized.exitCode; + } finally { + await removeTempDir(cacheDir); + } + }, + }; +} + +function buildPayload( + config: ResolvedPathgradeConfig, + runInput: Parameters[0], + cacheDir: string, +): StandaloneVitestPayload { + return { + root: path.resolve(runInput.cwd), + include: [...config.evals.include], + exclude: [...config.evals.exclude], + diagnostics: + config.diagnostics || runInput.env.PATHGRADE_DIAGNOSTICS === '1', + ...(config.reporter === undefined ? {} : { reporter: config.reporter }), + ...(config.ci.threshold === undefined ? {} : { threshold: config.ci.threshold }), + cacheDir, + }; +} + +async function defaultCreateTempDir(): Promise { + return fs.promises.mkdtemp(path.join(os.tmpdir(), 'pathgrade-standalone-')); +} + +async function defaultRemoveTempDir(directory: string): Promise { + await fs.promises.rm(directory, { recursive: true, force: true }); +} + +async function defaultSpawnStandaloneVitest( + request: SpawnStandaloneVitestRequest, +): Promise { + return await new Promise((resolve, reject) => { + const child = spawnChild(request.command, request.argv, { + stdio: ['inherit', 'pipe', 'pipe', 'pipe'], + cwd: request.cwd, + env: request.env, + }); + let retainedStderr: Buffer = Buffer.alloc(0); + const output = new StandaloneOutputController(request.env); + + child.stdout?.on('data', (chunk: Buffer | string) => { + output.stdout(chunk); + }); + child.stderr?.on('data', (chunk: Buffer | string) => { + output.stderr(chunk); + retainedStderr = retainTail(retainedStderr, chunk); + }); + child.stdio[3]?.on('data', (chunk: Buffer | string) => output.protocol(chunk)); + child.once('error', error => { + output.dispose(); + reject(error); + }); + child.once('close', (code) => { + output.finish(code ?? 1); + resolve({ + exitCode: code ?? 1, + stderr: retainedStderr.toString('utf8'), + }); + }); + }); +} + +function retainTail(previous: Buffer, chunk: Buffer | string): Buffer { + const next = Buffer.concat([ + previous, + Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk), + ]); + return next.length <= STDERR_LIMIT_BYTES + ? next + : next.subarray(next.length - STDERR_LIMIT_BYTES); +} diff --git a/src/standalone/vitest-runtime.ts b/src/standalone/vitest-runtime.ts new file mode 100644 index 0000000..11b7bc4 --- /dev/null +++ b/src/standalone/vitest-runtime.ts @@ -0,0 +1,44 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { selectStandaloneEsmExportTarget } from './module-aliases.js'; + +const BUNDLED_VITEST_VERSION = '4.1.7'; + +export interface BundledVitestRuntime { + cliPath: string; + version: string; + entryPath: string; +} + +export function resolveBundledVitestCli(): BundledVitestRuntime { + try { + const require = createRequire(import.meta.url); + const packageJsonPath = require.resolve('vitest/package.json'); + const packageJson = JSON.parse( + fs.readFileSync(packageJsonPath, 'utf8'), + ) as { + version?: string; + bin?: string | Record; + exports?: Record; + }; + const bin = typeof packageJson.bin === 'string' + ? packageJson.bin + : packageJson.bin?.vitest; + const entry = selectStandaloneEsmExportTarget(packageJson.exports?.['.']); + if (packageJson.version !== BUNDLED_VITEST_VERSION || !bin || !entry) { + throw new Error('unexpected bundled Vitest metadata'); + } + const packageRoot = path.dirname(packageJsonPath); + return { + cliPath: path.resolve(packageRoot, bin), + version: packageJson.version, + entryPath: path.resolve(packageRoot, entry), + }; + } catch { + throw new Error( + `pathgrade standalone: bundled Vitest ${BUNDLED_VITEST_VERSION} ` + + 'could not be resolved; this is a Pathgrade packaging defect', + ); + } +} diff --git a/src/types.ts b/src/types.ts index 5857a18..1004a02 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,6 +1,8 @@ import type { DiagnosticsReport } from './sdk/diagnostics.js'; import type { RuntimePolicyDescriptor } from './sdk/runtime-policy.js'; import type { LLMPort } from './utils/llm-types.js'; +import type { AgentInvocationProvenance } from './sdk/types.js'; +import type { StandaloneRunProvenance } from './standalone/provenance.js'; export interface CommandResult { stdout: string; @@ -181,6 +183,7 @@ export interface TrialResult { session_log: LogEntry[]; skills_used?: string[]; diagnostics?: DiagnosticsReport; + agent_provenance?: AgentInvocationProvenance; conversation?: { turns: ConversationTurn[]; total_turns: number; @@ -259,6 +262,7 @@ export interface PathgradeReport { * this field. */ selection?: PathgradeSelectionReport; + provenance?: StandaloneRunProvenance; } export interface TrialPaths { diff --git a/tests/agent-provenance-cleanup.test.ts b/tests/agent-provenance-cleanup.test.ts new file mode 100644 index 0000000..b90f41a --- /dev/null +++ b/tests/agent-provenance-cleanup.test.ts @@ -0,0 +1,41 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +const prepareWorkspaceMock = vi.fn(); +const verifyBundledClaudeRuntimeMock = vi.fn(); + +vi.mock('../src/providers/workspace', () => ({ + prepareWorkspace: (...args: unknown[]) => prepareWorkspaceMock(...args), +})); + +vi.mock('../src/agents/claude-runtime', () => ({ + verifyBundledClaudeRuntime: (...args: unknown[]) => verifyBundledClaudeRuntimeMock(...args), +})); + +describe('standalone agent provenance initialization', () => { + const originalStandalone = process.env.PATHGRADE_STANDALONE; + + afterEach(() => { + vi.clearAllMocks(); + if (originalStandalone === undefined) delete process.env.PATHGRADE_STANDALONE; + else process.env.PATHGRADE_STANDALONE = originalStandalone; + }); + + it('disposes the prepared workspace when bundled runtime verification fails', async () => { + process.env.PATHGRADE_STANDALONE = '1'; + const workspace = { + path: '/tmp/pathgrade-provenance-cleanup', + env: { ANTHROPIC_API_KEY: 'test-key' }, + exec: vi.fn(), + dispose: vi.fn().mockResolvedValue(undefined), + setupCommands: [], + mcpConfigPath: undefined, + }; + prepareWorkspaceMock.mockResolvedValue(workspace); + verifyBundledClaudeRuntimeMock.mockRejectedValue(new Error('bundled runtime probe failed')); + + const { createAgent } = await import('../src/sdk/agent.js'); + + await expect(createAgent({ agent: 'claude' })).rejects.toThrow('bundled runtime probe failed'); + expect(workspace.dispose).toHaveBeenCalledOnce(); + }); +}); diff --git a/tests/ask-batch-parity.test.ts b/tests/ask-batch-parity.test.ts index 6c4566e..b75905f 100644 --- a/tests/ask-batch-parity.test.ts +++ b/tests/ask-batch-parity.test.ts @@ -7,6 +7,7 @@ function canonicalCodexRequestUserInput(): ToolRequestUserInputParams { threadId: 'thread-1', turnId: 'turn-1', itemId: 'tool-use-abc', + autoResolutionMs: null, questions: [ { id: 'q-0', diff --git a/tests/claude-sdk-options.test.ts b/tests/claude-sdk-options.test.ts index 1351e92..adaac07 100644 --- a/tests/claude-sdk-options.test.ts +++ b/tests/claude-sdk-options.test.ts @@ -213,6 +213,14 @@ describe('resolveClaudeCodeExecutable — override precedence (TB6)', () => { expect(exe).toBe('/from/env/claude'); }); + it('rejects PATHGRADE_CLAUDE_CODE_EXECUTABLE in standalone mode', () => { + expect(() => resolveClaudeCodeExecutable({ + agentOptionsExecutable: undefined, + envExecutable: '/from/env/claude', + standalone: true, + })).toThrow(/PATHGRADE_CLAUDE_CODE_EXECUTABLE.*standalone/); + }); + it('AgentOptions.claudeCodeExecutable wins over the env variable', () => { // Run-level precedence: explicit AgentOptions beats process env. const exe = resolveClaudeCodeExecutable({ diff --git a/tests/cli-surface-local-first.test.ts b/tests/cli-surface-local-first.test.ts index c3d2921..db20b09 100644 --- a/tests/cli-surface-local-first.test.ts +++ b/tests/cli-surface-local-first.test.ts @@ -1,4 +1,4 @@ -import { execFileSync } from 'child_process'; +import { execFileSync, spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import { describe, expect, it } from 'vitest'; @@ -32,6 +32,32 @@ describe('local-first CLI surface', () => { expect(output).not.toContain('--provider='); }); + it('provides scoped standalone help and version', () => { + const help = execFileSync(process.execPath, [binEntry, 'standalone', '--help'], { + cwd: packageRoot, + encoding: 'utf-8', + env: { ...process.env, FORCE_COLOR: '0' }, + }); + const version = execFileSync(process.execPath, [binEntry, 'standalone', '--version'], { + cwd: packageRoot, + encoding: 'utf-8', + env: { ...process.env, FORCE_COLOR: '0' }, + }); + expect(help).toContain('pathgrade standalone - Run evals with the embedded Vitest runtime'); + expect(help).toContain('--testNamePattern'); + expect(version.trim()).toMatch(/^\d+\.\d+\.\d+$/); + }); + + it('rejects contradictory standalone output modes', () => { + const result = spawnSync(process.execPath, [binEntry, 'standalone', 'run', '--quiet', '--verbose'], { + cwd: packageRoot, + encoding: 'utf-8', + env: { ...process.env, FORCE_COLOR: '0' }, + }); + expect(result.status).toBe(1); + expect(result.stderr).toContain('--quiet and --verbose are mutually exclusive'); + }); + it('documents local-first usage in the README', () => { const readme = readRepoFile('README.md'); diff --git a/tests/codex-app-server-agent.test.ts b/tests/codex-app-server-agent.test.ts index be9b04f..b42a1f9 100644 --- a/tests/codex-app-server-agent.test.ts +++ b/tests/codex-app-server-agent.test.ts @@ -11,7 +11,10 @@ import type { AppServerTransport, TransportCloseInfo, } from '../src/agents/codex-app-server/transport.js'; -import { CodexAppServerAgent } from '../src/agents/codex-app-server/agent.js'; +import { + CodexAppServerAgent, + loginCodexAppServerWithApiKey, +} from '../src/agents/codex-app-server/agent.js'; import { createAskBus } from '../src/sdk/ask-bus/bus.js'; import { isMcpToolCall } from '../src/sdk/mcp-evidence.js'; import type { AgentSessionOptions, TrialRuntime } from '../src/types.js'; @@ -31,6 +34,7 @@ interface ClientResponseCapture { interface ServerSim { transport: AppServerTransport; + clientMethods: string[]; sendServerRequest: (method: string, params: unknown) => { id: number }; sendNotification: (method: string, params: unknown) => void; awaitRequest: (method: string) => Promise; @@ -58,6 +62,7 @@ function createServerSim(): ServerSim { resolve: (v: unknown) => void; } const awaiters: PendingMatcher[] = []; + const clientMethods: string[] = []; const bufferedRequests = new Map(); const bufferedNotifications = new Map(); const bufferedResponses = new Map(); @@ -75,6 +80,7 @@ function createServerSim(): ServerSim { } catch { return; } + if (parsed.method) clientMethods.push(parsed.method); if (parsed.method === 'initialize' && parsed.id !== undefined) { // Capture the initialize params so individual tests can assert on // the handshake capability shape. @@ -158,6 +164,7 @@ function createServerSim(): ServerSim { return { transport: driverTransport, + clientMethods, sendServerRequest(method, params) { const id = nextServerId++; serverToClient.write( @@ -264,6 +271,150 @@ async function runSingleTrivialTurn( } describe('CodexAppServerAgent — handshake', () => { + it('in standalone mode logs in after initialized and before thread/start', async () => { + vi.stubEnv('PATHGRADE_STANDALONE', '1'); + try { + await withAgent(async ({ sim, agent, options }) => { + const session = await agent.createSession( + { handle: '/tmp/ws', workspacePath: '/tmp/ws', env: { OPENAI_API_KEY: 'sk-secret' } }, + async () => ({ stdout: '', stderr: '', exitCode: 0 }), + options, + ); + const turnPromise = session.start({ message: 'hi' }); + + await sim.awaitRequest('initialize'); + await sim.awaitNotification('initialized'); + const login = await sim.awaitRequest('account/login/start'); + expect(login.params).toEqual({ type: 'apiKey', apiKey: 'sk-secret' }); + sim.sendResult(login.id, { type: 'apiKey' }); + await runSingleTrivialTurn(sim); + await turnPromise; + expect(sim.clientMethods).toEqual([ + 'initialize', + 'initialized', + 'account/login/start', + 'thread/start', + 'turn/start', + ]); + }); + } finally { + vi.unstubAllEnvs(); + } + }); + + it('does not expose the API key when isolated login fails', async () => { + const transport = { + sendRequest: async () => { throw new Error('upstream rejected sk-do-not-leak'); }, + } as unknown as AppServerTransport; + const rejected = await loginCodexAppServerWithApiKey(transport, 'sk-do-not-leak') + .then(() => undefined, (error: unknown) => error as Error); + expect(rejected?.message).toContain('upstream rejected [redacted]'); + expect(rejected?.message).not.toContain('sk-do-not-leak'); + await expect(loginCodexAppServerWithApiKey(transport, '')) + .rejects.toThrow(/OPENAI_API_KEY/); + }); + + it('fails missing standalone API key before requesting thread/start', async () => { + vi.stubEnv('PATHGRADE_STANDALONE', '1'); + let threadStarted = false; + const transport = { + sendRequest: async (method: string) => { + if (method === 'initialize') return {}; + if (method === 'thread/start') threadStarted = true; + return {}; + }, + sendNotification: () => undefined, + sendResponse: () => undefined, + sendErrorResponse: () => undefined, + onServerRequest: () => () => undefined, + onNotification: () => () => undefined, + onClose: () => () => undefined, + close: async () => undefined, + } as AppServerTransport; + try { + const agent = new CodexAppServerAgent({ + createTransport: async () => createAppServerSessionHandle({ transport, child: null }), + }); + const session = await agent.createSession( + { handle: '/tmp/ws', workspacePath: '/tmp/ws', env: {} }, + async () => ({ stdout: '', stderr: '', exitCode: 0 }), + { askBus: createAskBus({ askUserTimeoutMs: 1_000 }) }, + ); + await expect(session.start({ message: 'hi' })).rejects.toThrow(/OPENAI_API_KEY/); + expect(threadStarted).toBe(false); + } finally { + vi.unstubAllEnvs(); + } + }); + + it('retries the full standalone handshake after login failure and closes the failed transport', async () => { + vi.stubEnv('PATHGRADE_STANDALONE', '1'); + const first = createServerSim(); + const second = createServerSim(); + const firstClose = vi.spyOn(first.transport, 'close'); + let factoryCalls = 0; + let session: Awaited> | undefined; + let retryTurn: ReturnType['reply']> | undefined; + + try { + const agent = new CodexAppServerAgent({ + createTransport: async () => { + factoryCalls += 1; + const sim = factoryCalls === 1 ? first : second; + return createAppServerSessionHandle({ transport: sim.transport, child: null }); + }, + }); + session = await agent.createSession( + { + handle: '/tmp/ws', + workspacePath: '/tmp/ws', + env: { OPENAI_API_KEY: 'fixture-api-key' }, + }, + async () => ({ stdout: '', stderr: '', exitCode: 0 }), + { askBus: createAskBus({ askUserTimeoutMs: 1_000 }) }, + ); + + const failedTurn = session.start({ message: 'first' }); + const firstLogin = await first.awaitRequest('account/login/start'); + first.sendError(firstLogin.id, 401, 'login rejected'); + await expect(failedTurn).rejects.toThrow('login rejected'); + expect(first.clientMethods).not.toContain('thread/start'); + + retryTurn = session.reply({ message: 'second' }); + const retryOutcome = await Promise.race([ + second.awaitRequest('initialize').then(() => ({ kind: 'initialize' as const })), + first.awaitRequest('thread/start').then((request) => ({ + kind: 'bypassed' as const, + request, + })), + ]); + if (retryOutcome.kind === 'bypassed') { + first.sendResult(retryOutcome.request.id, { thread: { id: 'bypassed-thread' } }); + const bypassedTurn = await first.awaitRequest('turn/start'); + first.sendResult(bypassedTurn.id, { turnId: 'bypassed-turn' }); + first.sendNotification('turn/completed', {}); + await retryTurn; + } + expect(retryOutcome.kind).toBe('initialize'); + + await second.awaitNotification('initialized'); + const secondLogin = await second.awaitRequest('account/login/start'); + expect(second.clientMethods).not.toContain('thread/start'); + second.sendResult(secondLogin.id, { type: 'apiKey' }); + await runSingleTrivialTurn(second); + if (retryTurn) await retryTurn; + + expect(factoryCalls).toBe(2); + expect(firstClose).toHaveBeenCalledTimes(1); + } finally { + await session?.dispose(); + await retryTurn?.catch(() => undefined); + await first.transport.close(); + await second.transport.close(); + vi.unstubAllEnvs(); + } + }); + it('passes the isolated runtime env to the app-server transport factory', async () => { const sim = createServerSim(); let capturedCtx: unknown; @@ -326,6 +477,7 @@ describe('CodexAppServerAgent — handshake', () => { sim.sendResult(st.id, { turnId: 'u' }); sim.sendNotification('turn/completed', {}); await turnPromise; + expect(sim.clientMethods).not.toContain('account/login/start'); }); }); @@ -359,8 +511,8 @@ describe('CodexAppServerAgent — handshake', () => { expect(params.approvalPolicy).toBe('never'); expect(params.sandbox).toBe('workspace-write'); expect(params.ephemeral).toBe(true); - expect(params.experimentalRawEvents).toBe(false); - expect(params.persistExtendedHistory).toBe(false); + expect(params.experimentalRawEvents).toBeUndefined(); + expect(params.persistExtendedHistory).toBeUndefined(); expect(params.baseInstructions).toBeUndefined(); expect(params.developerInstructions).toBeUndefined(); sim.sendResult(nt.id, { thread: { id: 'thread-1' } }); diff --git a/tests/codex-app-server-fixtures.test.ts b/tests/codex-app-server-fixtures.test.ts index 07e6e56..7fab76d 100644 --- a/tests/codex-app-server-fixtures.test.ts +++ b/tests/codex-app-server-fixtures.test.ts @@ -30,8 +30,6 @@ suiteFn('codex app-server protocol fixture suite', () => { approvalPolicy: 'on-request', sandbox: 'workspace-write', ephemeral: true, - experimentalRawEvents: false, - persistExtendedHistory: false, }); expect(typeof response).toBe('object'); @@ -76,8 +74,6 @@ suiteFn('codex app-server protocol fixture suite', () => { approvalPolicy: 'never', sandbox: 'workspace-write', ephemeral: true, - experimentalRawEvents: false, - persistExtendedHistory: false, })) as { thread?: { id?: unknown } }; expect(typeof echo.thread?.id).toBe('string'); } finally { diff --git a/tests/codex-app-server-protocol.test.ts b/tests/codex-app-server-protocol.test.ts index a214f0e..672d891 100644 --- a/tests/codex-app-server-protocol.test.ts +++ b/tests/codex-app-server-protocol.test.ts @@ -23,7 +23,7 @@ import type { } from '../src/agents/codex-app-server/protocol/index.js'; describe('vendored codex-app-server protocol types', () => { - it('ClientRequestMethod enumerates the 8 methods the pathgrade driver may send', () => { + it('ClientRequestMethod enumerates the 9 methods the pathgrade driver may send', () => { // Switch-exhaustiveness-style probe. Response-shaped entries // (e.g. */answer) are deliberately excluded — they flow through // `transport.sendResponse(req.id, …)`, not through this union. @@ -36,8 +36,9 @@ describe('vendored codex-app-server protocol types', () => { 'thread/read': true, 'thread/list': true, 'review/start': true, + 'account/login/start': true, }; - expect(Object.keys(values)).toHaveLength(8); + expect(Object.keys(values)).toHaveLength(9); }); it('ClientRequestMethod excludes phantom response-shaped entries', () => { @@ -49,7 +50,7 @@ describe('vendored codex-app-server protocol types', () => { expect(true).toBe(true); }); - it('ServerRequestMethod enumerates the 9 server-request variants under rust-v0.124.0', () => { + it('ServerRequestMethod enumerates the 10 server-request variants under rust-v0.144.0', () => { const values: Record = { 'item/tool/requestUserInput': true, 'item/permissions/requestApproval': true, @@ -60,8 +61,9 @@ describe('vendored codex-app-server protocol types', () => { 'applyPatchApproval': true, 'execCommandApproval': true, 'account/chatgptAuthTokens/refresh': true, + 'attestation/generate': true, }; - expect(Object.keys(values)).toHaveLength(9); + expect(Object.keys(values)).toHaveLength(10); }); it('ServerRequestMethod lists each v0.124 rename target individually', () => { @@ -107,12 +109,13 @@ describe('vendored codex-app-server protocol types', () => { expect(q.options?.[0].label).toBe('us-east-1'); }); - it('ToolRequestUserInputParams has threadId/turnId/itemId/questions', () => { + it('ToolRequestUserInputParams has threadId/turnId/itemId/questions and auto-resolution', () => { const params: ToolRequestUserInputParams = { threadId: 't', turnId: 'turn', itemId: 'item', questions: [], + autoResolutionMs: null, }; expect(params.questions).toEqual([]); }); @@ -152,7 +155,7 @@ describe('vendored codex-app-server protocol types', () => { jsonrpc: '2.0', id: 1, method: 'turn/start', - params: { threadId: 't', turnId: 'turn', itemId: 'item', questions: [] }, + params: { threadId: 't', turnId: 'turn', itemId: 'item', questions: [], autoResolutionMs: null }, }; expect(req.jsonrpc).toBe('2.0'); }); @@ -161,7 +164,7 @@ describe('vendored codex-app-server protocol types', () => { const req: ServerRequest = { method: 'item/tool/requestUserInput', id: 42, - params: { threadId: 't', turnId: 'turn', itemId: 'item', questions: [] }, + params: { threadId: 't', turnId: 'turn', itemId: 'item', questions: [], autoResolutionMs: null }, }; if (req.method === 'item/tool/requestUserInput') { expect(req.params.itemId).toBe('item'); @@ -172,13 +175,13 @@ describe('vendored codex-app-server protocol types', () => { it('Other vendored types compile standalone', () => { // Ensure standalone vendored types import without errors. - const thread: ThreadStartParams = { - experimentalRawEvents: false, - persistExtendedHistory: false, - }; + const thread: ThreadStartParams = { threadSource: 'pathgrade' }; + // @ts-expect-error — rust-v0.144.0 ThreadSource is a string. + const invalidThreadSource: ThreadStartParams = { threadSource: { source: 'pathgrade' } }; const opt: ToolRequestUserInputOption = { label: 'x', description: 'y' }; const perm: PermissionsRequestApprovalParams = { threadId: 't', turnId: 'turn', itemId: 'item', cwd: '/tmp', reason: null, permissions: null, + environmentId: 'env-1', startedAtMs: 1, }; const dyn: DynamicToolCallParams = { threadId: 't', turnId: 'turn', callId: 'c', namespace: null, tool: 'x', arguments: null, @@ -192,6 +195,7 @@ describe('vendored codex-app-server protocol types', () => { threadId: 't', turnId: null, serverName: 'mcp', mode: 'url', _meta: null, message: 'hi', url: 'https://x', elicitationId: 'e', }; const turn: TurnCompletedNotification = { threadId: 't', turn: null }; - expect([thread.experimentalRawEvents, opt.label, perm.cwd, dyn.tool, elicit.mode, turn.threadId]).toBeDefined(); + expect([thread.threadSource, opt.label, perm.environmentId, perm.startedAtMs, dyn.tool, elicit.mode, turn.threadId]).toBeDefined(); + void invalidThreadSource; }); }); diff --git a/tests/codex-app-server-real-mcp.test.ts b/tests/codex-app-server-real-mcp.test.ts index 32278e5..0371ba0 100644 --- a/tests/codex-app-server-real-mcp.test.ts +++ b/tests/codex-app-server-real-mcp.test.ts @@ -63,8 +63,6 @@ suite('Codex app-server real MCP mounting', () => { approvalPolicy: 'never', sandbox: 'workspace-write', ephemeral: true, - experimentalRawEvents: false, - persistExtendedHistory: false, model: process.env.PATHGRADE_REAL_CODEX_MODEL ?? 'gpt-5.4', config: mcpConfig, }); diff --git a/tests/codex-app-server-transport.test.ts b/tests/codex-app-server-transport.test.ts index 9df9ad5..5750d2c 100644 --- a/tests/codex-app-server-transport.test.ts +++ b/tests/codex-app-server-transport.test.ts @@ -2,8 +2,10 @@ import { describe, expect, it, vi } from 'vitest'; import { PassThrough } from 'stream'; import { buildAppServerSpawnArgs, + buildAppServerProcessArgs, createAppServerSessionHandle, createNdjsonTransport, + spawnAppServerTransport, type SessionChildHandle, } from '../src/agents/codex-app-server/transport.js'; @@ -204,6 +206,86 @@ describe('buildAppServerSpawnArgs', () => { }); }); +describe('buildAppServerProcessArgs', () => { + it('places a bundled JavaScript launcher before Codex app-server arguments', () => { + expect(buildAppServerProcessArgs( + ['/tool/codex.js'], + ['--model', 'gpt-5.4'], + {}, + )).toEqual([ + '/tool/codex.js', + '-c', + 'features.default_mode_request_user_input=true', + '--model', + 'gpt-5.4', + 'app-server', + ]); + }); +}); + +describe('spawnAppServerTransport stderr diagnostics', () => { + it('redacts every API-key occurrence while retaining safe stderr context', async () => { + const sentinelKey = 'sentinel-key-must-not-leak'; + const error = vi.spyOn(console, 'error').mockImplementation(() => undefined); + const handle = spawnAppServerTransport({ + binary: process.execPath, + prefixArgs: [ + '-e', + [ + 'const key = process.env.OPENAI_API_KEY;', + 'process.stderr.write(`safe-prefix\\n${key.slice(0, 8)}`);', + 'setTimeout(() => {', + 'process.stderr.write(`${key.slice(8)}\\nrepeat=${key}\\n${key}\\nsafe-suffix\\n`);', + 'process.exit(17);', + '}, 10);', + ].join(''), + '--', + ], + env: { ...process.env, OPENAI_API_KEY: sentinelKey }, + }); + + try { + await new Promise((resolve) => { + handle.transport.onClose(() => resolve()); + }); + const diagnostic = error.mock.calls.flat().join('\n'); + expect(diagnostic).toContain('safe-prefix'); + expect(diagnostic).toContain('safe-suffix'); + expect(diagnostic).not.toContain(sentinelKey); + expect(diagnostic.match(/\[redacted\]/g)).toHaveLength(3); + } finally { + await handle.close(); + error.mockRestore(); + } + }); + + it('retains stderr unchanged when no API key is configured', async () => { + const error = vi.spyOn(console, 'error').mockImplementation(() => undefined); + const { OPENAI_API_KEY: _removed, ...envWithoutApiKey } = process.env; + const handle = spawnAppServerTransport({ + binary: process.execPath, + prefixArgs: [ + '-e', + 'process.stderr.write("safe-context-unchanged\\n");process.exit(17);', + '--', + ], + env: envWithoutApiKey, + }); + + try { + await new Promise((resolve) => { + handle.transport.onClose(() => resolve()); + }); + const diagnostic = error.mock.calls.flat().join('\n'); + expect(diagnostic).toContain('safe-context-unchanged'); + expect(diagnostic).not.toContain('[redacted]'); + } finally { + await handle.close(); + error.mockRestore(); + } + }); +}); + interface FakeChild extends SessionChildHandle { readonly killSignals: NodeJS.Signals[]; fireExit(): void; diff --git a/tests/commands.affected.test.ts b/tests/commands.affected.test.ts index 782face..ce63656 100644 --- a/tests/commands.affected.test.ts +++ b/tests/commands.affected.test.ts @@ -105,6 +105,35 @@ export default { } }); + it('standalone affected ignores legacy Vitest config', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'pathgrade-affected-standalone-')); + fs.mkdirSync(path.join(root, 'skills/alpha'), { recursive: true }); + fs.writeFileSync(path.join(root, 'skills/alpha/SKILL.md'), '# alpha'); + fs.writeFileSync( + path.join(root, 'skills/alpha/alpha.eval.ts'), + "import { createAgent } from '@wix/pathgrade';\nvoid createAgent;\n", + ); + fs.writeFileSync(path.join(root, 'vitest.config.ts'), ` +export default { + plugins: [{ + name: 'pathgrade', + __pathgradeOptions: { include: ['legacy/**/*.eval.ts'] }, + }], +}; +`); + fs.writeFileSync(changedFilesPath, 'skills/alpha/src/x.ts\n'); + + const cap = captureStd(); + try { + const code = await runAffected({ cwd: root, changedFilesPath, standalone: true }); + expect(code).toBe(0); + expect(cap.stdout()).toContain('skills/alpha/alpha.eval.ts'); + } finally { + cap.restore(); + try { fs.unlinkSync(changedFilesPath); } catch {} + } + }); + it('empty changed-files file → empty stdout, exit 0', async () => { fs.writeFileSync(changedFilesPath, ''); diff --git a/tests/credentials.test.ts b/tests/credentials.test.ts index f46802e..7c7c86a 100644 --- a/tests/credentials.test.ts +++ b/tests/credentials.test.ts @@ -15,6 +15,30 @@ function stubPorts(overrides?: Partial): CredentialPorts { } describe('resolveCredentials', () => { + it('requires Claude credentials in standalone mode', async () => { + await expect(resolveCredentials( + 'claude', + {}, + stubPorts({ platform: 'linux' }), + { mode: 'standalone' }, + )).rejects.toThrow(/Claude authentication required/); + }); + + it('requires an API key and defers login to the Codex app-server', async () => { + const ports = stubPorts({ + hostEnv: key => key === 'OPENAI_API_KEY' ? 'sk-test' : undefined, + }); + const result = await resolveCredentials( + 'codex', + {}, + ports, + { mode: 'standalone', transport: 'app-server' }, + ); + expect(result.env.OPENAI_API_KEY).toBe('sk-test'); + expect(result.setupCommands).toEqual([]); + expect(result.copyFromHome).toEqual([]); + }); + // --- Claude scenarios --- it('claude: user-provided ANTHROPIC_API_KEY → empty (trusted)', async () => { diff --git a/tests/docs-mcp-safety.test.ts b/tests/docs-mcp-safety.test.ts index 765bb50..de49c4c 100644 --- a/tests/docs-mcp-safety.test.ts +++ b/tests/docs-mcp-safety.test.ts @@ -27,4 +27,23 @@ describe('MCP safety documentation', () => { expect(guide).toContain('Denied calls are declined before approval'); expect(guide).toContain('secret-looking MCP arguments are redacted'); }); + + it('separates standalone prerequisites and environment behavior from project-local mode', async () => { + const guide = await fs.readFile(path.join(docsDir, 'USER_GUIDE.md'), 'utf8'); + + for (const promise of [ + 'Standalone—Node only', + 'Node.js 22 or 24', + 'macOS, Linux, or WSL', + 'root `vitest`', + 'Vitest subpaths', + 'project Vitest configuration is ignored', + 'does not load `.env`', + 'not a security sandbox', + 'application dependencies remain your responsibility', + 'project-local commands remain unchanged', + ]) { + expect(guide).toContain(promise); + } + }); }); diff --git a/tests/evaluate-from-snapshot.test.ts b/tests/evaluate-from-snapshot.test.ts index 9f1def4..d08d0d7 100644 --- a/tests/evaluate-from-snapshot.test.ts +++ b/tests/evaluate-from-snapshot.test.ts @@ -5,7 +5,7 @@ import * as path from 'path'; import type { Agent, JudgeScorer, Scorer } from '../src/sdk/types.js'; import type { CommandResult, LogEntry } from '../src/types.js'; import type { ToolEvent } from '../src/tool-events.js'; -import { buildRunSnapshot, evaluate, setRuntime, resetRuntime } from '../src/sdk/index.js'; +import { buildRunSnapshot, createAgent, evaluate, setRuntime, resetRuntime } from '../src/sdk/index.js'; import { resetAllResultObserversForTests, subscribeToEvalResults } from '../src/sdk/result-capture.js'; import { createMockLLM } from '../src/utils/llm-mocks.js'; @@ -20,6 +20,7 @@ function makeAgent(overrides?: { log?: LogEntry[]; messages?: Array<{ role: 'user' | 'agent'; content: string }>; transcriptStr?: string; + provenance?: Agent['provenance']; }): Agent { const workspace = overrides?.workspace ?? '/fake/workspace'; const log = overrides?.log ?? []; @@ -34,6 +35,7 @@ function makeAgent(overrides?: { log, messages, llm: createMockLLM(), + ...(overrides?.provenance ? { provenance: overrides.provenance } : {}), transcript: () => transcriptStr, exec: async (_cmd: string): Promise => ({ stdout: '', stderr: '', exitCode: 0, @@ -67,6 +69,174 @@ describe('evaluate.fromSnapshot', () => { tempPaths.length = 0; }); + it('records optional agent provenance for live and replayed trials', async () => { + const provenance = { + agent: 'claude' as const, + transport: 'native' as const, + model: { id: null, source: 'provider-default' as const }, + authentication: 'api-key' as const, + runtime: { + package: '@anthropic-ai/claude-agent-sdk', + package_version: '0.2.116', + embedded_binary_version: '2.1.116', + provenance: 'bundled' as const, + }, + }; + const agent = makeAgent({ provenance }); + const scorers: Scorer[] = [{ type: 'check', name: 'passes', weight: 1, fn: () => true }]; + const live = await evaluate(agent, scorers); + + const snapshotDir = path.join(os.tmpdir(), `pg-from-snapshot-provenance-${Math.random().toString(36).slice(2)}`); + tempPaths.push(snapshotDir); + await fs.ensureDir(snapshotDir); + const snapshotPath = path.join(snapshotDir, 'run-snapshot.json'); + await fs.writeJSON(snapshotPath, buildRunSnapshot({ + agent: 'claude', + agent_provenance: provenance, + messages: [], + log: [], + conversationResult: { turns: 0, completionReason: 'until', turnTimings: [], stepResults: [] }, + workspace: snapshotDir, + })); + const replayed = await evaluate.fromSnapshot(snapshotPath, scorers); + + expect(live.trial?.agent_provenance).toEqual(provenance); + expect(replayed.trial?.agent_provenance).toEqual(provenance); + }); + + it('records verified standalone Claude provenance', async () => { + const workspaceDir = path.join(os.tmpdir(), `pg-claude-provenance-${Math.random().toString(36).slice(2)}`); + tempPaths.push(workspaceDir); + await fs.ensureDir(workspaceDir); + const originalStandalone = process.env.PATHGRADE_STANDALONE; + process.env.PATHGRADE_STANDALONE = '1'; + + try { + const apiKeyAgent = await createAgent({ + agent: 'claude', workspace: workspaceDir, env: { ANTHROPIC_API_KEY: 'test-key' }, + }); + const oauthAgent = await createAgent({ + agent: 'claude', workspace: workspaceDir, env: { PATHGRADE_CLAUDE_LOCAL_OAUTH: '1' }, + }); + const scorers: Scorer[] = [{ type: 'check', name: 'passes', weight: 1, fn: () => true }]; + + const apiKeyTrial = (await evaluate(apiKeyAgent, scorers)).trial; + const oauthTrial = (await evaluate(oauthAgent, scorers)).trial; + + expect(apiKeyTrial?.agent_provenance).toEqual({ + agent: 'claude', + transport: 'native', + model: { id: null, source: 'provider-default' }, + authentication: 'api-key', + runtime: { + package: '@anthropic-ai/claude-agent-sdk', + package_version: '0.2.116', + embedded_binary_version: '2.1.116', + provenance: 'bundled', + }, + }); + expect(oauthTrial?.agent_provenance?.authentication).toBe('claude-oauth'); + + await apiKeyAgent.dispose(); + await oauthAgent.dispose(); + } finally { + if (originalStandalone === undefined) delete process.env.PATHGRADE_STANDALONE; + else process.env.PATHGRADE_STANDALONE = originalStandalone; + } + }); + + it('records Codex native runtime provenance after its bundled version probe', async () => { + const workspaceDir = path.join(os.tmpdir(), `pg-codex-provenance-${Math.random().toString(36).slice(2)}`); + tempPaths.push(workspaceDir); + await fs.ensureDir(workspaceDir); + const originalStandalone = process.env.PATHGRADE_STANDALONE; + process.env.PATHGRADE_STANDALONE = '1'; + + try { + const agent = await createAgent({ + agent: 'codex', workspace: workspaceDir, env: { OPENAI_API_KEY: 'test-key' }, + }); + const result = await evaluate(agent, [{ + type: 'check', name: 'passes', weight: 1, fn: () => true, + }]); + + expect(result.trial?.agent_provenance).toMatchObject({ + agent: 'codex', + transport: 'app-server', + model: { id: 'gpt-5.4', source: 'pathgrade-default' }, + runtime: { + package: '@openai/codex', + package_version: '0.144.0', + embedded_binary_version: '0.144.0', + provenance: 'bundled', + }, + }); + + await agent.dispose(); + } finally { + if (originalStandalone === undefined) delete process.env.PATHGRADE_STANDALONE; + else process.env.PATHGRADE_STANDALONE = originalStandalone; + } + }); + + it('accepts older version-1 snapshots that omit agent provenance', async () => { + const snapshotDir = path.join(os.tmpdir(), `pg-from-snapshot-no-provenance-${Math.random().toString(36).slice(2)}`); + tempPaths.push(snapshotDir); + await fs.ensureDir(snapshotDir); + const snapshotPath = path.join(snapshotDir, 'run-snapshot.json'); + await fs.writeJSON(snapshotPath, { + version: 1, + timestamp: '2026-01-01T00:00:00.000Z', + agent: 'claude', + messages: [], + log: [], + toolEvents: [], + turnTimings: [], + conversationResult: { turns: 0, completionReason: 'until', turnTimings: [] }, + workspace: null, + }); + + const replayed = await evaluate.fromSnapshot(snapshotPath, [{ + type: 'check', name: 'still loads', weight: 1, fn: () => true, + }]); + + expect(replayed.trial?.agent_provenance).toBeUndefined(); + }); + + it('rejects snapshots with an invalid provenance model union', async () => { + const snapshotDir = path.join(os.tmpdir(), `pg-from-snapshot-invalid-provenance-${Math.random().toString(36).slice(2)}`); + tempPaths.push(snapshotDir); + await fs.ensureDir(snapshotDir); + const snapshotPath = path.join(snapshotDir, 'run-snapshot.json'); + await fs.writeJSON(snapshotPath, { + version: 1, + timestamp: '2026-01-01T00:00:00.000Z', + agent: 'claude', + agent_provenance: { + agent: 'claude', + transport: 'native', + model: { id: null, source: 'user' }, + authentication: 'api-key', + runtime: { + package: '@anthropic-ai/claude-agent-sdk', + package_version: '0.2.116', + provenance: 'bundled', + }, + }, + messages: [], + log: [], + toolEvents: [], + turnTimings: [], + conversationResult: { turns: 0, completionReason: 'until', turnTimings: [] }, + workspace: null, + }); + + await expect(evaluate.fromSnapshot(snapshotPath, [])).rejects.toMatchObject({ + name: 'SnapshotParseError', + message: 'Snapshot agent_provenance is invalid', + }); + }); + it('matches live deterministic scorer results for the same artifacts', async () => { const toolEvent: ToolEvent = { action: 'run_shell', diff --git a/tests/fixtures/runner-cli-smoke/standalone-basic/.env b/tests/fixtures/runner-cli-smoke/standalone-basic/.env new file mode 100644 index 0000000..86a4ffd --- /dev/null +++ b/tests/fixtures/runner-cli-smoke/standalone-basic/.env @@ -0,0 +1 @@ +PATHGRADE_STANDALONE_ENV_SENTINEL=leaked diff --git a/tests/fixtures/runner-cli-smoke/standalone-basic/basic.eval.ts b/tests/fixtures/runner-cli-smoke/standalone-basic/basic.eval.ts new file mode 100644 index 0000000..a782bcd --- /dev/null +++ b/tests/fixtures/runner-cli-smoke/standalone-basic/basic.eval.ts @@ -0,0 +1,32 @@ +import { check, evaluate, type Agent } from '@wix/pathgrade'; +import { describe, expect, it } from 'vitest'; + +describe('standalone CLI Vitest smoke', () => { + it('uses the internal mode and ignores target environment files', async () => { + expect(process.env.PATHGRADE_STANDALONE).toBe('1'); + expect(process.env.PATHGRADE_STANDALONE_ENV_SENTINEL).toBeUndefined(); + + const result = await evaluate(fakeAgent(), [ + check('always passes', () => true), + ]); + expect(result.score).toBe(1); + }); +}); + +function fakeAgent(): Agent { + return { + workspace: process.cwd(), + log: [], + messages: [], + llm: { + tokenUsage: { inputTokens: 0, outputTokens: 0 }, + call: async () => ({ text: '', provider: 'cli', model: 'fake' }), + }, + transcript: () => '', + exec: async () => ({ exitCode: 0, stdout: '', stderr: '' }), + prompt: async () => { throw new Error('not used'); }, + runConversation: async () => { throw new Error('not used'); }, + startChat: async () => { throw new Error('not used'); }, + dispose: async () => undefined, + }; +} diff --git a/tests/fixtures/runner-cli-smoke/standalone-basic/vitest.config.ts b/tests/fixtures/runner-cli-smoke/standalone-basic/vitest.config.ts new file mode 100644 index 0000000..aa449a5 --- /dev/null +++ b/tests/fixtures/runner-cli-smoke/standalone-basic/vitest.config.ts @@ -0,0 +1 @@ +throw new Error('standalone must not load target vitest.config.ts'); diff --git a/tests/fixtures/standalone-package/.env b/tests/fixtures/standalone-package/.env new file mode 100644 index 0000000..371c253 --- /dev/null +++ b/tests/fixtures/standalone-package/.env @@ -0,0 +1 @@ +PATHGRADE_STANDALONE_ENV_SENTINEL=loaded-from-target diff --git a/tests/fixtures/standalone-package/basic.eval.ts b/tests/fixtures/standalone-package/basic.eval.ts new file mode 100644 index 0000000..666529a --- /dev/null +++ b/tests/fixtures/standalone-package/basic.eval.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from 'vitest'; +import { appendFile } from 'node:fs/promises'; +import { check, evaluate, type Agent } from '@wix/pathgrade'; + +const lifecycleStateFile = process.env.PATHGRADE_STANDALONE_SMOKE_STATE_FILE; + +async function recordLifecycle(event: string): Promise { + if (!lifecycleStateFile) { + throw new Error('missing PATHGRADE_STANDALONE_SMOKE_STATE_FILE'); + } + await appendFile(lifecycleStateFile, `${event}\n`, 'utf8'); +} + +describe('standalone package', () => { + it('runs a deterministic Pathgrade evaluation', async () => { + expect(process.env.PATHGRADE_STANDALONE_ENV_SENTINEL).toBeUndefined(); + const agent: Agent = { + workspace: process.cwd(), + log: [], + messages: [], + llm: { + tokenUsage: { inputTokens: 0, outputTokens: 0 }, + call: async () => ({ text: '', provider: 'cli', model: 'fake' }), + }, + transcript: () => '', + exec: async () => ({ exitCode: 0, stdout: '', stderr: '' }), + prompt: async () => { throw new Error('not used by deterministic eval'); }, + runConversation: async () => { throw new Error('not used by deterministic eval'); }, + startChat: async () => { throw new Error('not used by deterministic eval'); }, + dispose: async () => recordLifecycle('agent-disposed'), + }; + await recordLifecycle('evaluation-started'); + const result = await evaluate(agent, [check('standalone scorer', () => true)]); + expect(result.score).toBe(1); + await recordLifecycle('evaluation-scored'); + }); +}); diff --git a/tests/fixtures/standalone-package/missing-dependency.eval.ts b/tests/fixtures/standalone-package/missing-dependency.eval.ts new file mode 100644 index 0000000..4acb753 --- /dev/null +++ b/tests/fixtures/standalone-package/missing-dependency.eval.ts @@ -0,0 +1 @@ +import 'package-that-pathgrade-must-not-supply'; diff --git a/tests/fixtures/standalone-package/package.json b/tests/fixtures/standalone-package/package.json new file mode 100644 index 0000000..3a2a47a --- /dev/null +++ b/tests/fixtures/standalone-package/package.json @@ -0,0 +1,4 @@ +{ + "name": "pathgrade-standalone-smoke-target", + "private": true +} diff --git a/tests/fixtures/standalone-package/vitest.config.ts b/tests/fixtures/standalone-package/vitest.config.ts new file mode 100644 index 0000000..9e6c87f --- /dev/null +++ b/tests/fixtures/standalone-package/vitest.config.ts @@ -0,0 +1 @@ +throw new Error('standalone mode must not load project vitest.config.ts'); diff --git a/tests/package-surface.test.ts b/tests/package-surface.test.ts index 583a3e6..ccf83db 100644 --- a/tests/package-surface.test.ts +++ b/tests/package-surface.test.ts @@ -35,6 +35,8 @@ describe('package surface', () => { expect(packageJson.peerDependenciesMeta.jest.optional).toBe(true); expect(packageJson.peerDependencies.vitest).toBeDefined(); expect(packageJson.peerDependenciesMeta.vitest.optional).toBe(true); + expect(packageJson.dependencies.vitest).toBe('4.1.7'); + expect(packageJson.peerDependencies.vitest).toBe('^4.0.0'); }); it('exposes Jest through the built-in adapter subpaths', () => { @@ -51,4 +53,21 @@ describe('package surface', () => { default: './dist/adapters/jest/reporter.cjs', }); }); + + it('keeps the standalone runtime pins exact and publishes only the scoped package', () => { + expect(packageJson.name).toBe('@wix/pathgrade'); + expect(packageJson.bin).toBe('bin/pathgrade.js'); + expect(packageJson.repository.url).toBe('git+https://github.com/wix-incubator/pathgrade.git'); + expect(packageJson.dependencies.vitest).toBe('4.1.7'); + expect(packageJson.dependencies['@anthropic-ai/claude-agent-sdk']).toBe('0.2.116'); + expect(packageJson.dependencies['@openai/codex']).toBe('0.144.0'); + expect(packageJson.scripts['test:release-contracts']).toBe( + 'node tests/release-platform-evidence.test.mjs && node tests/publish-workflow-contract.test.mjs', + ); + expect(packageJson.scripts.test).toBe( + 'yarn build && vitest run && yarn test:runner-cli-smoke && yarn test:standalone-package-smoke && yarn test:release-contracts', + ); + expect(packageJson.files).toContain('dist/**/*.js'); + expect(packageJson).not.toHaveProperty('workspaces'); + }); }); diff --git a/tests/package-types.test.ts b/tests/package-types.test.ts new file mode 100644 index 0000000..72305c2 --- /dev/null +++ b/tests/package-types.test.ts @@ -0,0 +1,65 @@ +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { createRequire } from 'node:module'; +import { describe, expect, it } from 'vitest'; + +const require = createRequire(import.meta.url); + +describe('published package types', () => { + it('accepts both structural Agents without provenance and Agents with provenance', () => { + const consumerDir = mkdtempSync(join(tmpdir(), 'pathgrade-package-types-')); + const declarationPath = resolve(process.cwd(), 'dist/sdk/index.d.ts'); + const tscPath = require.resolve('typescript/lib/tsc.js'); + + try { + writeFileSync(join(consumerDir, 'consumer.ts'), ` +import type { Agent, AgentInvocationProvenance } from '@wix/pathgrade'; + +const base = { + workspace: '/workspace', + log: [], + messages: [], + llm: { call: async () => ({ text: '', provider: 'cli' as const, model: 'test' }) }, + prompt: async () => '', + runConversation: async () => ({ turns: 0, completionReason: 'until' as const, turnTimings: [], stepResults: [] }), + startChat: async () => { throw new Error('unused'); }, + exec: async () => ({ stdout: '', stderr: '', exitCode: 0 }), + transcript: () => '', + dispose: async () => {}, +}; + +const existingConsumerAgent: Agent = base; +const provenance: AgentInvocationProvenance = { + agent: 'claude', transport: 'native', model: { id: null, source: 'provider-default' }, + authentication: 'api-key', + runtime: { package: '@anthropic-ai/claude-agent-sdk', package_version: '0.2.116', embedded_binary_version: '2.1.116', provenance: 'bundled' }, +}; +const newConsumerAgent: Agent = { ...base, provenance }; +void existingConsumerAgent; +void newConsumerAgent; +`); + writeFileSync(join(consumerDir, 'tsconfig.json'), JSON.stringify({ + compilerOptions: { + strict: true, + noEmit: true, + target: 'ES2022', + module: 'NodeNext', + moduleResolution: 'NodeNext', + skipLibCheck: true, + baseUrl: '.', + paths: { '@wix/pathgrade': [declarationPath] }, + }, + files: ['consumer.ts'], + })); + + expect(() => execFileSync(process.execPath, [tscPath, '-p', join(consumerDir, 'tsconfig.json')], { + encoding: 'utf8', + stdio: 'pipe', + })).not.toThrow(); + } finally { + rmSync(consumerDir, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/pathgrade-config.test.ts b/tests/pathgrade-config.test.ts index a96acd7..e951028 100644 --- a/tests/pathgrade-config.test.ts +++ b/tests/pathgrade-config.test.ts @@ -90,6 +90,22 @@ export default { expect(config.affected.global).toEqual(['yarn.lock']); }); + it('skips legacy Vitest plugin options in standalone mode', async () => { + const root = makeRepo(); + writeFile(root, 'vitest.config.ts', ` +export default { + plugins: [{ + name: 'pathgrade', + __pathgradeOptions: { include: ['legacy/**/*.eval.ts'] }, + }], +}; +`); + + const resolved = await resolvePathgradeConfig({ cwd: root, standalone: true }); + + expect(resolved.evals.include).toEqual(['**/*.eval.ts']); + }); + it('prefers CLI overrides over pathgrade.config.ts and legacy Vitest fallback', async () => { const root = makeRepo(); writeFile(root, 'vitest.config.ts', ` diff --git a/tests/release-platform-evidence.test.mjs b/tests/release-platform-evidence.test.mjs new file mode 100644 index 0000000..63a8df8 --- /dev/null +++ b/tests/release-platform-evidence.test.mjs @@ -0,0 +1,208 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const validator = path.join(repoRoot, 'scripts/release/verify-platform-evidence.mjs'); +const commit = '0123456789abcdef0123456789abcdef01234567'; +const tarballSha512 = 'a'.repeat(128); +const tuples = { + 'darwin-arm64-node24': ['darwin', 'arm64'], + 'darwin-x64-node24': ['darwin', 'x64'], + 'linux-arm64-node24': ['linux', 'arm64'], + 'linux-x64-node24': ['linux', 'x64'], + 'wsl-x64-node24': ['linux', 'x64'], +}; + +test('accepts exactly one current passing record for every supported tuple', () => { + const result = runValidator(Object.entries(tuples).map(([tupleId, [platform, arch]]) => ( + evidence(tupleId, platform, arch) + ))); + + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(result.stdout), { + status: 'pass', + tuple_ids: Object.keys(tuples), + }); +}); + +for (const missingTuple of Object.keys(tuples)) { + test(`rejects a missing ${missingTuple} record`, () => { + const records = Object.entries(tuples) + .filter(([tupleId]) => tupleId !== missingTuple) + .map(([tupleId, [platform, arch]]) => evidence(tupleId, platform, arch)); + + assertRejected(runValidator(records), missingTuple); + }); +} + +test('rejects duplicate and unknown tuple records', () => { + const records = validRecords(); + assertRejected(runValidator([...records, records[0]]), 'duplicate darwin-arm64-node24'); + assertRejected(runValidator([...records, evidence('solaris-x64-node24', 'sunos', 'x64')]), 'unknown tuple_id solaris-x64-node24'); +}); + +test('rejects stale commits and wrong tarball hashes by field', () => { + const stale = validRecords(); + stale[0] = { ...stale[0], source_commit: 'f'.repeat(40) }; + assertRejected(runValidator(stale), 'darwin-arm64-node24 source_commit'); + + const wrongHash = validRecords(); + wrongHash[1] = { ...wrongHash[1], tarball_sha512: 'b'.repeat(128) }; + assertRejected(runValidator(wrongHash), 'darwin-x64-node24 tarball_sha512'); +}); + +test('rejects wrong runtime versions and non-passing records by field', () => { + const wrongRuntime = validRecords(); + wrongRuntime[2] = { + ...wrongRuntime[2], + runtimes: { + ...wrongRuntime[2].runtimes, + codex: { package_version: '0.145.0', native_version: '0.144.0' }, + }, + }; + assertRejected(runValidator(wrongRuntime), 'linux-arm64-node24 runtimes.codex.package_version'); + + const failed = validRecords(); + failed[3] = { ...failed[3], status: 'fail' }; + assertRejected(runValidator(failed), 'linux-x64-node24 status'); +}); + +test('rejects native Linux evidence relabeled as WSL', () => { + const records = validRecords(); + const index = records.findIndex(record => record.tuple_id === 'wsl-x64-node24'); + records[index] = { + ...records[index], + runtime_environment: { + kind: 'github-hosted', observed_platform: 'linux', observed_arch: 'x64', + wsl: false, kernel_release: '6.8.0-generic', wsl_interop: null, + }, + }; + assertRejected(runValidator(records), 'wsl-x64-node24 runtime_environment.kind'); +}); + +test('validates exact provider count, versions, model, authentication, transport, commit, and digest', () => { + const liveDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'pathgrade-live-evidence-')); + try { + fs.writeFileSync(path.join(liveDirectory, 'linux.json'), JSON.stringify(liveEvidence('linux', ['claude', 'codex']))); + fs.writeFileSync(path.join(liveDirectory, 'macos.json'), JSON.stringify(liveEvidence('darwin', ['codex']))); + const result = runValidator(validRecords(), liveDirectory); + assert.equal(result.status, 0, result.stderr); + + const forged = liveEvidence('linux', ['claude', 'codex']); + forged.providers[1].report.groups[0].trials[0].agent_provenance.model.id = 'forged-model'; + fs.writeFileSync(path.join(liveDirectory, 'linux.json'), JSON.stringify(forged)); + assertRejected(runValidator(validRecords(), liveDirectory), 'linux-codex model.id'); + } finally { + fs.rmSync(liveDirectory, { recursive: true, force: true }); + } +}); + +test('rejects malformed JSON with the evidence filename', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'pathgrade-platform-evidence-')); + try { + fs.writeFileSync(path.join(directory, 'broken.json'), '{not json'); + assertRejected(run(directory), 'broken.json'); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +function validRecords() { + return Object.entries(tuples).map(([tupleId, [platform, arch]]) => evidence(tupleId, platform, arch)); +} + +function evidence(tupleId, platform, arch) { + const wsl = tupleId === 'wsl-x64-node24'; + return { + tuple_id: tupleId, + platform, + arch, + node_major: 24, + package: { name: '@wix/pathgrade', version: '1.0.1' }, + runtimes: { + claude: { sdk_version: '0.2.116', claude_code_version: '2.1.116' }, + codex: { package_version: '0.144.0', native_version: '0.144.0' }, + }, + tarball_sha512: tarballSha512, + source_commit: commit, + status: 'pass', + runtime_environment: wsl ? { + kind: 'wsl', observed_platform: 'linux', observed_arch: 'x64', wsl: true, + kernel_release: '5.15.153.1-microsoft-standard-WSL2', wsl_interop: '/run/WSL/123_interop', + } : { + kind: 'github-hosted', observed_platform: platform, observed_arch: arch, wsl: false, + kernel_release: platform === 'linux' ? '6.8.0-generic' : null, wsl_interop: null, + }, + }; +} + +function runValidator(records, liveEvidenceDir) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'pathgrade-platform-evidence-')); + try { + records.forEach((record, index) => { + fs.writeFileSync(path.join(directory, `${index}-${record.tuple_id}.json`), JSON.stringify(record)); + }); + return run(directory, liveEvidenceDir); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } +} + +function run(evidenceDir, liveEvidenceDir) { + const args = [ + validator, + '--commit', commit, + '--tarball-sha512', tarballSha512, + '--evidence-dir', evidenceDir, + ]; + if (liveEvidenceDir) args.push('--live-evidence-dir', liveEvidenceDir); + return spawnSync(process.execPath, args, { cwd: repoRoot, encoding: 'utf8' }); +} + +function liveEvidence(platform, selected) { + const providers = ['claude', 'codex'].map(provider => { + const isSelected = selected.includes(provider); + const runtime = provider === 'claude' + ? { package: '@anthropic-ai/claude-agent-sdk', package_version: '0.2.116', embedded_binary_version: '2.1.116', provenance: 'bundled' } + : { package: '@openai/codex', package_version: '0.144.0', embedded_binary_version: '0.144.0', provenance: 'bundled' }; + const agentProvenance = { + agent: provider, transport: provider === 'claude' ? 'native' : 'app-server', + model: provider === 'claude' ? { id: null, source: 'provider-default' } : { id: 'gpt-5.4', source: 'pathgrade-default' }, + authentication: 'api-key', runtime, + }; + return { + id: `${platform === 'darwin' ? 'macos' : 'linux'}-${provider}`, provider, + status: isSelected ? 'pass' : 'skipped', skipped: isSelected ? 0 : 1, + ...(isSelected ? { report: { + version: 1, status: 'pass', overall_pass_rate: 1, + provenance: { + mode: 'standalone', runtimes: { + claude: { sdk_version: '0.2.116', claude_code_version: '2.1.116' }, + codex: { package_version: '0.144.0', native_version: '0.144.0' }, + }, + }, + groups: [{ trials: [{ reward: 1, agent_provenance: agentProvenance }] }], + } } : {}), + }; + }); + return { + schema: 'pathgrade-live-evidence/v1', status: 'pass', selected, + passed: selected.length, skipped: 2 - selected.length, providers, + tarball_sha512: tarballSha512, source_commit: commit, + runtime_environment: { platform, arch: platform === 'darwin' ? 'arm64' : 'x64', node_major: 24 }, + }; +} + +function assertRejected(result, diagnostic) { + assert.notEqual(result.status, 0, `expected rejection, got stdout: ${result.stdout}`); + assert.match(result.stderr, new RegExp(escapeRegExp(diagnostic))); +} + +function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/tests/reporting-core.contract.test.ts b/tests/reporting-core.contract.test.ts index 3efa359..a50c019 100644 --- a/tests/reporting-core.contract.test.ts +++ b/tests/reporting-core.contract.test.ts @@ -2,6 +2,17 @@ import { describe, expect, it } from 'vitest'; import { buildPathgradeReport } from '../src/reporting/core.js'; import type { ReportRunInput } from '../src/reporting/types.js'; +const provenance = { + mode: 'standalone' as const, + package_version: '1.0.1', + vitest_version: '4.1.7', + runtimes: { + claude: { sdk_version: '0.2.116', claude_code_version: '2.1.116' }, + codex: { package_version: '0.144.0', native_version: '0.144.0' }, + }, + platform: { os: process.platform, arch: process.arch, node: process.version }, +}; + describe('runner-neutral reporting contract', () => { it('builds compatible report artifacts from pure normalized inputs', () => { const input: ReportRunInput = { @@ -195,6 +206,13 @@ describe('runner-neutral reporting contract', () => { }); }); + it('preserves standalone provenance without changing the version-1 report contract', () => { + expect(buildPathgradeReport({ + groups: [], + provenance, + }).report.provenance).toEqual(provenance); + }); + it('keeps empty-metadata warnings for skipped cases while excluding them from artifacts', () => { const built = buildPathgradeReport({ groups: [ diff --git a/tests/runner-cli-smoke/run-smokes.mjs b/tests/runner-cli-smoke/run-smokes.mjs index 9405244..4471bfc 100644 --- a/tests/runner-cli-smoke/run-smokes.mjs +++ b/tests/runner-cli-smoke/run-smokes.mjs @@ -1,10 +1,12 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; import { spawnSync } from 'node:child_process'; const repoRoot = path.resolve(import.meta.dirname, '../..'); const cliPath = path.join(repoRoot, 'dist/pathgrade.js'); +const failingShimDir = createFailingShimDir(); const smokes = [ { @@ -17,6 +19,14 @@ const smokes = [ cwd: fixture('vitest-basic'), args: ['run', '--adapter=vitest'], }, + { + name: 'standalone bundled Vitest', + cwd: fixture('standalone-basic'), + args: ['standalone', 'run'], + env: { + PATH: `${failingShimDir}${path.delimiter}${process.env.PATH ?? ''}`, + }, + }, { name: 'Jest adapter without node_modules/.bin on PATH', cwd: path.join(repoRoot, 'tests/fixtures/jest-adapter'), @@ -61,8 +71,12 @@ const smokes = [ }, ]; -for (const smoke of smokes) { - runSmoke(smoke); +try { + for (const smoke of smokes) { + runSmoke(smoke); + } +} finally { + fs.rmSync(failingShimDir, { recursive: true, force: true }); } function runSmoke(smoke) { @@ -128,3 +142,13 @@ function removeIfEmpty(dir) { if (err?.code !== 'ENOENT' && err?.code !== 'ENOTEMPTY') throw err; } } + +function createFailingShimDir() { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'pathgrade-failing-shims-')); + for (const command of ['vitest', 'codex', 'claude']) { + const shimPath = path.join(directory, command); + fs.writeFileSync(shimPath, '#!/bin/sh\nexit 97\n'); + fs.chmodSync(shimPath, 0o755); + } + return directory; +} diff --git a/tests/standalone-codex-runtime.test.ts b/tests/standalone-codex-runtime.test.ts new file mode 100644 index 0000000..998b964 --- /dev/null +++ b/tests/standalone-codex-runtime.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + mkdirSync, + mkdtempSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + resolveBundledCodexCommand, + resolveBundledCodexNativeArtifact, + verifyBundledCodexRuntime, +} from '../src/agents/codex-runtime.js'; + +describe('bundled Codex runtime', () => { + it('rejects native Windows before resolving a bundled artifact', () => { + const platform = vi.spyOn(process, 'platform', 'get').mockReturnValue('win32'); + try { + expect(() => resolveBundledCodexCommand()).toThrow( + 'pathgrade standalone: native Windows is unsupported; use WSL with x64 or arm64', + ); + } finally { + platform.mockRestore(); + } + }); + + it('resolves the exact official package through Node', () => { + const runtime = resolveBundledCodexCommand(); + expect(runtime.executable).toBe(process.execPath); + expect(runtime.argsPrefix[0]).toMatch(/@openai[/\\]codex[/\\]bin[/\\]codex\.js$/); + expect(runtime.packageVersion).toBe('0.144.0'); + expect(runtime.nativeVersion).toBe('0.144.0'); + expect(runtime.provenance).toBe('bundled'); + }); + + it('resolves the native artifact from a non-hoisted Codex installation', () => { + const fixture = createCodexFixture('0.144.0'); + try { + expect(resolveBundledCodexNativeArtifact(fixture.packageJsonPath)) + .toBe(realpathSync(fixture.nativeExecutable)); + } finally { + fixture.cleanup(); + } + }); + + it('rejects a mismatched native package version before reporting provenance', () => { + const fixture = createCodexFixture('0.143.0'); + try { + expect(() => resolveBundledCodexNativeArtifact(fixture.packageJsonPath)) + .toThrow(/native artifact version must be 0\.144\.0-(darwin|linux)-(arm64|x64)/); + } finally { + fixture.cleanup(); + } + }); + + it('executes the packaged native artifact version probe without PATH lookup', async () => { + await expect(verifyBundledCodexRuntime(resolveBundledCodexCommand())) + .resolves.toEqual({ + packageVersion: '0.144.0', + nativeVersion: '0.144.0', + provenance: 'bundled', + }); + }); +}); + +function createCodexFixture(nativeVersion: string): { + packageJsonPath: string; + nativeExecutable: string; + cleanup: () => void; +} { + const fixtureRoot = mkdtempSync(join(tmpdir(), 'pathgrade-codex-non-hoisted-')); + const codexRoot = join(fixtureRoot, 'node_modules', '@openai', 'codex'); + const packageByTarget: Record = { + 'darwin:x64': { + packageName: '@openai/codex-darwin-x64', + triple: 'x86_64-apple-darwin', + versionSuffix: 'darwin-x64', + }, + 'darwin:arm64': { + packageName: '@openai/codex-darwin-arm64', + triple: 'aarch64-apple-darwin', + versionSuffix: 'darwin-arm64', + }, + 'linux:x64': { + packageName: '@openai/codex-linux-x64', + triple: 'x86_64-unknown-linux-musl', + versionSuffix: 'linux-x64', + }, + 'linux:arm64': { + packageName: '@openai/codex-linux-arm64', + triple: 'aarch64-unknown-linux-musl', + versionSuffix: 'linux-arm64', + }, + }; + const target = packageByTarget[`${process.platform}:${process.arch}`]!; + const platformRoot = join( + codexRoot, + 'node_modules', + ...target.packageName.split('/'), + ); + const packageJsonPath = join(codexRoot, 'package.json'); + const nativeExecutable = join(platformRoot, 'vendor', target.triple, 'bin', 'codex'); + + mkdirSync(join(codexRoot, 'bin'), { recursive: true }); + mkdirSync(join(platformRoot, 'vendor', target.triple, 'bin'), { recursive: true }); + writeFileSync(packageJsonPath, JSON.stringify({ + name: '@openai/codex', + version: '0.144.0', + bin: { codex: 'bin/codex.js' }, + })); + writeFileSync(join(codexRoot, 'bin', 'codex.js'), ''); + writeFileSync( + join(platformRoot, 'package.json'), + JSON.stringify({ + name: '@openai/codex', + version: `${nativeVersion}-${target.versionSuffix}`, + }), + ); + writeFileSync(nativeExecutable, ''); + + return { + packageJsonPath, + nativeExecutable, + cleanup: () => rmSync(fixtureRoot, { recursive: true, force: true }), + }; +} diff --git a/tests/standalone-diagnostics.test.ts b/tests/standalone-diagnostics.test.ts new file mode 100644 index 0000000..e389959 --- /dev/null +++ b/tests/standalone-diagnostics.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest'; +import { classifyStandaloneVitestFailure } from '../src/standalone/diagnostics.js'; + +describe('standalone Vitest diagnostics', () => { + it('classifies arbitrary unresolved imports as project dependencies', () => { + expect(classifyStandaloneVitestFailure( + 'Failed to resolve import "zod" from "/repo/example.eval.ts"', + )).toEqual({ + kind: 'project-dependency', + message: 'pathgrade standalone: project dependency "zod" is unavailable in /repo/example.eval.ts; install the application dependencies or remove that import', + }); + }); + + it.each(['vitest', '@wix/pathgrade'])( + 'classifies unresolved %s as a packaging defect', + (specifier) => { + expect(classifyStandaloneVitestFailure( + `Cannot find package '${specifier}' imported from /repo/example.eval.ts`, + )).toMatchObject({ kind: 'packaging' }); + }, + ); + + it('classifies a transitive failure inside the installed tool graph as packaging', () => { + expect(classifyStandaloneVitestFailure( + 'Failed to resolve import "tiny-invariant" from "/tool/node_modules/@wix/pathgrade/dist/sdk/index.js"', + )).toMatchObject({ kind: 'packaging' }); + }); + + it('keeps a bare package imported by the eval classified as a project dependency', () => { + expect(classifyStandaloneVitestFailure( + 'Failed to resolve import "tiny-invariant" from "/repo/example.eval.ts"', + )).toMatchObject({ kind: 'project-dependency' }); + }); + + it('leaves relative import failures as the original Vitest diagnostic', () => { + expect(classifyStandaloneVitestFailure( + 'Failed to resolve import "./missing.js" from "/repo/example.eval.ts"', + )).toBeUndefined(); + }); +}); diff --git a/tests/standalone-live-smoke.test.ts b/tests/standalone-live-smoke.test.ts new file mode 100644 index 0000000..c67257a --- /dev/null +++ b/tests/standalone-live-smoke.test.ts @@ -0,0 +1,574 @@ +import { afterAll, afterEach, describe, expect, it } from 'vitest'; +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { loginCodexAppServerWithApiKey } from '../src/agents/codex-app-server/agent.js'; +import { collectClaudeSdkMessages } from '../src/agents/claude.js'; + +type Provider = 'claude' | 'codex'; +type ProviderResult = { + id: string; + provider: Provider; + status: 'pass' | 'fail' | 'skipped'; + skipped: 0 | 1; + report?: any; +}; + +const repoRoot = path.resolve(import.meta.dirname, '..'); +const missingCredential = '__PATHGRADE_TEST_MISSING_CREDENTIAL__'; +const isGateChild = process.env.PATHGRADE_LIVE_GATE_CHILD === '1'; +const temporaryDirectories: string[] = []; +const gate = readGate(process.env); +const results = new Map(); + +for (const provider of ['claude', 'codex'] as const) { + const selected = gate.selected.includes(provider); + results.set(provider, { + id: `${process.platform === 'darwin' ? 'macos' : 'linux'}-${provider}`, + provider, + status: selected && gate.enabled ? 'fail' : 'skipped', + skipped: selected && gate.enabled ? 0 : 1, + }); +} + +describe('packed standalone live runtimes', () => { + it.skipIf(!shouldRun('claude'))('runs one Claude turn with bundled provenance and no credential leak', async () => { + await runLiveProvider('claude'); + }, 360_000); + + it.skipIf(!shouldRun('codex'))('runs one Codex app-server turn with isolated API-key login and no credential leak', async () => { + await runLiveProvider('codex'); + }, 360_000); +}); + +if (!isGateChild) describe('live-smoke gate contract', () => { + it('snapshots unexpected empty directories with their entry type', () => { + const target = makeTempDir('pathgrade-target-tree-'); + const baseline = snapshotTree(target); + fs.mkdirSync(path.join(target, 'unexpected-empty')); + expect(snapshotTree(target).get('unexpected-empty')).toEqual({ type: 'directory' }); + expect(() => assertExactTargetTree(target, baseline, new Map())).toThrow(); + }); + + it.skipIf(process.platform === 'win32')('snapshots unexpected symlinks and their target', () => { + const target = makeTempDir('pathgrade-target-tree-'); + const baseline = snapshotTree(target); + fs.symlinkSync('missing-target', path.join(target, 'unexpected-link')); + expect(snapshotTree(target).get('unexpected-link')).toEqual({ + type: 'symlink', target: 'missing-target', + }); + expect(() => assertExactTargetTree(target, baseline, new Map())).toThrow(); + }); + + it('redacts deterministic injected login and query failures at the provider boundaries', async () => { + const secret = 'pathgrade-live-redaction-sentinel'; + const codexCalls: Array<{ method: string; params: unknown }> = []; + const transport = { + sendRequest: async (method: string, params: unknown) => { + codexCalls.push({ method, params }); + throw new Error(`login sentinel Authorization: Bearer ${secret}`); + }, + } as any; + const loginError = await loginCodexAppServerWithApiKey(transport, secret).catch(error => error); + expect(loginError.message).toContain('login sentinel'); + expect(loginError.message).not.toContain(secret); + expect(codexCalls.map(call => call.method)).toEqual(['account/login/start']); + + const queryArgs: unknown[] = []; + const failingQuery = (args: unknown) => { + queryArgs.push(args); + return { + [Symbol.asyncIterator]() { return this; }, + async next() { throw new Error(`query sentinel _auth=${secret}`); }, + } as any; + }; + const queryError = await collectClaudeSdkMessages( + failingQuery as any, { prompt: 'safe prompt', options: {} }, [secret], + ).catch(error => error); + expect(queryError.message).toContain('query sentinel'); + expect(queryError.message).not.toContain(secret); + expect(String(queryError.cause ?? '')).not.toContain(secret); + expect(queryArgs).toHaveLength(1); + + const surfaces = { + errors: [loginError.message, queryError.message], command_args: ['standalone', 'run', 'live.eval.ts'], + stdout: '', stderr: '', report: { status: 'fail', diagnostic: queryError.message }, + snapshot: JSON.stringify(codexCalls.map(call => call.method)), debug: queryError.message, + }; + expect(JSON.stringify(surfaces)).not.toContain(secret); + }); + + it('exits zero with exactly two skipped paid cases when live flags are unset', () => { + const result = runGateChild({}); + + expect(result.status).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ + numFailedTests: 0, + numPendingTests: 2, + numPassedTests: 0, + }); + }); + + it('hard-fails protected mode for each missing retained-artifact prerequisite', () => { + const fixture = makeFakeTarball(); + const common = { + PATHGRADE_REQUIRE_LIVE_SMOKE: '1', + PATHGRADE_STANDALONE_LIVE_SMOKE: '1', + PATHGRADE_STANDALONE_LIVE_PROVIDER: 'codex', + OPENAI_API_KEY: 'pathgrade-gate-contract-key', + }; + + expectRejected(runGateChild({ PATHGRADE_REQUIRE_LIVE_SMOKE: '1' }), 'enable flag'); + expectRejected(runGateChild({ + ...common, + PATHGRADE_STANDALONE_TARBALL: undefined, + PATHGRADE_STANDALONE_TARBALL_SHA512: fixture.sha512, + }), 'retained tarball'); + expectRejected(runGateChild({ + ...common, + PATHGRADE_STANDALONE_TARBALL: fixture.filename, + PATHGRADE_STANDALONE_TARBALL_SHA512: undefined, + }), 'expected SHA-512'); + expectRejected(runGateChild({ + ...common, + PATHGRADE_STANDALONE_TARBALL: fixture.filename, + PATHGRADE_STANDALONE_TARBALL_SHA512: '0'.repeat(128), + }), 'SHA-512 mismatch'); + }); + + it('does not let one provider credential satisfy the other protected gate', () => { + const fixture = makeFakeTarball(); + const retained = { + PATHGRADE_REQUIRE_LIVE_SMOKE: '1', + PATHGRADE_STANDALONE_LIVE_SMOKE: '1', + PATHGRADE_STANDALONE_TARBALL: fixture.filename, + PATHGRADE_STANDALONE_TARBALL_SHA512: fixture.sha512, + }; + + expectRejected(runGateChild({ + ...retained, + PATHGRADE_STANDALONE_LIVE_PROVIDER: 'claude', + PATHGRADE_LIVE_GATE_MASK_CREDENTIAL: 'claude', + OPENAI_API_KEY: 'codex-only-key', + }), 'Claude credential'); + expectRejected(runGateChild({ + ...retained, + PATHGRADE_STANDALONE_LIVE_PROVIDER: 'codex', + PATHGRADE_LIVE_GATE_MASK_CREDENTIAL: 'codex', + ANTHROPIC_API_KEY: 'claude-only-key', + }), 'OPENAI_API_KEY'); + }); + + it('forwards only the selected provider base URL into the isolated live run', () => { + const installation = { + home: '/isolated/home', + codexHome: '/isolated/codex', + cache: '/isolated/cache', + temporary: '/isolated/tmp', + shims: '/isolated/shims', + debugDir: '/isolated/debug', + } as ReturnType; + const hostEnv = { + PATH: '/host/bin', + ANTHROPIC_BASE_URL: 'https://anthropic.example.test', + OPENAI_BASE_URL: 'https://openai.example.test/v1', + }; + const claude = liveEnvironment('claude', installation, 'claude-key', hostEnv); + const codex = liveEnvironment('codex', installation, 'codex-key', hostEnv); + expect(claude).toMatchObject({ + ANTHROPIC_API_KEY: 'claude-key', + ANTHROPIC_BASE_URL: 'https://anthropic.example.test', + }); + expect(claude.OPENAI_BASE_URL).toBe(''); + expect(codex).toMatchObject({ + OPENAI_API_KEY: 'codex-key', + OPENAI_BASE_URL: 'https://openai.example.test/v1', + }); + expect(codex.ANTHROPIC_BASE_URL).toBe(''); + }); +}); + +afterEach(() => { + cleanupTemporaryDirectories(); +}); + +afterAll(() => { + if (!gate.required && !process.env.PATHGRADE_LIVE_EVIDENCE_FILE) return; + const providers = [...results.values()]; + const summary = { + schema: 'pathgrade-live-evidence/v1', + status: providers.every(result => ( + !gate.selected.includes(result.provider) || (result.status === 'pass' && result.skipped === 0) + )) ? 'pass' : 'fail', + selected: gate.selected, + passed: providers.filter(result => result.status === 'pass').length, + skipped: providers.filter(result => result.skipped === 1).length, + providers, + tarball: gate.tarball, + tarball_sha512: gate.sha512, + source_commit: process.env.GITHUB_SHA ?? null, + runtime_environment: { + platform: process.platform, arch: process.arch, + node_major: Number(process.versions.node.split('.')[0]), + }, + }; + if (process.env.PATHGRADE_LIVE_EVIDENCE_FILE) { + fs.writeFileSync(process.env.PATHGRADE_LIVE_EVIDENCE_FILE, JSON.stringify(summary, null, 2)); + } + process.stdout.write(`PATHGRADE_LIVE_SUMMARY=${JSON.stringify(summary)}\n`); + if (gate.required && summary.status !== 'pass') process.exitCode = 1; +}); + +async function runLiveProvider(provider: Provider): Promise { + const secret = credential(provider); + const installation = createInstallation(provider); + const liveRun = invokeStandalone(installation, liveEnvironment(provider, installation, secret)); + try { + expect(liveRun.status, `${liveRun.stdout}\n${liveRun.stderr}`).toBe(0); + const reportPath = path.join(installation.target, '.pathgrade/results.json'); + const report = JSON.parse(fs.readFileSync(reportPath, 'utf8')); + assertReport(provider, report); + assertSecretAbsent(secret, liveRun, installation.target, installation.debugDir); + assertTargetIsolation(installation, secret); + expect(readShimCalls(installation.shims)).toBe(''); + results.set(provider, { + ...results.get(provider)!, + status: 'pass', + skipped: 0, + report, + }); + } catch (error) { + results.set(provider, { ...results.get(provider)!, status: 'fail', skipped: 0 }); + throw error; + } +} + +function createInstallation(provider: Provider) { + const prefix = makeTempDir('pathgrade-live-prefix-'); + const target = makeTempDir('pathgrade-live-target-'); + const home = makeTempDir('pathgrade-live-home-'); + const codexHome = makeTempDir('pathgrade-live-codex-home-'); + const cache = makeTempDir('pathgrade-live-cache-'); + const temporary = makeTempDir('pathgrade-live-tmp-'); + const debugDir = makeTempDir('pathgrade-live-debug-'); + const shims = makeTempDir('pathgrade-live-shims-'); + createFailingShims(shims); + const install = spawnSync('npm', [ + 'install', '--global', '--prefix', prefix, '--ignore-scripts', gate.tarball!, + ], { cwd: repoRoot, encoding: 'utf8', env: { ...process.env, npm_config_cache: cache } }); + expect(install.status, `${install.stdout}\n${install.stderr}`).toBe(0); + fs.writeFileSync(path.join(target, '.env'), 'PATHGRADE_LIVE_ENV_SENTINEL=must-not-load\n'); + fs.writeFileSync(path.join(target, 'live.eval.ts'), liveEvalSource(provider)); + const baseline = snapshotTree(target); + return { provider, prefix, target, home, codexHome, cache, temporary, debugDir, shims, baseline }; +} + +function liveEnvironment( + provider: Provider, + installation: ReturnType, + secret: string, + hostEnv: NodeJS.ProcessEnv = process.env, +) { + const env = { + ...withoutHostCredentials(hostEnv), + HOME: installation.home, + CODEX_HOME: installation.codexHome, + XDG_CACHE_HOME: installation.cache, + npm_config_cache: installation.cache, + TMPDIR: installation.temporary, + PATH: `${installation.shims}${path.delimiter}${hostEnv.PATH ?? ''}`, + PATHGRADE_LIVE_DEBUG_DIR: installation.debugDir, + PATHGRADE_LIVE_PROVIDER_CASE: provider, + }; + if (provider === 'claude') { + return { + ...env, + ANTHROPIC_API_KEY: secret, + ...(hostEnv.ANTHROPIC_BASE_URL ? { ANTHROPIC_BASE_URL: hostEnv.ANTHROPIC_BASE_URL } : {}), + }; + } + return { + ...env, + OPENAI_API_KEY: secret, + ...(hostEnv.OPENAI_BASE_URL ? { OPENAI_BASE_URL: hostEnv.OPENAI_BASE_URL } : {}), + }; +} + +function invokeStandalone(installation: ReturnType, env: NodeJS.ProcessEnv) { + return spawnSync( + path.join(installation.prefix, 'bin', 'pathgrade'), + ['standalone', 'run', 'live.eval.ts'], + { cwd: installation.target, env, encoding: 'utf8', timeout: 300_000 }, + ); +} + +function assertReport(provider: Provider, report: any) { + expect(report).toMatchObject({ + version: 1, + status: 'pass', + provenance: { + mode: 'standalone', + runtimes: { + claude: { sdk_version: '0.2.116', claude_code_version: '2.1.116' }, + codex: { package_version: '0.144.0', native_version: '0.144.0' }, + }, + }, + }); + expect(report.groups).toHaveLength(1); + expect(report.groups[0].trials).toHaveLength(1); + expect(report.groups[0].trials[0].reward).toBe(1); + const provenance = report.groups[0].trials[0].agent_provenance; + if (provider === 'claude') { + expect(provenance).toEqual({ + agent: 'claude', + transport: 'native', + model: { id: null, source: 'provider-default' }, + authentication: 'api-key', + runtime: { + package: '@anthropic-ai/claude-agent-sdk', + package_version: '0.2.116', + embedded_binary_version: '2.1.116', + provenance: 'bundled', + }, + }); + } else { + expect(provenance).toEqual({ + agent: 'codex', + transport: 'app-server', + model: { id: 'gpt-5.4', source: 'pathgrade-default' }, + authentication: 'api-key', + runtime: { + package: '@openai/codex', + package_version: '0.144.0', + embedded_binary_version: '0.144.0', + provenance: 'bundled', + }, + }); + } +} + +function assertSecretAbsent(secret: string, run: ReturnType, target: string, debugDir: string) { + const output = `${run.error?.message ?? ''}\n${run.stdout}\n${run.stderr}`; + expect(output).not.toContain(secret); + for (const root of [path.join(target, '.pathgrade'), debugDir]) { + if (!fs.existsSync(root)) continue; + for (const filename of filesBelow(root)) { + expect(fs.readFileSync(filename, 'utf8')).not.toContain(secret); + } + } +} + +function assertTargetIsolation(installation: ReturnType, secret: string) { + const { target, baseline, provider, debugDir } = installation; + const expectedCreated = new Map([ + ['.pathgrade', { type: 'directory' }], + ['.pathgrade/.gitignore', { type: 'file' }], + ['.pathgrade/results.json', { type: 'file' }], + ['.pathgrade/traces', { type: 'directory' }], + [`.pathgrade/traces/live-eval-ts-packed-${provider}-live-smoke.json`, { type: 'file' }], + ]); + const createdFiles = assertExactTargetTree(target, baseline, expectedCreated); + for (const filename of [...createdFiles, ...filesBelow(debugDir)]) { + const contents = fs.readFileSync(filename, 'utf8'); + expect(contents, `${filename} leaked the credential`).not.toContain(secret); + expect(contents, `${filename} loaded the target .env`).not.toContain('must-not-load'); + } +} + +type TreeEntry = + | { type: 'file'; sha512?: string } + | { type: 'directory' } + | { type: 'symlink'; target: string } + | { type: 'socket' } + | { type: 'other' }; + +function snapshotTree(directory: string) { + const entries = new Map(); + const visit = (current: string) => { + for (const name of fs.readdirSync(current)) { + const filename = path.join(current, name); + const relative = path.relative(directory, filename); + const stat = fs.lstatSync(filename); + if (stat.isFile()) entries.set(relative, { type: 'file', sha512: sha512File(filename) }); + else if (stat.isDirectory()) { + entries.set(relative, { type: 'directory' }); + visit(filename); + } else if (stat.isSymbolicLink()) entries.set(relative, { type: 'symlink', target: fs.readlinkSync(filename) }); + else if (stat.isSocket()) entries.set(relative, { type: 'socket' }); + else entries.set(relative, { type: 'other' }); + } + }; + visit(directory); + return entries; +} + +function assertExactTargetTree( + target: string, + baseline: Map, + expectedCreated: Map, +) { + const after = snapshotTree(target); + for (const [filename, entry] of baseline) expect(after.get(filename), `${filename} changed`).toEqual(entry); + const created = new Map([...after].filter(([filename]) => !baseline.has(filename))); + expect([...created.keys()].sort(), 'target contains unexpected entries').toEqual([...expectedCreated.keys()].sort()); + for (const [filename, expected] of expectedCreated) { + const actual = created.get(filename); + expect(actual?.type, `${filename} has unexpected entry type`).toBe(expected.type); + if (expected.type === 'symlink') expect(actual).toEqual(expected); + } + return [...created] + .filter(([, entry]) => entry.type === 'file') + .map(([filename]) => path.join(target, filename)); +} + +function liveEvalSource(provider: Provider) { + return `import { describe, expect, it } from 'vitest'; +import { check, createAgent, evaluate } from '@wix/pathgrade'; +describe('packed ${provider} live smoke', () => { + it('completes one paid turn', async () => { + expect(process.env.PATHGRADE_LIVE_ENV_SENTINEL).toBeUndefined(); + const agent = await createAgent({ agent: '${provider}', ${provider === 'codex' ? "transport: 'app-server', " : ''}timeout: 180, debug: process.env.PATHGRADE_LIVE_DEBUG_DIR }); + const response = await agent.prompt('Reply briefly with the word ready. Do not use tools.'); + const result = await evaluate(agent, [check('provider returned a response', () => response.trim().length > 0)]); + expect(result.score).toBe(1); + }); +}); +`; +} + +function readGate(env: NodeJS.ProcessEnv) { + const enabled = env.PATHGRADE_STANDALONE_LIVE_SMOKE === '1'; + const required = env.PATHGRADE_REQUIRE_LIVE_SMOKE === '1'; + const provider = env.PATHGRADE_STANDALONE_LIVE_PROVIDER || 'all'; + if (provider !== 'all' && provider !== 'claude' && provider !== 'codex') { + throw new Error(`PATHGRADE_STANDALONE_LIVE_PROVIDER must be claude, codex, or all; got ${provider}`); + } + const selected: Provider[] = provider === 'all' ? ['claude', 'codex'] : [provider]; + if (required && !enabled) throw new Error('protected live smoke requires enable flag PATHGRADE_STANDALONE_LIVE_SMOKE=1'); + const maskedCredential = env.PATHGRADE_LIVE_GATE_MASK_CREDENTIAL; + if (required && selected.includes('claude') && (maskedCredential === 'claude' || !hasCredential(env.ANTHROPIC_API_KEY))) { + throw new Error('protected Claude credential ANTHROPIC_API_KEY is missing'); + } + if (required && selected.includes('codex') && (maskedCredential === 'codex' || !hasCredential(env.OPENAI_API_KEY))) { + throw new Error('protected Codex credential OPENAI_API_KEY is missing'); + } + const tarball = env.PATHGRADE_STANDALONE_TARBALL; + const sha512 = env.PATHGRADE_STANDALONE_TARBALL_SHA512; + if (required && (!tarball || !path.isAbsolute(tarball) || !tarball.endsWith('.tgz') || !fs.existsSync(tarball))) { + throw new Error('protected live smoke retained tarball must be an existing absolute .tgz path'); + } + if (required && (!sha512 || !/^[a-f\d]{128}$/i.test(sha512))) { + throw new Error('protected live smoke expected SHA-512 is missing or invalid'); + } + if (required && sha512File(tarball!) !== sha512) throw new Error('protected live smoke tarball SHA-512 mismatch'); + return { enabled, required, selected, tarball, sha512 }; +} + +function shouldRun(provider: Provider) { + if (!gate.enabled || !gate.selected.includes(provider)) return false; + if (!gate.required) { + if (!gate.tarball || !gate.sha512 || !path.isAbsolute(gate.tarball) || !fs.existsSync(gate.tarball)) return false; + if (provider === 'claude' && !hasCredential(process.env.ANTHROPIC_API_KEY)) return false; + if (provider === 'codex' && !hasCredential(process.env.OPENAI_API_KEY)) return false; + } + return true; +} + +function credential(provider: Provider) { + return provider === 'claude' ? process.env.ANTHROPIC_API_KEY! : process.env.OPENAI_API_KEY!; +} + +function runGateChild(overrides: Record) { + const env = withoutLiveEnvironment(process.env); + for (const [key, value] of Object.entries(overrides)) { + if (value === undefined) env[key] = ''; + else env[key] = value; + } + env.PATHGRADE_LIVE_GATE_CHILD = '1'; + return spawnSync(process.execPath, [ + path.join(repoRoot, 'node_modules/vitest/vitest.mjs'), + 'run', path.join(repoRoot, 'tests/standalone-live-smoke.test.ts'), '--reporter=json', '--silent', + ], { cwd: repoRoot, env, encoding: 'utf8', timeout: 60_000 }); +} + +function expectRejected(result: ReturnType, diagnostic: string) { + expect(result.status).not.toBe(0); + expect(`${result.stdout}\n${result.stderr}`).toContain(diagnostic); +} + +function makeFakeTarball() { + const directory = makeTempDir('pathgrade-live-gate-'); + const filename = path.join(directory, 'retained.tgz'); + fs.writeFileSync(filename, 'gate contract only'); + return { filename, sha512: sha512File(filename) }; +} + +function withoutLiveEnvironment(env: NodeJS.ProcessEnv) { + const result = withoutHostCredentials(env); + for (const key of Object.keys(result)) { + if (key.startsWith('PATHGRADE_STANDALONE_LIVE_') || key.startsWith('PATHGRADE_REQUIRE_LIVE_') || key === 'PATHGRADE_LIVE_EVIDENCE_FILE') { + delete result[key]; + } + } + for (const key of [ + 'PATHGRADE_STANDALONE_LIVE_SMOKE', + 'PATHGRADE_REQUIRE_LIVE_SMOKE', + 'PATHGRADE_STANDALONE_LIVE_PROVIDER', + 'PATHGRADE_STANDALONE_TARBALL', + 'PATHGRADE_STANDALONE_TARBALL_SHA512', + 'PATHGRADE_LIVE_EVIDENCE_FILE', + ]) result[key] = ''; + return result; +} + +function withoutHostCredentials(env: NodeJS.ProcessEnv) { + const result = { ...env }; + for (const key of ['ANTHROPIC_API_KEY', 'OPENAI_API_KEY']) result[key] = missingCredential; + for (const key of [ + 'ANTHROPIC_AUTH_TOKEN', 'CLAUDE_CODE_OAUTH_TOKEN', 'ANTHROPIC_BASE_URL', + 'OPENAI_BASE_URL', 'CURSOR_API_KEY', 'CURSOR_API_BASE_URL', + 'PATHGRADE_CLAUDE_CODE_EXECUTABLE', 'PATHGRADE_CODEX_TRANSPORT', + ]) result[key] = ''; + return result; +} + +function hasCredential(value: string | undefined) { + return Boolean(value && value !== missingCredential); +} + +function createFailingShims(directory: string) { + for (const command of ['claude', 'codex', 'vitest']) { + const filename = path.join(directory, command); + fs.writeFileSync(filename, `#!/bin/sh\necho ${command} >> "${path.join(directory, 'calls')}"\nexit 97\n`); + fs.chmodSync(filename, 0o755); + } +} + +function readShimCalls(directory: string) { + const filename = path.join(directory, 'calls'); + return fs.existsSync(filename) ? fs.readFileSync(filename, 'utf8') : ''; +} + +function filesBelow(directory: string): string[] { + return fs.readdirSync(directory, { recursive: true, withFileTypes: true }) + .filter(entry => entry.isFile()) + .map(entry => path.join(entry.parentPath, entry.name)); +} + +function sha512File(filename: string) { + return createHash('sha512').update(fs.readFileSync(filename)).digest('hex'); +} + +function makeTempDir(prefix: string) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + temporaryDirectories.push(directory); + return directory; +} + +function cleanupTemporaryDirectories() { + for (const directory of temporaryDirectories.splice(0).reverse()) { + fs.rmSync(directory, { recursive: true, force: true }); + } +} diff --git a/tests/standalone-mode.test.ts b/tests/standalone-mode.test.ts new file mode 100644 index 0000000..581ebac --- /dev/null +++ b/tests/standalone-mode.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'vitest'; +import { + isStandaloneMode, + parseStandaloneCommand, + PATHGRADE_STANDALONE_ENV, +} from '../src/standalone/mode.js'; + +describe('standalone mode', () => { + it('uses an exact internal marker', () => { + expect(PATHGRADE_STANDALONE_ENV).toBe('PATHGRADE_STANDALONE'); + expect(isStandaloneMode({ PATHGRADE_STANDALONE: '1' })).toBe(true); + expect(isStandaloneMode({ PATHGRADE_STANDALONE: 'true' })).toBe(false); + expect(isStandaloneMode({})).toBe(false); + }); + + it('selects standalone only through the scoped command namespace', () => { + expect(parseStandaloneCommand(['run'])).toEqual({ + standalone: false, + args: ['run'], + }); + expect(parseStandaloneCommand(['standalone'])).toEqual({ + standalone: true, + args: ['run'], + }); + expect(parseStandaloneCommand(['standalone', 'run', 'example.eval.ts'])).toEqual({ + standalone: true, + args: ['run', 'example.eval.ts'], + }); + expect(parseStandaloneCommand(['standalone', 'affected', '--json'])).toEqual({ + standalone: true, + args: ['affected', '--json'], + }); + }); +}); diff --git a/tests/standalone-module-aliases.test.ts b/tests/standalone-module-aliases.test.ts new file mode 100644 index 0000000..0b22225 --- /dev/null +++ b/tests/standalone-module-aliases.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from 'vitest'; +import path from 'node:path'; +import { + decodeStandaloneVitestPayload, + encodeStandaloneVitestPayload, + resolveStandaloneModuleAliases, + selectStandaloneEsmExportTarget, +} from '../src/standalone/module-aliases.js'; + +describe('standalone Vitest aliases and payload', () => { + it('round-trips only validated payload fields', () => { + const payload = { + root: '/tmp/target', + include: ['**/*.eval.ts'], + exclude: ['**/fixtures/**'], + diagnostics: true, + reporter: 'cli' as const, + threshold: 0.8, + cacheDir: '/tmp/pathgrade-cache', + }; + expect(decodeStandaloneVitestPayload(encodeStandaloneVitestPayload(payload))) + .toEqual(payload); + }); + + it('rejects malformed or unsafe payloads', () => { + expect(() => decodeStandaloneVitestPayload(undefined)).toThrow(); + expect(() => decodeStandaloneVitestPayload(Buffer.from(JSON.stringify({ + root: 'relative', + include: [], + exclude: [], + diagnostics: false, + cacheDir: '/tmp/cache', + })).toString('base64url'))).toThrow(/absolute/); + }); + + it('maps root Vitest plus supported scoped exports exactly', () => { + const aliases = resolveStandaloneModuleAliases( + path.resolve('.'), + '/tool/node_modules/vitest/dist/index.js', + ); + expect(aliases).toContainEqual({ + find: /^@wix\/pathgrade$/, + replacement: path.resolve('dist/sdk/index.js'), + }); + expect(aliases).toContainEqual({ + find: /^@wix\/pathgrade\/mcp-mock$/, + replacement: path.resolve('dist/core/mcp-mock.js'), + }); + expect(aliases).toContainEqual({ + find: /^vitest$/, + replacement: '/tool/node_modules/vitest/dist/index.js', + }); + expect(aliases.map(alias => alias.find.source)).not.toContain('vitest\\/config'); + expect(aliases).not.toContainEqual(expect.objectContaining({ find: /^pathgrade$/ })); + }); + + it('selects standalone ESM targets without requiring a default condition', () => { + expect(selectStandaloneEsmExportTarget('./dist/sdk/index.js')) + .toBe('./dist/sdk/index.js'); + expect(selectStandaloneEsmExportTarget({ + types: './dist/sdk/index.d.ts', + import: './dist/sdk/index.js', + })).toBe('./dist/sdk/index.js'); + expect(selectStandaloneEsmExportTarget({ + node: './dist/sdk/node.js', + default: './dist/sdk/browser.js', + })).toBe('./dist/sdk/node.js'); + expect(selectStandaloneEsmExportTarget({ + import: { default: './dist/sdk/nested.js' }, + })).toBe('./dist/sdk/nested.js'); + expect(selectStandaloneEsmExportTarget({ + require: './dist/sdk/index.cjs', + })).toBeUndefined(); + }); + + it('does not alias non-eval public entries', () => { + const sources = resolveStandaloneModuleAliases( + path.resolve('.'), + '/tool/node_modules/vitest/dist/index.js', + ).map(alias => alias.find.source); + expect(sources).not.toContain('@wix\\/pathgrade\\/package\\.json'); + expect(sources).not.toContain('@wix\\/pathgrade\\/adapters\\/jest\\/reporter'); + }); +}); diff --git a/tests/standalone-package-smoke/run-smoke.mjs b/tests/standalone-package-smoke/run-smoke.mjs new file mode 100644 index 0000000..2398439 --- /dev/null +++ b/tests/standalone-package-smoke/run-smoke.mjs @@ -0,0 +1,221 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const fixtureRoot = path.join(repoRoot, 'tests/fixtures/standalone-package'); +const options = parseArgs(process.argv.slice(2)); +const temporaryDirectories = []; + +try { + const artifact = resolveArtifact(options); + const shims = makeTempDir('pathgrade-standalone-shims-'); + createFailingShims(shims); + const firstPrefix = installTarball(artifact.tarball, shims); + + for (const manifest of [false, true]) { + runSuccessfulCases({ artifact, prefix: firstPrefix, shims, manifest, command: 'npx' }); + runSuccessfulCases({ artifact, prefix: firstPrefix, shims, manifest, command: 'global' }); + } + runMissingDependency({ artifact, prefix: firstPrefix, shims }); + runPackagingDefect({ artifact, shims }); + assert.equal(readShimCalls(shims), '', 'standalone must not invoke PATH shims'); + process.stdout.write(`${JSON.stringify({ tarball: artifact.tarball, sha512: artifact.sha512 })}\n`); +} finally { + for (const directory of temporaryDirectories.reverse()) { + fs.rmSync(directory, { recursive: true, force: true }); + } +} + +function parseArgs(args) { + const result = {}; + for (let index = 0; index < args.length; index += 1) { + const option = args[index]; + if (option !== '--tarball' && option !== '--expected-sha512') { + throw new Error(`unknown argument: ${option}`); + } + const value = args[++index]; + if (!value) throw new Error(`missing value for ${option}`); + result[option.slice(2).replace('-', '')] = value; + } + if (result.tarball && !path.isAbsolute(result.tarball)) { + throw new Error('--tarball must be an absolute path'); + } + return result; +} + +function resolveArtifact(options) { + if (options.tarball) { + assert.ok(fs.existsSync(options.tarball), `tarball does not exist: ${options.tarball}`); + assert.ok(options.tarball.endsWith('.tgz'), '--tarball must name a .tgz file'); + const sha512 = sha512File(options.tarball); + if (options.expectedsha512) assert.equal(sha512, options.expectedsha512, 'tarball SHA-512'); + assertPackedSurface(options.tarball); + return { tarball: options.tarball, sha512 }; + } + run('yarn', ['build'], repoRoot); + const artifacts = makeTempDir('pathgrade-standalone-artifact-'); + const packed = run('npm', ['pack', '--json', '--pack-destination', artifacts], repoRoot); + const parsed = JSON.parse(packed.stdout); + assert.equal(parsed.length, 1, 'npm pack should create exactly one artifact'); + const tarball = path.join(artifacts, parsed[0].filename); + assertPackedSurface(tarball); + return { tarball, sha512: sha512File(tarball) }; +} + +function runSuccessfulCases({ artifact, prefix, shims, manifest, command }) { + const target = createTarget(manifest); + const state = path.join(makeTempDir('pathgrade-standalone-state-'), 'lifecycle.txt'); + const changed = path.join(target, 'changed-files.txt'); + fs.writeFileSync(changed, 'basic.eval.ts\n'); + const baseEnv = smokeEnv({ + shims, + state, + ...(command === 'global' ? { prefix } : {}), + }); + const invoke = (args) => command === 'npx' + ? run('npx', ['--yes', '--package', artifact.tarball, 'pathgrade', ...args], target, baseEnv) + : run(path.join(prefix, 'bin', 'pathgrade'), args, target, baseEnv); + + invoke(['standalone', 'run', 'basic.eval.ts']); + assertSuccessfulTarget(target, state, artifact, manifest); + resetRunState(target, state); + invoke(['standalone']); + assertSuccessfulTarget(target, state, artifact, manifest); + resetRunState(target, state); + invoke(['standalone', 'run', '--changed', '--changed-files=changed-files.txt']); + assertSuccessfulTarget(target, state, artifact, manifest); + resetRunState(target, state); + const affected = invoke(['standalone', 'affected', '--changed-files=changed-files.txt']); + assert.match(affected.stdout, /basic\.eval\.ts/); + assertTargetShape(target, manifest); +} + +function runMissingDependency({ artifact, prefix, shims }) { + const target = createTarget(false, { missing: true }); + const result = runResult(path.join(prefix, 'bin', 'pathgrade'), ['standalone', 'run', 'missing-dependency.eval.ts'], target, smokeEnv({ shims, prefix })); + assert.notEqual(result.status, 0, 'missing project dependency should fail'); + assert.match(`${result.stdout}\n${result.stderr}`, /project dependency "package-that-pathgrade-must-not-supply" is unavailable/); + assert.match(`${result.stdout}\n${result.stderr}`, /missing-dependency\.eval\.ts/); +} + +function runPackagingDefect({ artifact, shims }) { + const prefix = installTarball(artifact.tarball, shims); + const installedVitest = path.join(prefix, 'lib/node_modules/@wix/pathgrade/node_modules/vitest'); + assert.ok(fs.existsSync(installedVitest), 'temporary install must contain bundled Vitest'); + fs.rmSync(installedVitest, { recursive: true, force: true }); + const target = createTarget(false); + const result = runResult(path.join(prefix, 'bin', 'pathgrade'), ['standalone', 'run', 'basic.eval.ts'], target, smokeEnv({ shims, prefix })); + assert.notEqual(result.status, 0, 'installed tool graph defect should fail'); + assert.match(`${result.stdout}\n${result.stderr}`, /bundled Vitest 4\.1\.7 could not be resolved; this is a Pathgrade packaging defect/); +} + +function assertSuccessfulTarget(target, state, artifact, manifest) { + assert.deepEqual(fs.readFileSync(state, 'utf8').trim().split('\n').sort(), ['agent-disposed', 'evaluation-scored', 'evaluation-started']); + const report = JSON.parse(fs.readFileSync(path.join(target, '.pathgrade/results.json'), 'utf8')); + assert.equal(report.version, 1); + assert.equal(report.status, 'pass'); + assert.equal(report.groups.length, 1); + assert.equal(report.groups[0].trials.length, 1); + assert.equal(report.groups[0].trials[0].reward, 1); + assert.equal(report.provenance.mode, 'standalone'); + assert.equal(report.provenance.package_version, packageVersion()); + assert.equal(report.provenance.vitest_version, '4.1.7'); + assert.deepEqual(report.provenance.runtimes, { + claude: { sdk_version: '0.2.116', claude_code_version: '2.1.116' }, + codex: { package_version: '0.144.0', native_version: '0.144.0' }, + }); + assertTargetShape(target, manifest); + assert.ok(artifact.sha512.length === 128); +} + +function assertTargetShape(target, manifest) { + assert.equal(fs.existsSync(path.join(target, 'node_modules')), false, 'target must not gain node_modules'); + for (const name of ['.vite', 'coverage', 'attachments', 'blob-reports', '.vitest']) { + assert.equal(fs.existsSync(path.join(target, name)), false, `target must not gain ${name}`); + } + assert.equal(fs.existsSync(path.join(target, 'package.json')), manifest); +} + +function resetRunState(target, state) { + fs.rmSync(path.join(target, '.pathgrade'), { recursive: true, force: true }); + fs.rmSync(state, { force: true }); +} + +function createTarget(manifest, options = {}) { + const target = makeTempDir('pathgrade-standalone-target-'); + for (const name of ['basic.eval.ts', 'vitest.config.ts', '.env']) { + fs.copyFileSync(path.join(fixtureRoot, name), path.join(target, name)); + } + if (manifest) fs.copyFileSync(path.join(fixtureRoot, 'package.json'), path.join(target, 'package.json')); + if (options.missing) fs.copyFileSync(path.join(fixtureRoot, 'missing-dependency.eval.ts'), path.join(target, 'missing-dependency.eval.ts')); + return target; +} + +function installTarball(tarball, shims) { + const prefix = makeTempDir('pathgrade-standalone-prefix-'); + run('npm', ['install', '--global', '--prefix', prefix, '--ignore-scripts', tarball], repoRoot, smokeEnv({ shims, prefix })); + return prefix; +} + +function smokeEnv({ shims, state, prefix }) { + return { + ...process.env, + PATH: `${shims}${path.delimiter}${process.env.PATH ?? ''}`, + ...(state ? { PATHGRADE_STANDALONE_SMOKE_STATE_FILE: state } : {}), + ...(prefix ? { npm_config_prefix: prefix } : {}), + }; +} + +function createFailingShims(directory) { + for (const command of ['vitest', 'codex', 'claude']) { + const filename = path.join(directory, command); + fs.writeFileSync(filename, `#!/bin/sh\necho ${command} >> "${path.join(directory, 'calls')}"\nexit 97\n`); + fs.chmodSync(filename, 0o755); + } +} + +function readShimCalls(directory) { + const calls = path.join(directory, 'calls'); + return fs.existsSync(calls) ? fs.readFileSync(calls, 'utf8') : ''; +} + +function packageVersion() { + return JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8')).version; +} + +function assertPackedSurface(tarball) { + const entries = run('tar', ['-tzf', tarball], repoRoot).stdout.split('\n'); + for (const entry of [ + 'package/bin/pathgrade.js', + 'package/dist/pathgrade.js', + 'package/dist/sdk/index.js', + 'package/dist/sdk/index.d.ts', + ]) { + assert.ok(entries.includes(entry), `packed artifact is missing ${entry}`); + } +} + +function sha512File(filename) { + return createHash('sha512').update(fs.readFileSync(filename)).digest('hex'); +} + +function makeTempDir(prefix) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + temporaryDirectories.push(directory); + return directory; +} + +function run(command, args, cwd, env = process.env) { + const result = runResult(command, args, cwd, env); + assert.equal(result.status, 0, `${command} ${args.join(' ')} failed:\n${result.stdout}\n${result.stderr}`); + return result; +} + +function runResult(command, args, cwd, env) { + return spawnSync(command, args, { cwd, env, encoding: 'utf8' }); +} diff --git a/tests/standalone-provenance-no-codex-launch.test.ts b/tests/standalone-provenance-no-codex-launch.test.ts new file mode 100644 index 0000000..7b67de1 --- /dev/null +++ b/tests/standalone-provenance-no-codex-launch.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it, vi } from 'vitest'; + +vi.mock('node:child_process', async importOriginal => { + const actual = await importOriginal(); + return { + ...actual, + spawn: ((command: string, args?: readonly string[], options?: object) => { + const runsCodexLauncher = args?.some(arg => arg.includes('/@openai/codex/bin/codex.js')); + const runsCodexNative = command.endsWith('/codex') + && command.includes('/@openai/codex-'); + if (runsCodexLauncher || runsCodexNative) { + throw new Error('standalone run provenance executed bundled Codex'); + } + return actual.spawn(command, args, options); + }) as typeof actual.spawn, + }; +}); + +import { buildStandaloneRunProvenance } from '../src/standalone/provenance.js'; + +describe('standalone run provenance collection', () => { + it('does not execute bundled Codex before a Codex agent is selected', async () => { + await expect(buildStandaloneRunProvenance()).resolves.toMatchObject({ + runtimes: { + codex: { + package_version: '0.144.0', + native_version: '0.144.0', + }, + }, + }); + }); +}); diff --git a/tests/standalone-provenance.test.ts b/tests/standalone-provenance.test.ts new file mode 100644 index 0000000..ac5a78e --- /dev/null +++ b/tests/standalone-provenance.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from 'vitest'; +import { + buildStandaloneRunProvenance, + encodeStandaloneRunProvenance, + readStandaloneRunProvenance, + STANDALONE_PROVENANCE_ENV, +} from '../src/standalone/provenance.js'; +import { resolveCredentials } from '../src/providers/credentials.js'; + +describe('standalone run provenance', () => { + it('builds the exact bundled package and platform manifest', async () => { + await expect(buildStandaloneRunProvenance()).resolves.toMatchObject({ + mode: 'standalone', + package_version: '1.0.1', + vitest_version: '4.1.7', + runtimes: { + claude: { + sdk_version: '0.2.116', + claude_code_version: '2.1.116', + }, + codex: { + package_version: '0.144.0', + native_version: '0.144.0', + }, + }, + platform: { + os: process.platform, + arch: process.arch, + node: process.version, + }, + }); + }); + + it('round-trips base64url provenance and rejects malformed worker values', async () => { + const provenance = await buildStandaloneRunProvenance(); + const encoded = encodeStandaloneRunProvenance(provenance); + + expect(STANDALONE_PROVENANCE_ENV).toBe('PATHGRADE_STANDALONE_PROVENANCE'); + expect(readStandaloneRunProvenance({ [STANDALONE_PROVENANCE_ENV]: encoded })).toEqual(provenance); + expect(() => readStandaloneRunProvenance({ + [STANDALONE_PROVENANCE_ENV]: `${encoded}!`, + })).toThrow(/pathgrade standalone: invalid provenance payload; this is a Pathgrade packaging defect/); + expect(() => readStandaloneRunProvenance({ [STANDALONE_PROVENANCE_ENV]: 'not-provenance' })).toThrow( + /pathgrade standalone: invalid provenance payload; this is a Pathgrade packaging defect/, + ); + }); + + it('accepts the workspace Claude OAuth marker in standalone mode', async () => { + await expect(resolveCredentials( + 'claude', + { PATHGRADE_CLAUDE_LOCAL_OAUTH: '1' }, + { + hostEnv: key => key === 'ANTHROPIC_API_KEY' ? 'host-api-key' : undefined, + platform: 'linux', + homedir: () => '/tmp', + readKeychainToken: async () => undefined, + keychainEntryExists: async () => false, + fileExists: async () => false, + }, + { mode: 'standalone' }, + )).resolves.toEqual({ env: {}, setupCommands: [], copyFromHome: [] }); + }); +}); diff --git a/tests/standalone-run-changed.test.ts b/tests/standalone-run-changed.test.ts new file mode 100644 index 0000000..f6beba2 --- /dev/null +++ b/tests/standalone-run-changed.test.ts @@ -0,0 +1,52 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { expect, it, vi } from 'vitest'; + +vi.mock('../src/affected/git.js', () => ({ + resolveBaseRef: vi.fn(() => ({ base: 'origin/main', sha: 'abc1234' })), + computeChangedFiles: vi.fn(() => ['skills/alpha/src/x.ts']), +})); + +import { runChanged } from '../src/commands/run-changed.js'; +import type { RunnerInvocationAdapter } from '../src/runners/invocation.js'; + +it('standalone changed runs ignore legacy Vitest config', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'pathgrade-standalone-changed-')); + fs.mkdirSync(path.join(root, 'skills/alpha'), { recursive: true }); + fs.writeFileSync(path.join(root, 'skills/alpha/SKILL.md'), '# alpha'); + fs.writeFileSync( + path.join(root, 'skills/alpha/a.eval.ts'), + "import { createAgent } from '@wix/pathgrade';\nvoid createAgent;\n", + ); + fs.writeFileSync(path.join(root, 'vitest.config.ts'), ` +export default { + plugins: [{ + name: 'pathgrade', + __pathgradeOptions: { include: ['legacy/**/*.eval.ts'] }, + }], +}; +`); + const runnerInvocation: RunnerInvocationAdapter = { + name: 'vitest', + run: vi.fn(async () => 0), + }; + + const code = await runChanged({ + cwd: root, + standalone: true, + parsed: { + runnerArgs: [], + forceDiagnostics: false, + forceVerbose: false, + changed: true, + quiet: true, + }, + runnerInvocation, + }); + + expect(code).toBe(0); + expect(runnerInvocation.run).toHaveBeenCalledWith(expect.objectContaining({ + selectedFiles: ['skills/alpha/a.eval.ts'], + })); +}); diff --git a/tests/standalone-ui.test.ts b/tests/standalone-ui.test.ts new file mode 100644 index 0000000..28ed55a --- /dev/null +++ b/tests/standalone-ui.test.ts @@ -0,0 +1,150 @@ +import { PassThrough } from 'node:stream'; +import { describe, expect, it, vi } from 'vitest'; +import { CalmRenderer } from '../src/standalone/ui/calm-renderer.js'; +import { displayWidth, stripAnsi, truncateEnd, truncateMiddle } from '../src/standalone/ui/format.js'; +import { StandaloneOutputController } from '../src/standalone/ui/output-controller.js'; +import { StandaloneUiDecoder } from '../src/standalone/ui/protocol.js'; +import { createStandaloneTheme } from '../src/standalone/ui/theme.js'; +import type { StandaloneUiEvent } from '../src/standalone/ui/events.js'; + +function outputStream(columns = 80, isTTY = false): PassThrough & { + columns: number; + isTTY: boolean; + getColorDepth: (env?: NodeJS.ProcessEnv) => number; +} { + const stream = new PassThrough() as ReturnType; + stream.columns = columns; + stream.isTTY = isTTY; + stream.getColorDepth = env => env?.FORCE_COLOR === '0' ? 1 : 8; + return stream; +} + +function read(stream: PassThrough): string { + return stream.read()?.toString() ?? ''; +} + +describe('standalone UI formatting', () => { + it('truncates ANSI, emoji, and CJK content without exceeding the width', () => { + const end = truncateEnd('\x1b[31m報告🙂-a-very-long-case\x1b[0m', 12); + const middle = truncateMiddle('fixtures/報告🙂/authentication.eval.ts', 16); + expect(displayWidth(end)).toBeLessThanOrEqual(12); + expect(displayWidth(middle)).toBeLessThanOrEqual(16); + expect(end).not.toContain('\x1b'); + expect(middle).toContain('…'); + }); + + it.each([40, 80, 120])('keeps static output within %i columns', width => { + const stream = outputStream(width); + const renderer = new CalmRenderer({ stream, env: { FORCE_COLOR: '0' }, now: () => 2_000 }); + renderer.handle({ v: 1, type: 'run_start', files: [{ id: 'f', name: 'f' }], startedAt: 0 }); + renderer.handle({ v: 1, type: 'case_start', id: 'c', file: 'a/very/long/報告.eval.ts', name: 'rejects a very long leaked credential name', startedAt: 0 }); + renderer.handle({ v: 1, type: 'case_finish', id: 'c', file: 'a/very/long/報告.eval.ts', name: 'rejects a very long leaked credential name', state: 'passed', durationMs: 2_000, score: 1 }); + renderer.handle({ v: 1, type: 'run_finish', status: 'pass', fileCount: 1, passed: 1, failed: 0, skipped: 0, durationMs: 2_000, overallScore: 1, resultsPath: '.pathgrade/results.json' }); + const lines = read(stream).split('\n').filter(Boolean); + expect(Math.max(...lines.map(line => displayWidth(stripAnsi(line))))).toBeLessThanOrEqual(Math.max(width, 40)); + expect(lines.join('\n')).toContain('PASS 1 file · 1 passed · 2s'); + }); + + it('does not fabricate a score when evaluate was not called', () => { + const stream = outputStream(); + const renderer = new CalmRenderer({ stream, env: { FORCE_COLOR: '0' } }); + renderer.handle({ v: 1, type: 'case_finish', id: 'c', file: 'compat.eval.ts', name: 'legacy pass', state: 'passed', durationMs: 10 }); + expect(read(stream)).not.toContain('score'); + }); + + it('uses words instead of decorative status symbols outside a TTY', () => { + const stream = outputStream(); + const renderer = new CalmRenderer({ stream, env: { FORCE_COLOR: '0' } }); + renderer.handle({ v: 1, type: 'case_finish', id: 'c', file: 'x.eval.ts', name: 'works', state: 'passed', durationMs: 10, score: 1 }); + expect(read(stream)).toMatch(/^PASS x\.eval\.ts/); + }); + + it('bounds concurrent active rows and restores the cursor', () => { + const stream = outputStream(80, true); + const renderer = new CalmRenderer({ stream, env: { FORCE_COLOR: '0' }, now: () => 1_000 }); + for (let index = 0; index < 7; index++) { + renderer.handle({ + v: 1, + type: 'case_start', + id: String(index), + file: `case-${index}.eval.ts`, + name: `trial ${index}`, + startedAt: 0, + }); + } + renderer.dispose(); + const output = read(stream); + expect(output).toContain('and 1 more'); + expect(output).toContain('\x1b[?25l'); + expect(output).toContain('\x1b[?25h'); + }); +}); + +describe('standalone UI protocol', () => { + it('decodes split and combined NDJSON frames', () => { + const events: StandaloneUiEvent[] = []; + const invalid = vi.fn(); + const decoder = new StandaloneUiDecoder(event => events.push(event), invalid); + decoder.push('{"v":1,"type":"run_error"}\n{"v":'); + decoder.push('1,"type":"run_error"}\n'); + decoder.end(); + expect(events).toHaveLength(2); + expect(invalid).not.toHaveBeenCalled(); + }); + + it.each([ + '{not json}\n', + '{"v":2,"type":"run_error"}\n', + '{"v":1,"type":"unknown"}\n', + ])('rejects malformed or unsupported input without throwing', frame => { + const invalid = vi.fn(); + const decoder = new StandaloneUiDecoder(() => {}, invalid); + expect(() => decoder.push(frame)).not.toThrow(); + expect(invalid).toHaveBeenCalledOnce(); + }); +}); + +describe('standalone output controller', () => { + it('passes successful JSON reporter output through without fd3 events', () => { + const stdout = outputStream(); + const stderr = outputStream(); + const controller = new StandaloneOutputController( + { PATHGRADE_REPORTER_MODE: 'json', FORCE_COLOR: '0' }, + { stdout, stderr }, + ); + controller.stdout('{"status":"pass"}\n'); + controller.finish(0); + expect(read(stdout)).toBe('{"status":"pass"}\n'); + expect(read(stderr)).toBe(''); + }); + + it('falls back to raw Vitest output when the fd3 protocol is malformed', () => { + const stdout = outputStream(); + const stderr = outputStream(); + const controller = new StandaloneOutputController( + { PATHGRADE_REPORTER_MODE: 'cli', FORCE_COLOR: '0' }, + { stdout, stderr }, + ); + controller.stdout('raw vitest output\n'); + controller.protocol('{not json}\n'); + controller.finish(1); + expect(read(stdout)).toBe('raw vitest output\n'); + expect(read(stderr)).toContain('rich output unavailable; using raw Vitest output'); + }); +}); + +describe('standalone UI color controls', () => { + it('respects NO_COLOR and FORCE_COLOR at runtime', () => { + const stream = outputStream(80, true); + expect(createStandaloneTheme(stream, { NO_COLOR: '1' }).green('ok')).toBe('ok'); + expect(createStandaloneTheme(stream, { FORCE_COLOR: '0' }).green('ok')).toBe('ok'); + expect(createStandaloneTheme(stream, { FORCE_COLOR: '1' }).green('ok')).toContain('\x1b[32m'); + }); + + it('disables motion independently for CI, dumb terminals, and quiet mode', () => { + const stream = outputStream(80, true); + expect(createStandaloneTheme(stream, { CI: '1' }).interactive).toBe(false); + expect(createStandaloneTheme(stream, { TERM: 'dumb' }).interactive).toBe(false); + expect(createStandaloneTheme(stream, { PATHGRADE_QUIET: '1' }).interactive).toBe(false); + }); +}); diff --git a/tests/standalone-validation.test.ts b/tests/standalone-validation.test.ts new file mode 100644 index 0000000..e638c3a --- /dev/null +++ b/tests/standalone-validation.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest'; +import { + assertStandalonePlatform, + StandaloneConfigurationError, + validateStandaloneInvocation, +} from '../src/standalone/validation.js'; +import { assertStandaloneAgent } from '../src/sdk/agent-resolution.js'; + +describe('standalone invocation validation', () => { + it.each(['jest', 'node-test', './adapter.mjs'])('rejects adapter %s', (adapterName) => { + expect(() => validateStandaloneInvocation({ adapterName, runnerArgs: [] })) + .toThrow(StandaloneConfigurationError); + }); + + it.each([ + ['--config', 'vitest.config.ts'], + ['--config=vitest.config.ts'], + ['-c', 'vitest.config.ts'], + ['--reporter=dot'], + ['--root', '/tmp/elsewhere'], + ['--dir=other'], + ['--project', 'other'], + ['--watch'], + ['--browser'], + ['--mergeReports'], + ['--environment', 'jsdom'], + ['--unknown-future-flag'], + ])('rejects runner arguments outside the allowlist', (...runnerArgs) => { + expect(() => validateStandaloneInvocation({ adapterName: 'vitest', runnerArgs })) + .toThrow(/standalone runner allows only eval-file filters and -t\/--testNamePattern/); + }); + + it.each([ + ['sample.eval.ts'], + ['sample.eval.ts', '--testNamePattern=creates file'], + ['sample.eval.ts', '--testNamePattern', 'creates file'], + ['sample.eval.ts', '-t', 'creates file'], + ['one.eval.ts', 'two.eval.ts', '-t=creates file'], + ])('accepts positional filters and test-name filters: %j', (...runnerArgs) => { + expect(() => validateStandaloneInvocation({ + adapterName: 'vitest', + runnerArgs, + })).not.toThrow(); + }); + + it('rejects options after the runner separator when they are not allowlisted', () => { + expect(() => validateStandaloneInvocation({ + adapterName: 'vitest', + runnerArgs: ['--', '--root', '/tmp/elsewhere'], + })).toThrow(StandaloneConfigurationError); + }); + + it.each([ + { nodeMajor: 20, platform: 'linux' as const, arch: 'x64' }, + { nodeMajor: 23, platform: 'linux' as const, arch: 'x64' }, + { nodeMajor: 25, platform: 'linux' as const, arch: 'x64' }, + { nodeMajor: 22, platform: 'win32' as const, arch: 'x64' }, + { nodeMajor: 24, platform: 'darwin' as const, arch: 'ia32' }, + ])('rejects unsupported platform input %#', (input) => { + expect(() => assertStandalonePlatform(input)).toThrow(StandaloneConfigurationError); + }); + + it.each([ + { nodeMajor: 22, platform: 'linux' as const, arch: 'x64' }, + { nodeMajor: 24, platform: 'darwin' as const, arch: 'arm64' }, + ])('accepts supported platform input %#', (input) => { + expect(() => assertStandalonePlatform(input)).not.toThrow(); + }); + + it('rejects Cursor and Codex exec before standalone workspace setup', () => { + expect(() => assertStandaloneAgent('cursor')).toThrow(/Cursor/); + expect(() => assertStandaloneAgent('codex', 'exec')).toThrow(/app-server/); + expect(() => assertStandaloneAgent('claude')).not.toThrow(); + expect(() => assertStandaloneAgent('codex', 'app-server')).not.toThrow(); + }); +}); diff --git a/tests/standalone-vitest-config.test.ts b/tests/standalone-vitest-config.test.ts new file mode 100644 index 0000000..2efb128 --- /dev/null +++ b/tests/standalone-vitest-config.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from 'vitest'; +import path from 'node:path'; +import { buildStandaloneVitestConfig } from '../src/standalone/vitest-config.js'; + +describe('standalone Vitest config', () => { + it('keeps configuration and generated output inside the owned cache directory', () => { + const targetRoot = '/tmp/target-project'; + const cacheDir = '/tmp/pathgrade-cache'; + const config = buildStandaloneVitestConfig({ + root: targetRoot, + include: ['**/*.eval.ts'], + exclude: ['**/fixtures/**'], + diagnostics: true, + reporter: 'json', + threshold: 0.8, + cacheDir, + }, { + packageRoot: path.resolve('.'), + vitestEntry: '/tool/node_modules/vitest/dist/index.js', + }); + + expect(config.root).toBe(targetRoot); + expect(config.envFile).toBe(false); + expect(config.cacheDir).toBe('/tmp/pathgrade-cache/vite'); + expect(config.test?.include).toEqual(['**/*.eval.ts']); + expect(config.test?.exclude).toEqual(['**/fixtures/**']); + const previousMode = process.env.PATHGRADE_STANDALONE; + process.env.PATHGRADE_STANDALONE = '1'; + try { + const pluginConfig = (config.plugins?.[0] as { + config(): { test: { reporters: unknown[] } }; + }).config(); + expect(pluginConfig.test.reporters[0]).toBe('minimal'); + } finally { + if (previousMode === undefined) delete process.env.PATHGRADE_STANDALONE; + else process.env.PATHGRADE_STANDALONE = previousMode; + } + expect(config.test?.coverage?.reportsDirectory) + .toBe('/tmp/pathgrade-cache/coverage'); + expect(config.test?.attachmentsDir).toBe('/tmp/pathgrade-cache/attachments'); + expect(config.test?.outputFile).toMatchObject({ + json: '/tmp/pathgrade-cache/reports/pathgrade.json', + blob: '/tmp/pathgrade-cache/reports/vitest.blob', + }); + + const outputs = [ + config.cacheDir, + config.test?.coverage?.reportsDirectory, + config.test?.attachmentsDir, + ...Object.values(config.test?.outputFile as Record), + ].filter((value): value is string => typeof value === 'string'); + expect(outputs.every(output => output.startsWith(`${cacheDir}/`))).toBe(true); + expect(outputs.some(output => output === targetRoot || output.startsWith(`${targetRoot}/`))) + .toBe(false); + }); +}); diff --git a/tests/standalone-vitest-invocation.test.ts b/tests/standalone-vitest-invocation.test.ts new file mode 100644 index 0000000..2b851e9 --- /dev/null +++ b/tests/standalone-vitest-invocation.test.ts @@ -0,0 +1,122 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { describe, expect, it, vi } from 'vitest'; +import { defaultPathgradeConfig } from '../src/config/pathgrade.js'; +import { decodeStandaloneVitestPayload, STANDALONE_VITEST_PAYLOAD_ENV } from '../src/standalone/module-aliases.js'; +import { + createStandaloneVitestInvocationAdapter, + resolveBundledVitestCli, + type SpawnStandaloneVitestRequest, +} from '../src/standalone/vitest-invocation.js'; + +describe('standalone Vitest invocation', () => { + it('uses Node, the bundled CLI, and the internal config', async () => { + const calls: SpawnStandaloneVitestRequest[] = []; + const config = { + ...defaultPathgradeConfig(), + reporter: 'json' as const, + diagnostics: true, + ci: { threshold: 0.8 }, + }; + const adapter = createStandaloneVitestInvocationAdapter({ + config, + spawn: async (request) => { + calls.push(request); + return 0; + }, + resolveRuntime: () => ({ + cliPath: '/tool/node_modules/vitest/vitest.mjs', + entryPath: '/tool/node_modules/vitest/dist/index.js', + version: '4.1.7', + }), + internalConfigPath: '/tool/dist/standalone/vitest-config.js', + createTempDir: () => '/tmp/pathgrade-owned/run-1', + removeTempDir: async () => {}, + }); + + await adapter.run({ + cwd: '/repo', + runnerArgs: ['--testNamePattern=smoke'], + selectedFiles: ['one.eval.ts'], + env: {}, + }); + + expect(calls[0].command).toBe(process.execPath); + expect(calls[0].argv).toEqual([ + '/tool/node_modules/vitest/vitest.mjs', + 'run', + 'one.eval.ts', + '--testNamePattern=smoke', + '--config', + '/tool/dist/standalone/vitest-config.js', + ]); + expect(decodeStandaloneVitestPayload( + calls[0].env[STANDALONE_VITEST_PAYLOAD_ENV], + )).toEqual({ + root: '/repo', + include: config.evals.include, + exclude: config.evals.exclude, + diagnostics: true, + reporter: 'json', + threshold: 0.8, + cacheDir: '/tmp/pathgrade-owned/run-1', + }); + }); + + it.each([ + ['exit zero', async () => 0], + ['nonzero exit', async () => 1], + ['spawn failure', async () => { throw new Error('spawn failed'); }], + ])('cleans its temporary directory after %s', async (_label, spawnImpl) => { + const removed: string[] = []; + const adapter = createStandaloneVitestInvocationAdapter({ + config: defaultPathgradeConfig(), + spawn: spawnImpl, + resolveRuntime: () => ({ + cliPath: '/tool/vitest.mjs', + entryPath: '/tool/index.js', + version: '4.1.7', + }), + internalConfigPath: '/tool/config.js', + createTempDir: () => '/tmp/pathgrade-owned/run-cleanup', + removeTempDir: async dir => { removed.push(dir); }, + }); + + await adapter.run({ + cwd: '/repo', + runnerArgs: [], + env: {}, + }).catch(() => 1); + + expect(removed).toEqual(['/tmp/pathgrade-owned/run-cleanup']); + }); + + it('creates no cache or report output in the target cwd', async () => { + const target = fs.mkdtempSync(path.join(os.tmpdir(), 'pathgrade-target-')); + const adapter = createStandaloneVitestInvocationAdapter({ + config: defaultPathgradeConfig(), + spawn: async () => 0, + }); + + await adapter.run({ cwd: target, runnerArgs: [], env: {} }); + + for (const relative of [ + '.vite', + 'node_modules/.vite', + 'coverage', + '.vitest-attachments', + 'vitest.blob', + ]) { + expect(fs.existsSync(path.join(target, relative))).toBe(false); + } + }); + + it('resolves the exact package-owned Vitest runtime', () => { + expect(resolveBundledVitestCli()).toMatchObject({ + version: '4.1.7', + cliPath: expect.stringMatching(/vitest[\\/]vitest\.mjs$/), + entryPath: expect.stringMatching(/vitest[\\/]dist[\\/]index\.js$/), + }); + }); +}); diff --git a/tests/transport-flag.test.ts b/tests/transport-flag.test.ts index fe1e963..5491453 100644 --- a/tests/transport-flag.test.ts +++ b/tests/transport-flag.test.ts @@ -3,6 +3,7 @@ import { resolveAgentName, resolveCodexTransport, InvalidTransportEnvError, + StandaloneCodexTransportError, } from '../src/sdk/agent-resolution.js'; import { createAgentEnvironment } from '../src/agents/registry.js'; import { CodexAgent } from '../src/agents/codex.js'; @@ -73,6 +74,19 @@ describe('resolveCodexTransport', () => { expect(resolveCodexTransport({}, { PATHGRADE_CODEX_TRANSPORT: '' })).toBe('app-server'); expect(resolveCodexTransport({}, {})).toBe('app-server'); }); + + it('rejects standalone exec from either option or environment', () => { + const standalone = { PATHGRADE_STANDALONE: '1' }; + expect(() => resolveCodexTransport({ transport: 'exec' }, standalone)).toThrow( + StandaloneCodexTransportError, + ); + expect(() => resolveCodexTransport( + {}, + { ...standalone, PATHGRADE_CODEX_TRANSPORT: 'exec' }, + )).toThrow( + "pathgrade standalone supports Codex app-server only; remove transport: 'exec' or use project-local @wix/pathgrade", + ); + }); }); describe('createAgentEnvironment transport-aware routing', () => { diff --git a/tests/verbose-emitter.test.ts b/tests/verbose-emitter.test.ts index 00e1779..ba2bf6f 100644 --- a/tests/verbose-emitter.test.ts +++ b/tests/verbose-emitter.test.ts @@ -57,6 +57,19 @@ describe('createVerboseEmitter', () => { expect(stripAnsi(sink.lines[0])).toBe(' · read_file src/foo.ts'); }); + it('labels every standalone trace line with the agent', () => { + const sink = createFakeSink(); + const emitter = createVerboseEmitter({ enabled: true, sink, agentName: 'codex' }); + emitter.turnStart({ turn: 1, kind: 'agent_start', message: 'inspect' }); + emitter.toolEvent({ action: 'read_file', summary: 'src/foo.ts' }); + emitter.conversationEnd({ reason: 'done', turns: 1, durationMs: 10 }); + expect(sink.lines.map(stripAnsi)).toEqual([ + 'CODEX TURN 1 [agent_start] "inspect"', + 'CODEX TOOL read_file src/foo.ts', + 'CODEX END reason=done turns=1 0.0s', + ]); + }); + it('formats turnEnd as `← Turn N Ns Nl "preview"`', () => { const sink = createFakeSink(); const emitter = createVerboseEmitter({ enabled: true, sink }); diff --git a/tests/vitest-lifecycle-adapter.test.ts b/tests/vitest-lifecycle-adapter.test.ts index f70dac4..8637df3 100644 --- a/tests/vitest-lifecycle-adapter.test.ts +++ b/tests/vitest-lifecycle-adapter.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it, vi } from 'vitest'; import type { AdapterLifecycleHooks } from '../src/runners/adapter.js'; import { installVitestLifecycle } from '../src/runners/vitest-lifecycle.js'; +import type { Agent, RecordedEvalResult } from '../src/sdk/types.js'; +import { lifecycleCore } from '../src/sdk/lifecycle.js'; +import { createMockLLM } from '../src/utils/llm-mocks.js'; describe('Vitest adapter lifecycle wiring', () => { it('routes aroundEach, afterEach, and afterAll through adapter lifecycle hooks with stable case IDs', async () => { @@ -65,4 +68,60 @@ describe('Vitest adapter lifecycle wiring', () => { 'unsubscribe', ]); }); + + it('owns and disposes structural agents first observed through evaluate results', async () => { + let afterEachCallback: ((ctx: { task: { id: string; meta: Record } }) => Promise) | undefined; + let aroundEachCallback: ((runTest: () => Promise, ctx: { task: { id: string; name: string; meta: Record; suite?: unknown } }) => Promise) | undefined; + let resultCallback: ((event: { result: RecordedEvalResult; agent: Agent }) => void) | undefined; + const agent = structuralAgent(); + + const handle = installVitestLifecycle({ + afterEach: callback => { afterEachCallback = callback; }, + aroundEach: callback => { aroundEachCallback = callback; }, + subscribeToResults: callback => { + resultCallback = callback; + return { unsubscribe: vi.fn() }; + }, + installFileContextProvider: () => ({ restore: vi.fn() }), + }); + const task = { + id: 'structural-agent-case', + name: 'structural agent case', + meta: {} as Record, + suite: { filepath: '/repo/structural.eval.ts' }, + }; + + await aroundEachCallback?.(async () => { + resultCallback?.({ + result: { score: 1, scorers: [] }, + agent, + }); + }, { task }); + await afterEachCallback?.({ task }); + handle.restore(); + lifecycleCore.reset(); + + expect(task.meta.pathgrade).toEqual([expect.objectContaining({ score: 1 })]); + expect(agent.dispose).toHaveBeenCalledOnce(); + }); }); + +function structuralAgent(): Agent & { dispose: ReturnType } { + return { + workspace: '/fake', + log: [], + messages: [], + llm: createMockLLM(), + transcript: () => '', + exec: async () => ({ stdout: '', stderr: '', exitCode: 0 }), + prompt: async () => '', + startChat: async () => { throw new Error('not used'); }, + runConversation: async () => ({ + turns: 0, + completionReason: 'until' as const, + turnTimings: [], + stepResults: [], + }), + dispose: vi.fn().mockResolvedValue(undefined), + }; +} diff --git a/yarn.lock b/yarn.lock index 7defdc0..ebac2ca 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1210,6 +1210,77 @@ __metadata: languageName: node linkType: hard +"@openai/codex-darwin-arm64@npm:@openai/codex@0.144.0-darwin-arm64": + version: 0.144.0-darwin-arm64 + resolution: "@openai/codex@npm:0.144.0-darwin-arm64" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@openai/codex-darwin-x64@npm:@openai/codex@0.144.0-darwin-x64": + version: 0.144.0-darwin-x64 + resolution: "@openai/codex@npm:0.144.0-darwin-x64" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@openai/codex-linux-arm64@npm:@openai/codex@0.144.0-linux-arm64": + version: 0.144.0-linux-arm64 + resolution: "@openai/codex@npm:0.144.0-linux-arm64" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + +"@openai/codex-linux-x64@npm:@openai/codex@0.144.0-linux-x64": + version: 0.144.0-linux-x64 + resolution: "@openai/codex@npm:0.144.0-linux-x64" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + +"@openai/codex-win32-arm64@npm:@openai/codex@0.144.0-win32-arm64": + version: 0.144.0-win32-arm64 + resolution: "@openai/codex@npm:0.144.0-win32-arm64" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + +"@openai/codex-win32-x64@npm:@openai/codex@0.144.0-win32-x64": + version: 0.144.0-win32-x64 + resolution: "@openai/codex@npm:0.144.0-win32-x64" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + +"@openai/codex@npm:0.144.0": + version: 0.144.0 + resolution: "@openai/codex@npm:0.144.0" + dependencies: + "@openai/codex-darwin-arm64": "npm:@openai/codex@0.144.0-darwin-arm64" + "@openai/codex-darwin-x64": "npm:@openai/codex@0.144.0-darwin-x64" + "@openai/codex-linux-arm64": "npm:@openai/codex@0.144.0-linux-arm64" + "@openai/codex-linux-x64": "npm:@openai/codex@0.144.0-linux-x64" + "@openai/codex-win32-arm64": "npm:@openai/codex@0.144.0-win32-arm64" + "@openai/codex-win32-x64": "npm:@openai/codex@0.144.0-win32-x64" + dependenciesMeta: + "@openai/codex-darwin-arm64": + optional: true + "@openai/codex-darwin-x64": + optional: true + "@openai/codex-linux-arm64": + optional: true + "@openai/codex-linux-x64": + optional: true + "@openai/codex-win32-arm64": + optional: true + "@openai/codex-win32-x64": + optional: true + bin: + codex: bin/codex.js + checksum: 10c0/47ff1d98403611ae3d4cbf49b4584fb36833a65a40b2ba0a91fa48aae49568567dd22c000a7baf0f3e6131dd614f5bc0e1995df107cde4996eca09040bca5183 + languageName: node + linkType: hard + "@oxc-project/types@npm:=0.126.0": version: 0.126.0 resolution: "@oxc-project/types@npm:0.126.0" @@ -1823,6 +1894,7 @@ __metadata: dependencies: "@anthropic-ai/claude-agent-sdk": "npm:0.2.116" "@modelcontextprotocol/sdk": "npm:1.29.0" + "@openai/codex": "npm:0.144.0" "@types/fs-extra": "npm:^11.0.4" "@types/jest": "npm:^30.0.0" "@types/node": "npm:25.6.0"