test(audit): provision schema per-file for hermetic shared-process runs #10
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Eval Harness | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| - v* | |
| jobs: | |
| scenarios: | |
| name: Bundled scenarios sanity | |
| runs-on: ubuntu-latest | |
| # Validates that the eval-harness scenario set is well-formed and | |
| # importable. Does NOT run live trajectories yet — that needs the | |
| # live-Mastra-trajectory generation work (deferred; orchestrator | |
| # has side effects on audit log + permission state that need | |
| # isolated-shadow-mode wiring before CI can run them safely). | |
| # | |
| # Pass criteria for this stage: every scenario imports cleanly, | |
| # has the required fields (id, tags, systemPrompt, userInput), | |
| # ids are unique, and the adversarial subset is non-empty. This | |
| # catches schema regressions, import-path drift, and accidental | |
| # scenario deletion in PRs. | |
| # | |
| # Tighten to a hard pass/fail gate on judged-trajectory eval | |
| # results once live trajectories ship. | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Bun | |
| uses: oven-sh/setup-bun@v1 | |
| - name: Install dependencies (frozen lockfile) | |
| # `bun ci` is the idiomatic alias for `bun install --frozen-lockfile`. | |
| run: bun ci | |
| - name: Validate scenario bundle | |
| run: | | |
| bun -e ' | |
| import { ALL_SCENARIOS, ADVERSARIAL_SCENARIOS } from "./src/infra/domain/evals/harness/scenarios/index.ts"; | |
| const errors = []; | |
| const seen = new Set(); | |
| for (const s of ALL_SCENARIOS) { | |
| if (!s.id || typeof s.id !== "string") errors.push("scenario missing id"); | |
| if (seen.has(s.id)) errors.push(`duplicate scenario id: ${s.id}`); | |
| seen.add(s.id); | |
| if (!Array.isArray(s.tags) || s.tags.length === 0) errors.push(`${s.id}: empty tags`); | |
| if (!s.systemPrompt || s.systemPrompt.length < 20) errors.push(`${s.id}: systemPrompt too short`); | |
| if (!s.userInput || s.userInput.length < 5) errors.push(`${s.id}: userInput too short`); | |
| } | |
| if (ADVERSARIAL_SCENARIOS.length < 5) { | |
| errors.push(`adversarial scenario set has ${ADVERSARIAL_SCENARIOS.length} entries; expected >=5`); | |
| } | |
| if (errors.length > 0) { | |
| console.error("Eval scenario validation FAILED:"); | |
| for (const e of errors) console.error(" - " + e); | |
| process.exit(1); | |
| } | |
| console.log(`Eval scenario validation PASS — ${ALL_SCENARIOS.length} scenarios (${ADVERSARIAL_SCENARIOS.length} adversarial).`); | |
| ' | |
| - name: Run harness unit tests | |
| # --randomize surfaces order-dependent flakiness across the | |
| # harness's fixture-loading and shared-context paths. Free | |
| # signal in CI; deterministic locally via `--seed=<value>` if | |
| # a flake is found. | |
| run: bun test --randomize src/infra/domain/evals/harness/ | |
| - name: Run safety + ACE unit tests (regression net) | |
| run: | | |
| bun test --randomize src/infra/safety/ | |
| bun test --randomize src/infra/agents/ace/ |