From d69db7d28aa9aefbc3f53917cbaf9c8151865bfa Mon Sep 17 00:00:00 2001 From: Eric Juta Date: Wed, 26 Aug 2026 20:14:17 +0000 Subject: [PATCH 1/2] feat: add Codex repository marketplace --- .agents/plugins/marketplace.json | 20 + .github/workflows/ci.yml | 35 + docs/architecture.md | 30 +- package.json | 1 + .../__tests__/check-codex-marketplace.test.ts | 399 +++++++++ packages/evals/__tests__/codex-cli.test.ts | 16 +- packages/evals/package.json | 3 +- .../evals/src/bin/check-codex-marketplace.ts | 761 ++++++++++++++++++ packages/evals/src/bin/live.ts | 24 +- packages/evals/src/codex-cli.ts | 9 +- .../openai => }/.codex-plugin/plugin.json | 0 .../ask-gina/{targets/openai => }/.mcp.json | 0 .../ask-gina/__tests__/plugin-core.test.ts | 48 +- .../__tests__/sync-plugin-skills.test.ts | 145 ++-- .../{targets/openai => }/assets/icon.svg | 0 tools/__tests__/pack-artifacts.test.ts | 171 ++++ tools/__tests__/verify-artifacts.test.ts | 235 +++++- tools/check-target-conformance.ts | 393 ++++++++- tools/pack-artifacts.ts | 108 ++- tools/sync-plugin-skills.ts | 109 ++- tools/verify-artifacts.ts | 53 ++ vite.config.ts | 1 + 22 files changed, 2424 insertions(+), 137 deletions(-) create mode 100644 .agents/plugins/marketplace.json create mode 100644 packages/evals/__tests__/check-codex-marketplace.test.ts create mode 100755 packages/evals/src/bin/check-codex-marketplace.ts rename plugins/ask-gina/{targets/openai => }/.codex-plugin/plugin.json (100%) rename plugins/ask-gina/{targets/openai => }/.mcp.json (100%) rename plugins/ask-gina/{targets/openai => }/assets/icon.svg (100%) diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json new file mode 100644 index 0000000..d244fca --- /dev/null +++ b/.agents/plugins/marketplace.json @@ -0,0 +1,20 @@ +{ + "name": "ask-gina-plugins", + "interface": { + "displayName": "Ask Gina Plugins" + }, + "plugins": [ + { + "name": "ask-gina", + "source": { + "source": "local", + "path": "./plugins/ask-gina" + }, + "policy": { + "installation": "AVAILABLE", + "authentication": "ON_INSTALL" + }, + "category": "Finance" + } + ] +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47b78fd..b51abad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,41 @@ jobs: - run: bun run verify:artifacts - run: bun run smoke:install - run: bun run check:public-boundary + - name: Install pinned Codex marketplace CLI + if: >- + github.event_name == 'push' || + (github.event.pull_request.head.repo.full_name != '' && + github.event.pull_request.head.sha != '') + env: + BUN_INSTALL_CACHE_DIR: ${{ runner.temp }}/codex-cli-cache + CODEX_CLI_ROOT: ${{ runner.temp }}/codex-cli + run: | + mkdir -p "$CODEX_CLI_ROOT" + printf '{"private":true}\n' > "$CODEX_CLI_ROOT/package.json" + bun add --cwd "$CODEX_CLI_ROOT" --exact @openai/codex@0.149.0-alpha.4.3 + - name: Smoke Codex marketplace at published revision + if: >- + github.event_name == 'push' || + (github.event.pull_request.head.repo.full_name != '' && + github.event.pull_request.head.sha != '') + env: + CODEX_MARKETPLACE_EXECUTABLE: ${{ runner.temp }}/codex-cli/node_modules/.bin/codex + CODEX_MARKETPLACE_REF: >- + ${{ github.event.pull_request.head.sha || github.sha }} + CODEX_MARKETPLACE_REPOSITORY: >- + ${{ github.event.pull_request.head.repo.full_name || github.repository }} + run: >- + bun run check:marketplace:codex -- + --executable "$CODEX_MARKETPLACE_EXECUTABLE" + --repository "$CODEX_MARKETPLACE_REPOSITORY" + --ref "$CODEX_MARKETPLACE_REF" + - name: Remove temporary Codex CLI state + if: >- + always() && + (github.event_name == 'push' || + (github.event.pull_request.head.repo.full_name != '' && + github.event.pull_request.head.sha != '')) + run: rm -rf "$RUNNER_TEMP/codex-cli" "$RUNNER_TEMP/codex-cli-cache" - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: receipts diff --git a/docs/architecture.md b/docs/architecture.md index a777c2e..9503af2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -15,11 +15,22 @@ CommonJS, browser, edge, or subpath entrypoints. They support Node >=24 and Bun > =1.4. The compiled CLI and eval executables run on Bun 1.4.x only. -## Skills +## Marketplace and skills -`plugins/ask-gina/skills/` is the only authoring source. Pack-time generation -builds temporary host trees and the skills candidate. Generated -`targets//skills/` copies are not source. +`.agents/plugins/marketplace.json` is the repository marketplace descriptor. It +points directly at `plugins/ask-gina/`, so a clean checkout exposes the OpenAI +source without a generation step. The directly loadable OpenAI files live at the +plugin root: `.codex-plugin/plugin.json`, `.mcp.json`, `assets/icon.svg`, and `skills/`. + +`plugins/ask-gina/skills/` is the only authored skill tree. OpenAI-specific skill +metadata remains beside each canonical `SKILL.md` under `agents/openai.yaml`. +`plugins/ask-gina/targets/` contains only the Cursor, Claude, Copilot, and Gemini +overlays; there is no `targets/openai/` source tree. + +Sync and pack operations build temporary host targets from these sources. The +OpenAI target selects only the root OpenAI files and canonical skills. Every +other host combines its `targets//` overlay with the same canonical +skills. Generated host trees are output, never authoring source. ## Runtime @@ -43,10 +54,13 @@ enabled compiler and Effect diagnostic; there is no baseline or count ratchet. source maps; every map embeds committed TypeScript through relative paths. The repository's custom packer stages those validated outputs with metadata and assets as five npm-style package archives, five host archives, one skills candidate, and -four bounded receipts under ignored `dist/`. Verification clean-installs package -tarballs in temporary projects, checks their closures, runs their actual runtime -entrypoints, reruns target conformance, and regenerates the four-dimension -hermetic eval aggregate from synthetic files. Generated `dist/` content is evidence, never authoring source. +four bounded receipts under ignored `dist/`. Host archives remain lean: each +contains only the selected host surface and skills, never repository workspace +files or foreign-host overlays. Verification clean-installs package tarballs in +temporary projects, checks their closures, runs their actual runtime entrypoints, +reruns target conformance, and regenerates the four-dimension hermetic eval +aggregate from synthetic files. Generated `dist/` content is evidence, never +authoring source. ## CI diff --git a/package.json b/package.json index 43d1c02..125e83d 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "eval:replay": "bun run build && bun packages/evals/dist/bin/replay.js", "eval:responses": "bun run build && bun packages/evals/dist/bin/live.js --runner responses", "eval:codex": "bun run build && bun packages/evals/dist/bin/live.js --runner codex", + "check:marketplace:codex": "bun run build && bun packages/evals/dist/bin/check-codex-marketplace.js", "effect-tsgo:patch": "bun scripts/effect-tsgo-patch-if-needed.ts", "prepare": "bun run effect-tsgo:patch", "typecheck": "bun run build && node_modules/.bin/tsc --noEmit -p tsconfig.json", diff --git a/packages/evals/__tests__/check-codex-marketplace.test.ts b/packages/evals/__tests__/check-codex-marketplace.test.ts new file mode 100644 index 0000000..2f0e6bf --- /dev/null +++ b/packages/evals/__tests__/check-codex-marketplace.test.ts @@ -0,0 +1,399 @@ +import * as BunServices from "@effect/platform-bun/BunServices"; +import { ASK_GINA_SKILL_DEFINITIONS, PRODUCTION_MCP_URL } from "@askgina/contracts"; +import { assert, describe, it } from "@effect/vitest"; +import { Effect, FileSystem, Path, Schema } from "effect"; + +import { + buildCodexMarketplaceEnvironment, + CodexMarketplaceSmokeError, + inspectInstalledAskGinaPlugin, + type MarketplaceCommandInput, + type MarketplaceCommandRunner, + parseCodexMarketplaceSmokeArgs, + parseMarketplaceAddOutput, + parseMarketplaceListOutput, + parsePluginAddOutput, + runCodexMarketplaceSmoke, + type CodexMarketplaceIsolation, +} from "../src/bin/check-codex-marketplace"; + +const IMMUTABLE_REF = "1234567890abcdef1234567890abcdef12345678"; +const OPTIONS = { + repository: "askgina/plugins", + ref: IMMUTABLE_REF, + executable: "codex-test", + timeoutMs: 30_000, +} as const; +const JsonString = Schema.fromJsonString(Schema.Unknown); +const encodeJson = Schema.encodeUnknownSync(JsonString); + +const writeInstalledPluginFixture = ( + installedRoot: string, +): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* Effect.forEach( + [path.join(installedRoot, ".codex-plugin"), path.join(installedRoot, "assets")], + (directory) => fs.makeDirectory(directory, { recursive: true }), + { discard: true }, + ); + yield* Effect.all([ + fs.writeFileString( + path.join(installedRoot, ".codex-plugin", "plugin.json"), + encodeJson({ + name: "ask-gina", + skills: "./skills/", + mcpServers: "./.mcp.json", + interface: { + composerIcon: "./assets/icon.svg", + logo: "./assets/icon.svg", + }, + }), + ), + fs.writeFileString( + path.join(installedRoot, ".mcp.json"), + encodeJson({ + mcpServers: { + "ask-gina": { type: "http", url: PRODUCTION_MCP_URL }, + }, + }), + ), + fs.writeFileString(path.join(installedRoot, "assets", "icon.svg"), ""), + ]); + yield* Effect.forEach( + ASK_GINA_SKILL_DEFINITIONS, + (skill) => + Effect.gen(function* () { + const skillRoot = path.join(installedRoot, "skills", skill.name); + yield* fs.makeDirectory(path.join(skillRoot, "agents"), { recursive: true }); + yield* Effect.all([ + fs.writeFileString(path.join(skillRoot, "SKILL.md"), `# ${skill.name}\n`), + fs.writeFileString(path.join(skillRoot, "agents", "openai.yaml"), "name: Ask Gina\n"), + ]); + }), + { concurrency: "unbounded", discard: true }, + ); + }).pipe(Effect.orDie); + +const marketplaceListing = (marketplaceRoot: string, authenticationPolicy = "ON_INSTALL"): string => + encodeJson({ + installed: [], + available: [ + { + pluginId: "ask-gina@ask-gina-plugins", + name: "ask-gina", + marketplaceName: "ask-gina-plugins", + version: "0.1.0", + installed: false, + enabled: false, + source: { source: "local", path: `${marketplaceRoot}/plugins/ask-gina` }, + marketplaceSource: { sourceType: "git", source: "askgina/plugins" }, + installPolicy: "AVAILABLE", + authPolicy: authenticationPolicy, + }, + ], + }); + +const makeFakeRunner = ( + commands: MarketplaceCommandInput[], + failStage?: MarketplaceCommandInput["stage"], +): MarketplaceCommandRunner => ({ + run: (input) => + Effect.gen(function* () { + commands.push(input); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const codexHome = input.environment.CODEX_HOME; + if (codexHome === undefined) { + return yield* new CodexMarketplaceSmokeError({ + stage: input.stage, + reason: "command-failed", + detail: "fake runner did not receive CODEX_HOME", + }); + } + if (input.stage === failStage) { + return yield* new CodexMarketplaceSmokeError({ + stage: input.stage, + reason: "command-failed", + detail: `${input.stage} failed (exit 7): synthetic diagnostic`, + }); + } + const isolationRoot = path.dirname(codexHome); + if (input.stage === "marketplace-add") { + const marketplaceRoot = path.join(isolationRoot, "remote", "askgina-plugins"); + yield* fs.makeDirectory(marketplaceRoot, { recursive: true }).pipe(Effect.orDie); + return { + stdout: encodeJson({ + marketplaceName: "ask-gina-plugins", + installedRoot: marketplaceRoot, + alreadyAdded: false, + }), + stderr: "", + }; + } + if (input.stage === "marketplace-list") { + return { + stdout: marketplaceListing(path.join(isolationRoot, "remote", "askgina-plugins")), + stderr: "", + }; + } + if (input.stage === "plugin-add") { + const installedRoot = path.join( + codexHome, + "plugins", + "cache", + "ask-gina-plugins", + "ask-gina", + "0.1.0", + ); + yield* writeInstalledPluginFixture(installedRoot); + return { + stdout: encodeJson({ + pluginId: "ask-gina@ask-gina-plugins", + name: "ask-gina", + marketplaceName: "ask-gina-plugins", + version: "0.1.0", + installedPath: installedRoot, + authPolicy: "ON_INSTALL", + }), + stderr: "", + }; + } + return { stdout: "{}", stderr: "" }; + }), +}); + +describe("Codex marketplace smoke arguments and JSON", () => { + it.effect("requires an immutable ref and accepts a remote owner/repo", () => + Effect.gen(function* () { + const parsed = yield* parseCodexMarketplaceSmokeArgs([ + "--repository", + "askgina/plugins", + "--ref", + IMMUTABLE_REF.toUpperCase(), + "--executable", + "/opt/codex", + "--timeout-ms", + "45000", + ]); + assert.deepStrictEqual(parsed, { + repository: "askgina/plugins", + ref: IMMUTABLE_REF, + executable: "/opt/codex", + timeoutMs: 45_000, + }); + + const missingRef = yield* Effect.result(parseCodexMarketplaceSmokeArgs([])); + assert.strictEqual(missingRef._tag, "Failure"); + if (missingRef._tag === "Failure") { + assert.strictEqual(missingRef.failure.reason, "invalid-arguments"); + assert.include(missingRef.failure.detail, "immutable"); + } + const branchRef = yield* Effect.result(parseCodexMarketplaceSmokeArgs(["--ref", "main"])); + assert.strictEqual(branchRef._tag, "Failure"); + }), + ); + + it.effect("parses marketplace and install JSON without treating ON_INSTALL as OAuth", () => + Effect.gen(function* () { + assert.deepStrictEqual( + yield* parseMarketplaceAddOutput( + encodeJson({ + marketplaceName: "ask-gina-plugins", + installedRoot: "/tmp/marketplace", + alreadyAdded: false, + }), + ), + { marketplaceName: "ask-gina-plugins", installedRoot: "/tmp/marketplace" }, + ); + yield* parseMarketplaceListOutput(marketplaceListing("/tmp/marketplace"), "/tmp/marketplace"); + assert.deepStrictEqual( + yield* parsePluginAddOutput( + encodeJson({ + pluginId: "ask-gina@ask-gina-plugins", + name: "ask-gina", + marketplaceName: "ask-gina-plugins", + version: "0.1.0", + installedPath: "/tmp/plugin", + authPolicy: "ON_INSTALL", + }), + ), + { installedPath: "/tmp/plugin" }, + ); + + const oauth = yield* Effect.result( + parseMarketplaceListOutput( + marketplaceListing("/tmp/marketplace", "OAUTH"), + "/tmp/marketplace", + ), + ); + assert.strictEqual(oauth._tag, "Failure"); + if (oauth._tag === "Failure") { + assert.include(oauth.failure.detail, "ON_INSTALL"); + } + const malformed = yield* Effect.result(parseMarketplaceAddOutput("not-json")); + assert.strictEqual(malformed._tag, "Failure"); + if (malformed._tag === "Failure") { + assert.strictEqual(malformed.failure.reason, "invalid-json"); + } + }), + ); +}); + +describe("Codex marketplace environment isolation", () => { + it.effect("replaces home and XDG state while dropping credentials", () => + Effect.sync(() => { + const isolation: CodexMarketplaceIsolation = { + root: "/tmp/smoke", + home: "/tmp/smoke/home", + codexHome: "/tmp/smoke/codex", + configHome: "/tmp/smoke/xdg/config", + cacheHome: "/tmp/smoke/xdg/cache", + dataHome: "/tmp/smoke/xdg/data", + stateHome: "/tmp/smoke/xdg/state", + runtimeDirectory: "/tmp/smoke/xdg/runtime", + workingDirectory: "/tmp/smoke/work", + }; + const environment = buildCodexMarketplaceEnvironment(isolation, { + PATH: "/usr/bin:/bin", + HOME: "/home/user", + CODEX_HOME: "/home/user/.codex", + XDG_CONFIG_HOME: "/home/user/.config", + XDG_CACHE_HOME: "/home/user/.cache", + CI: "true", + OPENAI_API_KEY: "provider-secret", + ASK_GINA_ACCESS_TOKEN: "gina-secret", + GITHUB_TOKEN: "github-secret", + SSH_AUTH_SOCK: "/tmp/agent.sock", + HTTPS_PROXY: "http://proxy-secret", + }); + + assert.strictEqual(environment.HOME, isolation.home); + assert.strictEqual(environment.CODEX_HOME, isolation.codexHome); + assert.strictEqual(environment.XDG_CONFIG_HOME, isolation.configHome); + assert.strictEqual(environment.XDG_CACHE_HOME, isolation.cacheHome); + assert.strictEqual(environment.XDG_DATA_HOME, isolation.dataHome); + assert.strictEqual(environment.XDG_STATE_HOME, isolation.stateHome); + assert.strictEqual(environment.XDG_RUNTIME_DIR, isolation.runtimeDirectory); + assert.strictEqual(environment.GIT_CONFIG_NOSYSTEM, "1"); + assert.strictEqual(environment.GIT_TERMINAL_PROMPT, "0"); + assert.strictEqual(environment.CI, "true"); + assert.notProperty(environment, "OPENAI_API_KEY"); + assert.notProperty(environment, "ASK_GINA_ACCESS_TOKEN"); + assert.notProperty(environment, "GITHUB_TOKEN"); + assert.notProperty(environment, "SSH_AUTH_SOCK"); + assert.notProperty(environment, "HTTPS_PROXY"); + }), + ); +}); + +describe("Codex marketplace smoke lifecycle", () => { + it.layer(BunServices.layer)((it) => { + it.effect("inspects root MCP files and cleans plugin, marketplace, and temporary state", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const commands: MarketplaceCommandInput[] = []; + const result = yield* runCodexMarketplaceSmoke({ + ...OPTIONS, + runner: makeFakeRunner(commands), + parentEnvironment: { + PATH: "/usr/bin:/bin", + OPENAI_API_KEY: "must-not-leak", + }, + }); + + assert.deepStrictEqual(result, { + repository: "askgina/plugins", + ref: IMMUTABLE_REF, + marketplaceName: "ask-gina-plugins", + pluginId: "ask-gina@ask-gina-plugins", + }); + assert.deepStrictEqual( + commands.map((command) => command.stage), + [ + "marketplace-add", + "marketplace-list", + "plugin-add", + "plugin-remove", + "marketplace-remove", + ], + ); + assert.deepStrictEqual(commands[0]?.args, [ + "plugin", + "marketplace", + "add", + "askgina/plugins", + "--ref", + IMMUTABLE_REF, + "--json", + ]); + assert.isFalse( + commands.some((command) => + command.args.some((argument) => ["auth", "login", "oauth"].includes(argument)), + ), + ); + assert.notProperty(commands[0]?.environment ?? {}, "OPENAI_API_KEY"); + const codexHome = commands[0]?.environment.CODEX_HOME; + if (codexHome === undefined) return yield* Effect.die("fake runner did not capture home"); + assert.isFalse(yield* fs.exists(path.dirname(codexHome))); + }), + ); + + it.effect("runs cleanup and preserves failure diagnostics when a command fails", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const commands: MarketplaceCommandInput[] = []; + const result = yield* Effect.result( + runCodexMarketplaceSmoke({ + ...OPTIONS, + runner: makeFakeRunner(commands, "marketplace-list"), + parentEnvironment: { PATH: "/usr/bin:/bin" }, + }), + ); + + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") { + assert.strictEqual(result.failure.stage, "marketplace-list"); + assert.strictEqual( + result.failure.detail, + "marketplace-list failed (exit 7): synthetic diagnostic", + ); + } + assert.deepStrictEqual( + commands.map((command) => command.stage), + ["marketplace-add", "marketplace-list", "plugin-remove", "marketplace-remove"], + ); + const codexHome = commands[0]?.environment.CODEX_HOME; + if (codexHome === undefined) return yield* Effect.die("fake runner did not capture home"); + assert.isFalse(yield* fs.exists(path.dirname(codexHome))); + }), + ); + + it.effect("rejects the legacy gina MCP alias during deterministic inspection", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "marketplace-inspection-test-" }); + yield* writeInstalledPluginFixture(root); + yield* fs.writeFileString( + path.join(root, ".mcp.json"), + encodeJson({ + mcpServers: { + gina: { type: "http", url: PRODUCTION_MCP_URL }, + }, + }), + ); + + const result = yield* Effect.result(inspectInstalledAskGinaPlugin(root)); + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") { + assert.strictEqual(result.failure.reason, "invalid-plugin"); + assert.include(result.failure.detail, "ask-gina"); + } + }), + ); + }); +}); diff --git a/packages/evals/__tests__/codex-cli.test.ts b/packages/evals/__tests__/codex-cli.test.ts index 7f5e207..48d08ec 100644 --- a/packages/evals/__tests__/codex-cli.test.ts +++ b/packages/evals/__tests__/codex-cli.test.ts @@ -89,7 +89,7 @@ const jsonl = (...events: readonly Record[]): string => `${events.map((event) => Schema.encodeUnknownSync(JsonLine)(event)).join("\n")}\n`; describe("Codex CLI JSONL evidence", () => { - it.effect("records only Ask Gina skill-path reads and completed Gina tool calls", () => + it.effect("records only Ask Gina skill-path reads and completed Ask Gina tool calls", () => Effect.sync(() => { assert.strictEqual( extractAskGinaActivatedSkills(ASK_GINA_SKILL_PATH), @@ -174,15 +174,11 @@ describe("Codex CLI JSONL evidence", () => { ); assert.deepStrictEqual(parsed.activated_skills, ["research-spot-tokens"]); - assert.strictEqual(parsed.tool_calls.length, 2); - assert.strictEqual(parsed.unsupported_actions, 2); - assert.strictEqual(parsed.tool_calls[0]?.name, "spot.getSimplePrice"); + assert.strictEqual(parsed.tool_calls.length, 1); + assert.strictEqual(parsed.unsupported_actions, 3); + assert.strictEqual(parsed.tool_calls[0]?.name, "gina.getAccountAddresses"); assert.isUndefined(parsed.tool_calls[0]?.error); - assert.deepStrictEqual(parsed.tool_calls[0]?.arguments, { - ids: "ethereum", - vs_currencies: "usd", - }); - assert.strictEqual(parsed.tool_calls[1]?.name, "gina.getAccountAddresses"); + assert.deepStrictEqual(parsed.tool_calls[0]?.arguments, {}); assert.strictEqual(parsed.final_answer, "ETH is $1."); assert.deepStrictEqual(parsed.token_usage, { input_tokens: 10, @@ -474,7 +470,7 @@ describe("Codex CLI trial adapter", () => { assert.include(profile, '[mcp_servers."gina"]\nenabled = false'); assert.include( profile, - `[plugins.${encodeJsonString(trialOptions.pluginId)}.mcp_servers.gina]`, + `[plugins.${encodeJsonString(trialOptions.pluginId)}.mcp_servers.ask-gina]`, ); assert.include( profile, diff --git a/packages/evals/package.json b/packages/evals/package.json index 395aa31..9fec4db 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -17,7 +17,8 @@ } }, "scripts": { - "build": "vp -C ../.. pack --filter evals" + "build": "vp -C ../.. pack --filter evals", + "check:marketplace:codex": "bun run build && bun dist/bin/check-codex-marketplace.js" }, "dependencies": { "@askgina/contracts": "workspace:*", diff --git a/packages/evals/src/bin/check-codex-marketplace.ts b/packages/evals/src/bin/check-codex-marketplace.ts new file mode 100755 index 0000000..3a9aedf --- /dev/null +++ b/packages/evals/src/bin/check-codex-marketplace.ts @@ -0,0 +1,761 @@ +#!/usr/bin/env bun + +import * as BunRuntime from "@effect/platform-bun/BunRuntime"; +import * as BunServices from "@effect/platform-bun/BunServices"; +import { ASK_GINA_SKILL_DEFINITIONS, PRODUCTION_MCP_URL } from "@askgina/contracts"; +import { Console, Data, Duration, Effect, FileSystem, Function, Layer, Path, Schema } from "effect"; +import { ChildProcess } from "effect/unstable/process"; +import type { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner"; + +import { collectBoundedUtf8Output } from "../bounded-output"; +import { CODEX_CLI_ALLOWED_ENVIRONMENT_NAMES } from "../codex-cli"; + +const DEFAULT_REPOSITORY = "askgina/plugins"; +const DEFAULT_EXECUTABLE = "codex"; +const DEFAULT_TIMEOUT_MS = 120_000; +const MAXIMUM_OUTPUT_BYTES = 1_048_576; +const MARKETPLACE_NAME = "ask-gina-plugins"; +const PLUGIN_NAME = "ask-gina"; +const PLUGIN_ID = `${PLUGIN_NAME}@${MARKETPLACE_NAME}`; +const IMMUTABLE_REF_RE = /^[0-9a-f]{40}$/iu; +const REPOSITORY_RE = /^[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,38})\/[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,99})$/u; +const UnknownJson = Schema.fromJsonString(Schema.Unknown); +const decodeUnknownJson = Schema.decodeUnknownOption(UnknownJson); + +export type CodexMarketplaceSmokeStage = + | "arguments" + | "environment" + | "marketplace-add" + | "marketplace-list" + | "plugin-add" + | "inspection" + | "plugin-remove" + | "marketplace-remove"; + +export class CodexMarketplaceSmokeError extends Data.TaggedError("CodexMarketplaceSmokeError")<{ + readonly stage: CodexMarketplaceSmokeStage; + readonly reason: + | "command-failed" + | "invalid-arguments" + | "invalid-json" + | "invalid-output" + | "invalid-plugin" + | "io-failed" + | "timeout"; + readonly detail: string; +}> {} + +export interface CodexMarketplaceSmokeOptions { + readonly repository: string; + readonly ref: string; + readonly executable: string; + readonly timeoutMs: number; +} + +export interface CodexMarketplaceIsolation { + readonly root: string; + readonly home: string; + readonly codexHome: string; + readonly configHome: string; + readonly cacheHome: string; + readonly dataHome: string; + readonly stateHome: string; + readonly runtimeDirectory: string; + readonly workingDirectory: string; +} + +export interface MarketplaceCommandInput { + readonly stage: Exclude; + readonly executable: string; + readonly args: readonly string[]; + readonly cwd: string; + readonly environment: Readonly>; + readonly timeoutMs: number; +} + +export interface MarketplaceCommandResult { + readonly stdout: string; + readonly stderr: string; +} + +export interface MarketplaceCommandRunner { + readonly run: ( + input: MarketplaceCommandInput, + ) => Effect.Effect< + MarketplaceCommandResult, + CodexMarketplaceSmokeError, + ChildProcessSpawner | FileSystem.FileSystem | Path.Path + >; +} +export interface CodexMarketplaceSmokeRunOptions extends CodexMarketplaceSmokeOptions { + readonly runner?: MarketplaceCommandRunner; + readonly parentEnvironment?: Readonly>; +} + +export interface CodexMarketplaceSmokeResult { + readonly repository: string; + readonly ref: string; + readonly marketplaceName: string; + readonly pluginId: string; +} + +const fail = ( + stage: CodexMarketplaceSmokeStage, + reason: CodexMarketplaceSmokeError["reason"], + detail: string, +): Effect.Effect => + Effect.fail(new CodexMarketplaceSmokeError({ stage, reason, detail })); + +const isJsonObject = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +const decodeJson = ( + stage: CodexMarketplaceSmokeStage, + output: string, +): Effect.Effect => { + const decoded = decodeUnknownJson(output); + return decoded._tag === "Some" + ? Effect.succeed(decoded.value) + : fail(stage, "invalid-json", `${stage} did not emit valid JSON`); +}; + +const stringProperty = ( + object: Readonly>, + names: readonly string[], +): string | undefined => { + for (const name of names) { + const value = object[name]; + if (typeof value === "string" && value.length > 0) return value; + } + return undefined; +}; + +export const parseCodexMarketplaceSmokeArgs = ( + argv: readonly string[], +): Effect.Effect => + Effect.gen(function* () { + let repository = DEFAULT_REPOSITORY; + let ref: string | undefined; + let executable = DEFAULT_EXECUTABLE; + let timeoutMs = DEFAULT_TIMEOUT_MS; + const seen = new Set(); + + for (let index = 0; index < argv.length; index += 2) { + const name = argv[index]; + const value = argv[index + 1]; + if ( + name === undefined || + value === undefined || + !["--repository", "--ref", "--executable", "--timeout-ms"].includes(name) || + seen.has(name) + ) { + return yield* fail("arguments", "invalid-arguments", "invalid marketplace smoke arguments"); + } + seen.add(name); + if (name === "--repository") repository = value; + else if (name === "--ref") ref = value; + else if (name === "--executable") executable = value; + else { + timeoutMs = Number(value); + } + } + + if (!REPOSITORY_RE.test(repository)) { + return yield* fail( + "arguments", + "invalid-arguments", + "--repository must be a remote owner/repo", + ); + } + if (ref === undefined || !IMMUTABLE_REF_RE.test(ref)) { + return yield* fail( + "arguments", + "invalid-arguments", + "--ref must be a full immutable 40-character commit SHA", + ); + } + if (executable.length === 0 || !Number.isSafeInteger(timeoutMs) || timeoutMs < 1_000) { + return yield* fail("arguments", "invalid-arguments", "invalid executable or timeout"); + } + return { repository, ref: ref.toLowerCase(), executable, timeoutMs }; + }); + +export const buildCodexMarketplaceEnvironment: { + ( + parentEnvironment: Readonly>, + ): (isolation: CodexMarketplaceIsolation) => Record; + ( + isolation: CodexMarketplaceIsolation, + parentEnvironment: Readonly>, + ): Record; +} = Function.dual( + 2, + ( + isolation: CodexMarketplaceIsolation, + parentEnvironment: Readonly>, + ): Record => { + const environment: Record = {}; + for (const name of CODEX_CLI_ALLOWED_ENVIRONMENT_NAMES) { + const value = parentEnvironment[name]; + if (value !== undefined) environment[name] = value; + } + return { + ...environment, + HOME: isolation.home, + CODEX_HOME: isolation.codexHome, + USERPROFILE: isolation.home, + XDG_CONFIG_HOME: isolation.configHome, + XDG_CACHE_HOME: isolation.cacheHome, + XDG_DATA_HOME: isolation.dataHome, + XDG_STATE_HOME: isolation.stateHome, + XDG_RUNTIME_DIR: isolation.runtimeDirectory, + GIT_CONFIG_NOSYSTEM: "1", + GIT_TERMINAL_PROMPT: "0", + NO_COLOR: "1", + }; + }, +); + +export const parseMarketplaceAddOutput = ( + output: string, +): Effect.Effect< + { readonly marketplaceName: string; readonly installedRoot: string }, + CodexMarketplaceSmokeError +> => + Effect.gen(function* () { + const decoded = yield* decodeJson("marketplace-add", output); + if (!isJsonObject(decoded)) { + return yield* fail( + "marketplace-add", + "invalid-output", + "marketplace add JSON must be an object", + ); + } + const marketplaceName = stringProperty(decoded, ["marketplaceName"]); + const installedRoot = stringProperty(decoded, ["installedRoot"]); + if ( + marketplaceName !== MARKETPLACE_NAME || + installedRoot === undefined || + decoded.alreadyAdded !== false + ) { + return yield* fail( + "marketplace-add", + "invalid-output", + `expected a fresh ${MARKETPLACE_NAME} marketplace and an installed root`, + ); + } + return { marketplaceName, installedRoot }; + }); + +export const parseMarketplaceListOutput: { + (marketplaceRoot: string): (output: string) => Effect.Effect; + (output: string, marketplaceRoot: string): Effect.Effect; +} = Function.dual(2, (output: string, marketplaceRoot: string) => + Effect.gen(function* () { + const decoded = yield* decodeJson("marketplace-list", output); + if ( + !isJsonObject(decoded) || + !Array.isArray(decoded.installed) || + !Array.isArray(decoded.available) || + decoded.installed.length !== 0 || + decoded.available.length !== 1 || + !isJsonObject(decoded.available[0]) + ) { + return yield* fail( + "marketplace-list", + "invalid-output", + "available listing must contain exactly one uninstalled Ask Gina plugin", + ); + } + const entry = decoded.available[0]; + if ( + entry.pluginId !== PLUGIN_ID || + entry.name !== PLUGIN_NAME || + entry.marketplaceName !== MARKETPLACE_NAME || + entry.installed !== false || + entry.enabled !== false || + typeof entry.version !== "string" || + entry.version.length === 0 + ) { + return yield* fail( + "marketplace-list", + "invalid-output", + `available listing did not contain ${PLUGIN_ID}`, + ); + } + if (entry.installPolicy !== "AVAILABLE") { + return yield* fail( + "marketplace-list", + "invalid-output", + "Ask Gina installation policy must be AVAILABLE", + ); + } + if (entry.authPolicy !== "ON_INSTALL") { + return yield* fail( + "marketplace-list", + "invalid-output", + "Ask Gina authentication listing metadata must be ON_INSTALL", + ); + } + const source = entry.source; + const sourcePath = + isJsonObject(source) && typeof source.path === "string" ? source.path : undefined; + const normalizedSourcePath = sourcePath?.replaceAll("\\", "/").replace(/\/+$/u, ""); + const normalizedMarketplaceRoot = marketplaceRoot.replaceAll("\\", "/").replace(/\/+$/u, ""); + const marketplaceSource = entry.marketplaceSource; + if ( + !isJsonObject(source) || + source.source !== "local" || + normalizedSourcePath !== `${normalizedMarketplaceRoot}/plugins/ask-gina` || + !isJsonObject(marketplaceSource) || + marketplaceSource.sourceType !== "git" || + typeof marketplaceSource.source !== "string" || + marketplaceSource.source.length === 0 + ) { + return yield* fail( + "marketplace-list", + "invalid-output", + "Ask Gina marketplace source must resolve to plugins/ask-gina", + ); + } + }), +); + +export const parsePluginAddOutput = ( + output: string, +): Effect.Effect<{ readonly installedPath: string }, CodexMarketplaceSmokeError> => + Effect.gen(function* () { + const decoded = yield* decodeJson("plugin-add", output); + if (!isJsonObject(decoded)) { + return yield* fail("plugin-add", "invalid-output", "plugin add JSON must be an object"); + } + const pluginId = stringProperty(decoded, ["pluginId"]); + const installedPath = stringProperty(decoded, ["installedPath"]); + if ( + pluginId !== PLUGIN_ID || + decoded.name !== PLUGIN_NAME || + decoded.marketplaceName !== MARKETPLACE_NAME || + typeof decoded.version !== "string" || + decoded.version.length === 0 || + decoded.authPolicy !== "ON_INSTALL" || + installedPath === undefined + ) { + return yield* fail( + "plugin-add", + "invalid-output", + `expected installed plugin ${PLUGIN_ID} with ON_INSTALL metadata and an installed path`, + ); + } + return { installedPath }; + }); + +const readJsonObject = ( + file: string, +): Effect.Effect< + Readonly>, + CodexMarketplaceSmokeError, + FileSystem.FileSystem +> => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const text = yield* fs.readFileString(file).pipe( + Effect.mapError( + () => + new CodexMarketplaceSmokeError({ + stage: "inspection", + reason: "io-failed", + detail: `cannot read installed file ${file}`, + }), + ), + ); + const decoded = yield* decodeJson("inspection", text); + if (!isJsonObject(decoded)) { + return yield* fail("inspection", "invalid-plugin", `${file} must contain a JSON object`); + } + return decoded; + }); + +const requireNonEmptyRegularFile = ( + file: string, +): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const info = yield* fs.stat(file).pipe( + Effect.mapError( + () => + new CodexMarketplaceSmokeError({ + stage: "inspection", + reason: "io-failed", + detail: `cannot inspect installed file ${file}`, + }), + ), + ); + if (info.type !== "File" || Number(info.size) <= 0) { + return yield* fail( + "inspection", + "invalid-plugin", + `${file} must be a non-empty regular file`, + ); + } + }); + +export const inspectInstalledAskGinaPlugin = ( + installedRoot: string, +): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const manifest = yield* readJsonObject( + path.join(installedRoot, ".codex-plugin", "plugin.json"), + ); + const pluginInterface = manifest.interface; + if ( + manifest.name !== PLUGIN_NAME || + manifest.skills !== "./skills/" || + manifest.mcpServers !== "./.mcp.json" || + !isJsonObject(pluginInterface) || + pluginInterface.composerIcon !== "./assets/icon.svg" || + pluginInterface.logo !== "./assets/icon.svg" + ) { + return yield* fail( + "inspection", + "invalid-plugin", + "installed Codex manifest does not match the Ask Gina root contract", + ); + } + + const mcp = yield* readJsonObject(path.join(installedRoot, ".mcp.json")); + const mcpServers = mcp.mcpServers; + const server = isJsonObject(mcpServers) ? mcpServers[PLUGIN_NAME] : undefined; + if ( + !isJsonObject(mcpServers) || + Object.keys(mcpServers).length !== 1 || + !isJsonObject(server) || + server.type !== "http" || + server.url !== PRODUCTION_MCP_URL + ) { + return yield* fail( + "inspection", + "invalid-plugin", + "installed root MCP configuration must contain only the ask-gina HTTP server", + ); + } + + yield* requireNonEmptyRegularFile(path.join(installedRoot, "assets", "icon.svg")); + const skillsRoot = path.join(installedRoot, "skills"); + const actualSkills = (yield* fs.readDirectory(skillsRoot).pipe( + Effect.mapError( + () => + new CodexMarketplaceSmokeError({ + stage: "inspection", + reason: "io-failed", + detail: "cannot list installed Ask Gina skills", + }), + ), + )).sort(); + const expectedSkills = ASK_GINA_SKILL_DEFINITIONS.map((skill) => skill.name).sort(); + if ( + actualSkills.length !== expectedSkills.length || + !actualSkills.every((skill, index) => skill === expectedSkills[index]) + ) { + return yield* fail( + "inspection", + "invalid-plugin", + "installed Ask Gina skill inventory does not match the canonical catalog", + ); + } + yield* Effect.forEach( + expectedSkills, + (skill) => + Effect.all([ + requireNonEmptyRegularFile(path.join(skillsRoot, skill, "SKILL.md")), + requireNonEmptyRegularFile(path.join(skillsRoot, skill, "agents", "openai.yaml")), + ]), + { concurrency: "unbounded", discard: true }, + ); + }); + +const commandDiagnostic = (result: MarketplaceCommandResult): string => { + const diagnostic = result.stderr.trim() || result.stdout.trim() || "no diagnostic output"; + return diagnostic.replaceAll(/\s+/gu, " ").slice(0, 2_000); +}; + +export const effectMarketplaceCommandRunner: MarketplaceCommandRunner = { + run: (input) => + Effect.scoped( + Effect.gen(function* () { + const child = yield* ChildProcess.make(input.executable, input.args, { + cwd: input.cwd, + env: { ...input.environment }, + extendEnv: false, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }).pipe( + Effect.mapError( + () => + new CodexMarketplaceSmokeError({ + stage: input.stage, + reason: "command-failed", + detail: `${input.stage} could not start`, + }), + ), + ); + const [stdout, stderr, exitCode] = yield* Effect.all( + [ + collectBoundedUtf8Output(child.stdout, MAXIMUM_OUTPUT_BYTES), + collectBoundedUtf8Output(child.stderr, MAXIMUM_OUTPUT_BYTES), + child.exitCode, + ], + { concurrency: "unbounded" }, + ).pipe( + Effect.mapError( + () => + new CodexMarketplaceSmokeError({ + stage: input.stage, + reason: "command-failed", + detail: `${input.stage} output could not be collected`, + }), + ), + ); + const result = { stdout: stdout.text, stderr: stderr.text }; + if (exitCode !== 0 || stdout.truncated || stderr.truncated) { + return yield* fail( + input.stage, + "command-failed", + `${input.stage} failed (exit ${exitCode}): ${commandDiagnostic(result)}`, + ); + } + return result; + }).pipe( + Effect.timeoutOrElse({ + duration: Duration.millis(input.timeoutMs), + orElse: () => + fail(input.stage, "timeout", `${input.stage} exceeded ${input.timeoutMs}ms`), + }), + ), + ), +}; + +const isWithin = (path: Path.Path, parent: string, child: string): boolean => { + const relative = path.relative(parent, child); + return relative !== "" && !relative.startsWith("..") && !path.isAbsolute(relative); +}; + +export const runCodexMarketplaceSmoke = ( + options: CodexMarketplaceSmokeRunOptions, +): Effect.Effect< + CodexMarketplaceSmokeResult, + CodexMarketplaceSmokeError, + ChildProcessSpawner | FileSystem.FileSystem | Path.Path +> => { + const runner = options.runner ?? effectMarketplaceCommandRunner; + const parentEnvironment = options.parentEnvironment ?? process.env; + return Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs + .makeTempDirectoryScoped({ prefix: "ask-gina-codex-marketplace-" }) + .pipe( + Effect.mapError( + () => + new CodexMarketplaceSmokeError({ + stage: "environment", + reason: "io-failed", + detail: "cannot create isolated marketplace directory", + }), + ), + ); + const isolation: CodexMarketplaceIsolation = { + root, + home: path.join(root, "home"), + codexHome: path.join(root, "codex"), + configHome: path.join(root, "xdg", "config"), + cacheHome: path.join(root, "xdg", "cache"), + dataHome: path.join(root, "xdg", "data"), + stateHome: path.join(root, "xdg", "state"), + runtimeDirectory: path.join(root, "xdg", "runtime"), + workingDirectory: path.join(root, "work"), + }; + yield* Effect.forEach( + [ + isolation.home, + isolation.codexHome, + isolation.configHome, + isolation.cacheHome, + isolation.dataHome, + isolation.stateHome, + isolation.runtimeDirectory, + isolation.workingDirectory, + ], + (directory) => fs.makeDirectory(directory, { recursive: true }), + { concurrency: "unbounded", discard: true }, + ).pipe( + Effect.mapError( + () => + new CodexMarketplaceSmokeError({ + stage: "environment", + reason: "io-failed", + detail: "cannot initialize isolated marketplace directories", + }), + ), + ); + yield* fs + .writeFileString(path.join(isolation.codexHome, "config.toml"), "", { + flag: "wx", + mode: 0o600, + }) + .pipe( + Effect.mapError( + () => + new CodexMarketplaceSmokeError({ + stage: "environment", + reason: "io-failed", + detail: "cannot initialize isolated Codex configuration", + }), + ), + ); + const environment = buildCodexMarketplaceEnvironment(isolation, parentEnvironment); + const runCommand = ( + stage: MarketplaceCommandInput["stage"], + args: readonly string[], + ): Effect.Effect< + MarketplaceCommandResult, + CodexMarketplaceSmokeError, + ChildProcessSpawner | FileSystem.FileSystem | Path.Path + > => + runner.run({ + stage, + executable: options.executable, + args, + cwd: isolation.workingDirectory, + environment, + timeoutMs: options.timeoutMs, + }); + const cleanup = Effect.gen(function* () { + yield* runCommand("plugin-remove", ["plugin", "remove", PLUGIN_ID, "--json"]).pipe( + Effect.ignore, + ); + yield* runCommand("marketplace-remove", [ + "plugin", + "marketplace", + "remove", + MARKETPLACE_NAME, + "--json", + ]).pipe(Effect.ignore); + }); + + return yield* Effect.gen(function* () { + const marketplaceResult = yield* runCommand("marketplace-add", [ + "plugin", + "marketplace", + "add", + options.repository, + "--ref", + options.ref, + "--json", + ]); + const marketplace = yield* parseMarketplaceAddOutput(marketplaceResult.stdout); + const marketplaceRoot = yield* fs.realPath(marketplace.installedRoot).pipe( + Effect.mapError( + () => + new CodexMarketplaceSmokeError({ + stage: "marketplace-add", + reason: "invalid-output", + detail: "marketplace installed root cannot be resolved", + }), + ), + ); + if (!isWithin(path, root, marketplaceRoot)) { + return yield* fail( + "marketplace-add", + "invalid-output", + "marketplace installed outside the isolated temporary root", + ); + } + + const listResult = yield* runCommand("marketplace-list", [ + "plugin", + "list", + "--marketplace", + MARKETPLACE_NAME, + "--available", + "--json", + ]); + yield* parseMarketplaceListOutput(listResult.stdout, marketplaceRoot); + + const installResult = yield* runCommand("plugin-add", [ + "plugin", + "add", + PLUGIN_ID, + "--json", + ]); + const install = yield* parsePluginAddOutput(installResult.stdout); + const installedRoot = yield* fs.realPath(install.installedPath).pipe( + Effect.mapError( + () => + new CodexMarketplaceSmokeError({ + stage: "plugin-add", + reason: "invalid-output", + detail: "plugin installed path cannot be resolved", + }), + ), + ); + if (!isWithin(path, isolation.codexHome, installedRoot)) { + return yield* fail( + "plugin-add", + "invalid-output", + "plugin installed outside the isolated Codex home", + ); + } + yield* inspectInstalledAskGinaPlugin(installedRoot); + return { + repository: options.repository, + ref: options.ref, + marketplaceName: MARKETPLACE_NAME, + pluginId: PLUGIN_ID, + }; + }).pipe(Effect.ensuring(cleanup)); + }), + ); +}; + +const usage = + "usage: check-codex-marketplace --ref <40-character-sha> [--repository owner/repo] [--executable path] [--timeout-ms ms]"; + +const program = parseCodexMarketplaceSmokeArgs(process.argv.slice(2)).pipe( + Effect.flatMap(runCodexMarketplaceSmoke), + Effect.matchEffect({ + onFailure: (error) => + Console.error( + error.stage === "arguments" + ? usage + : `Codex marketplace smoke failed at ${error.stage}: ${error.detail}`, + ).pipe(Effect.as(1)), + onSuccess: (result) => + Console.log( + JSON.stringify({ + ok: true, + repository: result.repository, + ref: result.ref, + marketplaceName: result.marketplaceName, + pluginId: result.pluginId, + }), + ).pipe(Effect.as(0)), + }), + Effect.tap((exitCode) => + Effect.sync(() => { + if (exitCode !== 0) process.exitCode = exitCode; + }), + ), +); + +const main = Layer.build(BunServices.layer).pipe( + Effect.flatMap((context) => program.pipe(Effect.provide(context))), + Effect.scoped, +); + +if (import.meta.main) { + BunRuntime.runMain(main); +} diff --git a/packages/evals/src/bin/live.ts b/packages/evals/src/bin/live.ts index 31f3886..8574f97 100755 --- a/packages/evals/src/bin/live.ts +++ b/packages/evals/src/bin/live.ts @@ -48,7 +48,7 @@ const GIT_STATUS_LIMIT_BYTES = 65_536; const CODEX_PREFLIGHT_LIMIT_BYTES = 1_048_576; const CODEX_MARKETPLACE_NAME = "ask-gina-plugins"; const CODEX_PLUGIN_ID = `ask-gina@${CODEX_MARKETPLACE_NAME}`; -const CODEX_MCP_SERVER_NAME = "gina"; +const CODEX_MCP_SERVER_NAME = "ask-gina"; const JsonObjectString = Schema.fromJsonString(Schema.JsonObject); const UnknownJsonString = Schema.fromJsonString(Schema.Unknown); const PrettyUnknownJsonString = Schema.fromJsonString(Schema.Unknown, { space: 2 }); @@ -390,25 +390,23 @@ const setupCodexRuntime = ( ); if ( pluginManifest?.name !== "ask-gina" || - pluginManifest.skills !== "./skills" || - pluginManifest.mcpServers !== "./targets/claude/.mcp.json" + pluginManifest.skills !== "./skills/" || + pluginManifest.mcpServers !== "./.mcp.json" ) { return yield* new LiveEvalCliError({ reason: "codex-preflight-failed" }); } - const mcpConfig = yield* fs - .readFileString(path.join(installedRoot, "targets", "claude", ".mcp.json")) - .pipe( - Effect.map(parseJsonObject), - Effect.mapError(() => new LiveEvalCliError({ reason: "codex-preflight-failed" })), - ); + const mcpConfig = yield* fs.readFileString(path.join(installedRoot, ".mcp.json")).pipe( + Effect.map(parseJsonObject), + Effect.mapError(() => new LiveEvalCliError({ reason: "codex-preflight-failed" })), + ); const mcpServers = mcpConfig?.mcpServers; - const ginaServer = isJsonObject(mcpServers) ? mcpServers[CODEX_MCP_SERVER_NAME] : undefined; + const askGinaServer = isJsonObject(mcpServers) ? mcpServers[CODEX_MCP_SERVER_NAME] : undefined; if ( !isJsonObject(mcpServers) || - !isJsonObject(ginaServer) || + !isJsonObject(askGinaServer) || Object.keys(mcpServers).length !== 1 || - ginaServer.type !== "http" || - ginaServer.url !== PRODUCTION_MCP_URL + askGinaServer.type !== "http" || + askGinaServer.url !== PRODUCTION_MCP_URL ) { return yield* new LiveEvalCliError({ reason: "codex-preflight-failed" }); } diff --git a/packages/evals/src/codex-cli.ts b/packages/evals/src/codex-cli.ts index 80e502f..186ad10 100644 --- a/packages/evals/src/codex-cli.ts +++ b/packages/evals/src/codex-cli.ts @@ -49,11 +49,7 @@ const ASK_GINA_SKILL_NAMES: Readonly> = { - gina: true, - "ask-gina": true, -}; +const ASK_GINA_MCP_SERVER = "ask-gina"; const CODEX_NON_ACTION_ITEM_TYPES: Readonly> = { agent_message: true, reasoning: true, @@ -623,8 +619,7 @@ const isMcpToolItem = (item: Readonly>): boolean => { const server = asString(item.server); return ( type === "mcp_tool_call" && - server !== undefined && - Object.hasOwn(ASK_GINA_MCP_SERVER_NAMES, server) && + server === ASK_GINA_MCP_SERVER && name !== undefined && isGinaReadToolName(name) ); diff --git a/plugins/ask-gina/targets/openai/.codex-plugin/plugin.json b/plugins/ask-gina/.codex-plugin/plugin.json similarity index 100% rename from plugins/ask-gina/targets/openai/.codex-plugin/plugin.json rename to plugins/ask-gina/.codex-plugin/plugin.json diff --git a/plugins/ask-gina/targets/openai/.mcp.json b/plugins/ask-gina/.mcp.json similarity index 100% rename from plugins/ask-gina/targets/openai/.mcp.json rename to plugins/ask-gina/.mcp.json diff --git a/plugins/ask-gina/__tests__/plugin-core.test.ts b/plugins/ask-gina/__tests__/plugin-core.test.ts index de8af66..a46f263 100644 --- a/plugins/ask-gina/__tests__/plugin-core.test.ts +++ b/plugins/ask-gina/__tests__/plugin-core.test.ts @@ -16,7 +16,7 @@ import { const pluginRoot = fileURLToPath(new URL("../", import.meta.url)); const repositoryRoot = fileURLToPath(new URL("../../../", import.meta.url)); -const MarketplaceJson = Schema.fromJsonString( +const ClaudeMarketplaceJson = Schema.fromJsonString( Schema.Struct({ name: Schema.String, owner: Schema.Struct({ name: Schema.String, url: Schema.String }), @@ -35,6 +35,20 @@ const MarketplaceJson = Schema.fromJsonString( }), ); +const OpenAiMarketplaceJson = Schema.fromJsonString( + Schema.Struct({ + name: Schema.String, + interface: Schema.Struct({ displayName: Schema.String }), + plugins: Schema.Array( + Schema.Struct({ + name: Schema.String, + source: Schema.Struct({ source: Schema.String, path: Schema.String }), + policy: Schema.Struct({ installation: Schema.String, authentication: Schema.String }), + category: Schema.String, + }), + ), + }), +); const collectSkillDocuments = ( directory: string, ): Effect.Effect< @@ -101,11 +115,41 @@ describe("Ask Gina portable plugin core", () => { }), ); + it.effect("publishes OpenAI marketplace metadata for the loadable root plugin", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const paths = yield* Path.Path; + const marketplace = yield* Schema.decodeEffect(OpenAiMarketplaceJson)( + yield* fs.readFileString( + paths.join(repositoryRoot, ".agents", "plugins", "marketplace.json"), + ), + ); + + assert.strictEqual(marketplace.name, "ask-gina-plugins"); + assert.deepStrictEqual(marketplace.interface, { displayName: "Ask Gina Plugins" }); + assert.strictEqual(marketplace.plugins.length, 1); + const plugin = marketplace.plugins[0]; + assert.isDefined(plugin); + assert.deepStrictEqual(plugin, { + name: "ask-gina", + source: { source: "local", path: "./plugins/ask-gina" }, + policy: { installation: "AVAILABLE", authentication: "ON_INSTALL" }, + category: "Finance", + }); + const sourceRoot = paths.resolve(repositoryRoot, plugin?.source.path ?? ""); + assert.strictEqual(sourceRoot, paths.resolve(pluginRoot)); + assert.isTrue(yield* fs.exists(paths.join(sourceRoot, ".codex-plugin", "plugin.json"))); + assert.isTrue(yield* fs.exists(paths.join(sourceRoot, ".mcp.json"))); + assert.isTrue(yield* fs.exists(paths.join(sourceRoot, "assets", "icon.svg"))); + assert.isTrue(yield* fs.exists(paths.join(sourceRoot, "skills"))); + }), + ); + it.effect("publishes Claude marketplace metadata for the canonical plugin directory", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const paths = yield* Path.Path; - const marketplace = yield* Schema.decodeEffect(MarketplaceJson)( + const marketplace = yield* Schema.decodeEffect(ClaudeMarketplaceJson)( yield* fs.readFileString( paths.join(repositoryRoot, ".claude-plugin", "marketplace.json"), ), diff --git a/plugins/ask-gina/__tests__/sync-plugin-skills.test.ts b/plugins/ask-gina/__tests__/sync-plugin-skills.test.ts index 8f58610..20bdd1e 100644 --- a/plugins/ask-gina/__tests__/sync-plugin-skills.test.ts +++ b/plugins/ask-gina/__tests__/sync-plugin-skills.test.ts @@ -3,6 +3,7 @@ import { assert, describe, it } from "@effect/vitest"; import { Effect, FileSystem, Path } from "effect"; import { + createGeneratedPluginTarget, createGeneratedPluginTargets, SKILL_NAMES, TARGET_NAMES, @@ -15,69 +16,117 @@ const makeGeneratedTargets = Effect.acquireRelease( describe("sync-plugin-skills", () => { it.layer(BunServices.layer)((it) => { - it.effect( - "copies every canonical skill into every host and keeps the OpenAI overlay isolated", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const paths = yield* Path.Path; - const generatedTargets = yield* makeGeneratedTargets; - const canonicalSkillsRoot = paths.resolve("plugins/ask-gina/skills"); + it.effect("loads OpenAI from the plugin root and retains non-OpenAI overlays", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const paths = yield* Path.Path; + const generatedTargets = yield* makeGeneratedTargets; + const packageRoot = paths.resolve("plugins/ask-gina"); + const canonicalSkillsRoot = paths.join(packageRoot, "skills"); - for (const hostName of TARGET_NAMES) { - for (const skillName of SKILL_NAMES) { - const canonicalSkillPath = paths.join(canonicalSkillsRoot, skillName, "SKILL.md"); - const generatedSkillPath = paths.join( - generatedTargets.targets[hostName], - "skills", - skillName, - "SKILL.md", - ); + assert.isTrue(yield* fs.exists(paths.join(packageRoot, ".codex-plugin", "plugin.json"))); + assert.isTrue(yield* fs.exists(paths.join(packageRoot, ".mcp.json"))); + assert.isTrue(yield* fs.exists(paths.join(packageRoot, "assets", "icon.svg"))); + assert.isFalse(yield* fs.exists(paths.join(packageRoot, "targets", "openai"))); + assert.deepStrictEqual( + (yield* fs.readDirectory(generatedTargets.targets.openai)).sort(), + [".codex-plugin", ".mcp.json", "assets", "skills"], + ); - assert.isTrue( - yield* fs.exists(generatedSkillPath), - `${hostName} must include ${skillName}/SKILL.md`, - ); - assert.strictEqual( - yield* fs.readFileString(generatedSkillPath), - yield* fs.readFileString(canonicalSkillPath), - `${hostName} must preserve canonical ${skillName}/SKILL.md`, - ); - } - } + for (const relative of [ + [".codex-plugin", "plugin.json"], + [".mcp.json"], + ["assets", "icon.svg"], + ] as const) { + assert.strictEqual( + yield* fs.readFileString(paths.join(generatedTargets.targets.openai, ...relative)), + yield* fs.readFileString(paths.join(packageRoot, ...relative)), + ); + } + for (const hostName of TARGET_NAMES.filter((hostName) => hostName !== "openai")) { + assert.deepStrictEqual( + (yield* fs.readDirectory(generatedTargets.targets[hostName])) + .filter((entry) => entry !== "skills") + .sort(), + (yield* fs.readDirectory(paths.join(packageRoot, "targets", hostName))).sort(), + ); + } + + for (const hostName of TARGET_NAMES) { for (const skillName of SKILL_NAMES) { + const canonicalSkillPath = paths.join(canonicalSkillsRoot, skillName, "SKILL.md"); + const generatedSkillPath = paths.join( + generatedTargets.targets[hostName], + "skills", + skillName, + "SKILL.md", + ); + assert.isTrue( + yield* fs.exists(generatedSkillPath), + `${hostName} must include ${skillName}/SKILL.md`, + ); + assert.strictEqual( + yield* fs.readFileString(generatedSkillPath), + yield* fs.readFileString(canonicalSkillPath), + `${hostName} must preserve canonical ${skillName}/SKILL.md`, + ); + } + } + + for (const skillName of SKILL_NAMES) { + assert.isTrue( + yield* fs.exists( + paths.join( + generatedTargets.targets.openai, + "skills", + skillName, + "agents", + "openai.yaml", + ), + ), + `OpenAI must include ${skillName}/agents/openai.yaml`, + ); + + for (const hostName of TARGET_NAMES.filter((hostName) => hostName !== "openai")) { + assert.isFalse( yield* fs.exists( paths.join( - generatedTargets.targets.openai, + generatedTargets.targets[hostName], "skills", skillName, "agents", "openai.yaml", ), ), - `OpenAI must include ${skillName}/agents/openai.yaml`, + `${hostName} must not include the OpenAI-only overlay for ${skillName}`, ); - - for (const hostName of TARGET_NAMES.filter((hostName) => hostName !== "openai")) { - assert.isFalse( - yield* fs.exists( - paths.join( - generatedTargets.targets[hostName], - "skills", - skillName, - "agents", - "openai.yaml", - ), - ), - `${hostName} must not include the OpenAI-only overlay for ${skillName}`, - ); - } } - }), - ), + } + }), + ), + ); + + it.effect("rejects a legacy OpenAI target overlay", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const paths = yield* Path.Path; + const temporary = yield* fs.makeTempDirectoryScoped({ + prefix: "sync-plugin-skills-test-", + }); + const packageRoot = paths.join(temporary, "ask-gina"); + yield* fs.copy(paths.resolve("plugins/ask-gina"), packageRoot, { overwrite: true }); + const legacyOverlay = paths.join(packageRoot, "targets", "openai"); + yield* fs.makeDirectory(legacyOverlay, { recursive: true }); + + const error = yield* Effect.flip(createGeneratedPluginTarget("openai", { packageRoot })); + assert.strictEqual(error.path, legacyOverlay); + assert.include(error.reason, "legacy OpenAI target overlay"); + }), + ), ); }); }); diff --git a/plugins/ask-gina/targets/openai/assets/icon.svg b/plugins/ask-gina/assets/icon.svg similarity index 100% rename from plugins/ask-gina/targets/openai/assets/icon.svg rename to plugins/ask-gina/assets/icon.svg diff --git a/tools/__tests__/pack-artifacts.test.ts b/tools/__tests__/pack-artifacts.test.ts index 5715043..67a0dcc 100644 --- a/tools/__tests__/pack-artifacts.test.ts +++ b/tools/__tests__/pack-artifacts.test.ts @@ -7,6 +7,8 @@ import { ArtifactPackError, buildArtifacts, stagePackage, + stagePluginTarget, + validateTargetVersion, verifyCompiledPackageOutput, } from "../pack-artifacts"; class TestCommandError extends Data.TaggedError("TestCommandError")<{ @@ -15,6 +17,65 @@ class TestCommandError extends Data.TaggedError("TestCommandError")<{ }> {} const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); +const SKILL_NAMES = [ + "research-hyperliquid", + "research-prediction-markets", + "research-spot-tokens", + "review-gina-account", +] as const; +const TARGET_MANIFESTS = { + cursor: [".cursor-plugin", "plugin.json"], + claude: [".claude-plugin", "plugin.json"], + copilot: ["plugin.json"], + gemini: ["gemini-extension.json"], +} as const; + +const makePluginFixture = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const temporary = yield* fs.makeTempDirectoryScoped({ + prefix: "pack-plugin-test-", + }); + const plugin = path.join(temporary, "ask-gina"); + const stage = path.join(temporary, "stage"); + const version = "1.2.3"; + + yield* fs.makeDirectory(path.join(plugin, ".codex-plugin"), { recursive: true }); + yield* fs.writeFileString( + path.join(plugin, ".codex-plugin", "plugin.json"), + `${encodeJson({ version, marker: "root-openai" })}\n`, + ); + yield* fs.writeFileString(path.join(plugin, ".mcp.json"), '{"mcpServers":{}}\n'); + yield* fs.makeDirectory(path.join(plugin, "assets"), { recursive: true }); + yield* fs.writeFileString(path.join(plugin, "assets", "icon.svg"), "\n"); + for (const skill of SKILL_NAMES) { + const skillRoot = path.join(plugin, "skills", skill); + yield* fs.makeDirectory(path.join(skillRoot, "agents"), { recursive: true }); + yield* fs.writeFileString(path.join(skillRoot, "SKILL.md"), `# ${skill}\n`); + yield* fs.writeFileString(path.join(skillRoot, "agents", "openai.yaml"), `name: ${skill}\n`); + } + + for (const [host, manifest] of Object.entries(TARGET_MANIFESTS)) { + const overlay = path.join(plugin, "targets", host); + const manifestPath = path.join(overlay, ...manifest); + yield* fs.makeDirectory(path.dirname(manifestPath), { recursive: true }); + yield* fs.writeFileString( + manifestPath, + `${encodeJson({ version, marker: `${host}-overlay` })}\n`, + ); + yield* fs.writeFileString(path.join(overlay, `${host}.txt`), `${host}\n`); + } + + for (const relative of ["package.json", "plugin.yaml", "README.md", "LICENSE"] as const) { + yield* fs.writeFileString(path.join(plugin, relative), `${relative}\n`); + } + for (const directory of ["src", "evals"] as const) { + yield* fs.makeDirectory(path.join(plugin, directory), { recursive: true }); + yield* fs.writeFileString(path.join(plugin, directory, "foreign.txt"), `${directory}\n`); + } + + return { plugin, stage, version }; +}); const json = (value: unknown): string => `${encodeJson(value)}\n`; const SOURCE_DIRECTORIES = [ @@ -324,3 +385,113 @@ describe("pack artifact source snapshot", () => { ); }); }); + +describe("plugin target packing", () => { + it.layer(BunServices.layer)((it) => { + it.effect("validates OpenAI at the root and other hosts in their target overlays", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const fixture = yield* makePluginFixture; + const legacyManifest = path.join( + fixture.plugin, + "targets", + "openai", + ".codex-plugin", + "plugin.json", + ); + yield* fs.makeDirectory(path.dirname(legacyManifest), { recursive: true }); + yield* fs.writeFileString(legacyManifest, `${encodeJson({ version: "9.9.9" })}\n`); + + yield* validateTargetVersion(fixture.plugin, "openai", fixture.version); + yield* fs.writeFileString( + path.join(fixture.plugin, ".codex-plugin", "plugin.json"), + `${encodeJson({ version: "8.8.8" })}\n`, + ); + yield* validateTargetVersion(fixture.plugin, "cursor", fixture.version); + + yield* fs.remove(path.join(fixture.plugin, ".codex-plugin", "plugin.json")); + const error = yield* validateTargetVersion(fixture.plugin, "openai", "9.9.9").pipe( + Effect.flip, + ); + assert.instanceOf(error, ArtifactPackError); + assert.include(error.message, path.join(".codex-plugin", "plugin.json")); + }), + ), + ); + + it.effect("stages a lean OpenAI target from root source surfaces", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const fixture = yield* makePluginFixture; + + yield* stagePluginTarget("openai", fixture.plugin, fixture.stage); + + assert.deepStrictEqual((yield* fs.readDirectory(fixture.stage)).sort(), [ + ".codex-plugin", + ".mcp.json", + "assets", + "skills", + ]); + for (const relative of [ + [".codex-plugin", "plugin.json"], + [".mcp.json"], + ["assets", "icon.svg"], + ] as const) { + assert.strictEqual( + yield* fs.readFileString(path.join(fixture.stage, ...relative)), + yield* fs.readFileString(path.join(fixture.plugin, ...relative)), + ); + } + for (const excluded of [ + "package.json", + "plugin.yaml", + "README.md", + "LICENSE", + "src", + "evals", + "targets", + ]) { + assert.isFalse(yield* fs.exists(path.join(fixture.stage, excluded))); + } + for (const skill of SKILL_NAMES) { + assert.isTrue( + yield* fs.exists(path.join(fixture.stage, "skills", skill, "agents", "openai.yaml")), + ); + } + }), + ), + ); + + it.effect("retains non-OpenAI overlay staging without OpenAI skill metadata", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const fixture = yield* makePluginFixture; + + yield* stagePluginTarget("cursor", fixture.plugin, fixture.stage); + + assert.deepStrictEqual((yield* fs.readDirectory(fixture.stage)).sort(), [ + ".cursor-plugin", + "cursor.txt", + "skills", + ]); + for (const relative of [[".cursor-plugin", "plugin.json"], ["cursor.txt"]] as const) { + assert.strictEqual( + yield* fs.readFileString(path.join(fixture.stage, ...relative)), + yield* fs.readFileString(path.join(fixture.plugin, "targets", "cursor", ...relative)), + ); + } + for (const skill of SKILL_NAMES) { + assert.isTrue(yield* fs.exists(path.join(fixture.stage, "skills", skill, "SKILL.md"))); + assert.isFalse(yield* fs.exists(path.join(fixture.stage, "skills", skill, "agents"))); + } + }), + ), + ); + }); +}); diff --git a/tools/__tests__/verify-artifacts.test.ts b/tools/__tests__/verify-artifacts.test.ts index 58e0295..9dc4e03 100644 --- a/tools/__tests__/verify-artifacts.test.ts +++ b/tools/__tests__/verify-artifacts.test.ts @@ -1,17 +1,95 @@ +import { fileURLToPath } from "node:url"; + import * as BunServices from "@effect/platform-bun/BunServices"; import { assert, describe, it } from "@effect/vitest"; import { ChildProcess } from "effect/unstable/process"; -import { Config, Effect, FileSystem, Path } from "effect"; +import { Config, Effect, FileSystem, Path, Schema } from "effect"; import { findEmbeddedSourceMapBoundaryRules, inspectSourceMapText } from "../check-public-boundary"; +import { checkRepositoryConformance, type RepositorySummary } from "../check-target-conformance"; import { + ArtifactVerificationError, runNodeEsmSmoke, snapshotArtifactInputs, verifyNoInstalledLibrarySources, verifyNode24Consumer, + verifyOpenAiArchivePayload, } from "../verify-artifacts"; -describe("artifact verification snapshots", () => { +const repositoryRoot = fileURLToPath(new URL("../../", import.meta.url)); +const SKILLS = [ + "research-hyperliquid", + "research-prediction-markets", + "research-spot-tokens", + "review-gina-account", +] as const; +const UnknownJsonString = Schema.fromJsonString(Schema.Unknown); +const decodeUnknownJson = Schema.decodeUnknownSync(UnknownJsonString); +const encodeUnknownJson = Schema.encodeUnknownSync(UnknownJsonString); + +const repositoryFixture = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "repository-conformance-test-" }); + yield* Effect.all( + [ + fs.copy(path.join(repositoryRoot, ".agents"), path.join(root, ".agents")), + fs.copy(path.join(repositoryRoot, ".claude-plugin"), path.join(root, ".claude-plugin")), + fs.copy( + path.join(repositoryRoot, "plugins", "ask-gina"), + path.join(root, "plugins", "ask-gina"), + ), + ], + { concurrency: "unbounded" }, + ); + return { root, packageRoot: path.join(root, "plugins", "ask-gina") }; +}); + +const readMarketplace = (root: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + return decodeUnknownJson( + yield* fs.readFileString(path.join(root, ".agents", "plugins", "marketplace.json")), + ) as Record; + }); + +const writeMarketplace = (root: string, marketplace: Record) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* fs.writeFileString( + path.join(root, ".agents", "plugins", "marketplace.json"), + `${encodeUnknownJson(marketplace)}\n`, + ); + }); + +const failedCheck = (report: RepositorySummary, id: string): boolean => + report.checks.some((check) => check.id === id && !check.passed); + +const makeLeanOpenAiPayload = (root: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const files = [ + ".codex-plugin/plugin.json", + ".mcp.json", + "assets/icon.svg", + ...SKILLS.flatMap((skill) => [ + `skills/${skill}/SKILL.md`, + `skills/${skill}/agents/openai.yaml`, + ]), + ]; + yield* Effect.forEach(files, (file) => + Effect.gen(function* () { + const destination = path.join(root, file); + yield* fs.makeDirectory(path.dirname(destination), { recursive: true }); + yield* fs.writeFileString(destination, `${file}\n`); + }), + ); + }); + +describe("artifact and source conformance verification", () => { it.layer(BunServices.layer)((it) => { it.effect("executes only snapshotted bytes after canonical comparison", () => Effect.scoped( @@ -170,5 +248,158 @@ describe("artifact verification snapshots", () => { )?.unsafeSourcePath, ); }); + it.effect("accepts the exact repository marketplace and clean root plugin layout", () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* repositoryFixture; + const report = yield* checkRepositoryConformance({ + repositoryRoot: fixture.root, + packageRoot: fixture.packageRoot, + }); + assert.isTrue(report.passed); + }), + ), + ); + + it.effect("rejects malformed marketplace policy, category, source, escape, and version", () => + Effect.gen(function* () { + const cases: readonly Readonly<{ + readonly expectedCheck: string; + readonly mutate: (marketplace: Record) => void; + }>[] = [ + { + expectedCheck: "marketplace.schema", + mutate: (marketplace) => { + const plugin = (marketplace.plugins as Record[])[0]; + if (plugin !== undefined) { + plugin.policy = { installation: "ENABLED", authentication: "ON_INSTALL" }; + } + }, + }, + { + expectedCheck: "marketplace.schema", + mutate: (marketplace) => { + const plugin = (marketplace.plugins as Record[])[0]; + if (plugin !== undefined) plugin.category = "Other"; + }, + }, + { + expectedCheck: "marketplace.schema", + mutate: (marketplace) => { + const plugin = (marketplace.plugins as Record[])[0]; + if (plugin !== undefined) plugin.source = "./plugins/ask-gina"; + }, + }, + { + expectedCheck: "marketplace.source_containment", + mutate: (marketplace) => { + const plugin = (marketplace.plugins as Record[])[0]; + if (plugin !== undefined) { + plugin.source = { source: "local", path: "../outside" }; + } + }, + }, + { + expectedCheck: "marketplace.no_version", + mutate: (marketplace) => { + marketplace.version = "0.1.0"; + }, + }, + ]; + + yield* Effect.forEach(cases, ({ expectedCheck, mutate }) => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* repositoryFixture; + const marketplace = yield* readMarketplace(fixture.root); + mutate(marketplace); + yield* writeMarketplace(fixture.root, marketplace); + const report = yield* checkRepositoryConformance({ + repositoryRoot: fixture.root, + packageRoot: fixture.packageRoot, + }); + assert.isTrue(failedCheck(report, expectedCheck)); + }), + ), + ); + }), + ); + + it.effect("rejects missing root files, version drift, and a legacy OpenAI overlay", () => + Effect.forEach(["missing-mcp", "version-drift", "legacy-overlay"] as const, (mutation) => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const fixture = yield* repositoryFixture; + if (mutation === "missing-mcp") { + yield* fs.remove(path.join(fixture.packageRoot, ".mcp.json")); + } else if (mutation === "version-drift") { + const manifestPath = path.join(fixture.packageRoot, ".codex-plugin", "plugin.json"); + const manifest = decodeUnknownJson(yield* fs.readFileString(manifestPath)) as Record< + string, + unknown + >; + manifest.version = "99.0.0"; + yield* fs.writeFileString(manifestPath, `${encodeUnknownJson(manifest)}\n`); + } else { + yield* fs.makeDirectory(path.join(fixture.packageRoot, "targets", "openai"), { + recursive: true, + }); + } + const report = yield* checkRepositoryConformance({ + repositoryRoot: fixture.root, + packageRoot: fixture.packageRoot, + }); + const expected = + mutation === "missing-mcp" + ? "repository.root_openai.mcp_exists" + : mutation === "version-drift" + ? "repository.root_openai.manifest_contract" + : "repository.legacy_openai.absent"; + assert.isTrue(failedCheck(report, expected)); + }), + ), + ), + ); + + it.effect("rejects symbolic links in the loadable plugin source", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const fixture = yield* repositoryFixture; + const external = path.join(fixture.root, "external-icon.svg"); + const icon = path.join(fixture.packageRoot, "assets", "icon.svg"); + yield* fs.writeFileString(external, "\n"); + yield* fs.remove(icon); + yield* fs.symlink(external, icon); + const report = yield* checkRepositoryConformance({ + repositoryRoot: fixture.root, + packageRoot: fixture.packageRoot, + }); + assert.isTrue(failedCheck(report, "repository.source.no_symlinks")); + }), + ), + ); + + it.effect("requires the exact lean OpenAI archive payload", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stage = yield* fs.makeTempDirectoryScoped({ prefix: "openai-payload-test-" }); + yield* makeLeanOpenAiPayload(stage); + yield* verifyOpenAiArchivePayload(stage); + + const foreign = path.join(stage, "targets", "claude", ".mcp.json"); + yield* fs.makeDirectory(path.dirname(foreign), { recursive: true }); + yield* fs.writeFileString(foreign, "{}\n"); + const error = yield* verifyOpenAiArchivePayload(stage).pipe(Effect.flip); + assert.instanceOf(error, ArtifactVerificationError); + assert.include(error.message, "OpenAI archive root"); + }), + ), + ); }); }); diff --git a/tools/check-target-conformance.ts b/tools/check-target-conformance.ts index 5aaa71c..0f16e41 100755 --- a/tools/check-target-conformance.ts +++ b/tools/check-target-conformance.ts @@ -41,7 +41,7 @@ type ConformanceEnvironment = FileSystem.FileSystem | Path.Path; export interface TargetConformanceCheck { readonly id: string; - readonly target: TargetName; + readonly target: TargetName | "repository"; readonly title: string; readonly passed: boolean; readonly detail?: string; @@ -53,8 +53,14 @@ export interface TargetSummary { readonly checks: readonly TargetConformanceCheck[]; } +export interface RepositorySummary { + readonly target: "repository"; + readonly passed: boolean; + readonly checks: readonly TargetConformanceCheck[]; +} export interface TargetConformanceReport { readonly targets: Partial>; + readonly repository?: RepositorySummary; readonly totalChecks: number; readonly totalPassed: number; readonly totalFailed: number; @@ -63,6 +69,8 @@ export interface TargetConformanceReport { export interface TargetConformanceOptions { readonly packageRoot?: string; + readonly repositoryRoot?: string; + readonly checkRepository?: boolean; } export class TargetConformanceError extends Data.TaggedError("TargetConformanceError")<{ @@ -129,6 +137,39 @@ const hasExactKeys = (value: unknown, expected: readonly string[]): boolean => { return sameSortedStrings(actual, wanted); }; +const isWithin = (paths: Path.Path, parent: string, child: string): boolean => { + const relative = paths.relative(parent, child); + return !relative.startsWith(`..${paths.sep}`) && relative !== ".." && !paths.isAbsolute(relative); +}; + +const findSymbolicLinks = ( + fs: FileSystem.FileSystem, + paths: Path.Path, + directory: string, +): Effect.Effect => + Effect.gen(function* () { + const exists = yield* fs.exists(directory); + if (!exists) return []; + const entries = yield* fs.readDirectory(directory); + const nested = yield* Effect.forEach( + entries, + (entry) => + Effect.gen(function* () { + const entryPath = paths.join(directory, entry); + const isLink = yield* fs + .readLink(entryPath) + .pipe(Effect.match({ onFailure: () => false, onSuccess: () => true })); + if (isLink) return [entryPath]; + const info = yield* fs.stat(entryPath); + if (info.type === "Directory" && entry !== "node_modules" && entry !== ".git") { + return yield* findSymbolicLinks(fs, paths, entryPath); + } + return []; + }), + { concurrency: "unbounded" }, + ); + return nested.flat(); + }); const advertisedToolIdentifiers = (markdown: string): readonly string[] => Array.from( markdown.matchAll(/`((?:gina|spot|perps|predictions)\.[A-Za-z][A-Za-z0-9]*)`/g), @@ -320,7 +361,7 @@ const validateMcp = (target: TargetName, manifest: unknown): boolean => { const defaultPackageRootFor = (paths: Path.Path): string => paths.resolve(here, "..", "plugins", "ask-gina"); - +const defaultRepositoryRootFor = (paths: Path.Path): string => paths.resolve(here, ".."); type GeneratedTargetConformanceEffect = Effect.Effect< TargetSummary, TargetConformanceError, @@ -615,6 +656,338 @@ export const checkTargetConformance: { ), ); +export const checkRepositoryConformance = ( + options: TargetConformanceOptions = {}, +): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const paths = yield* Path.Path; + const packageRoot = paths.resolve(options.packageRoot ?? defaultPackageRootFor(paths)); + const repositoryRoot = paths.resolve( + options.repositoryRoot ?? + (options.packageRoot === undefined + ? defaultRepositoryRootFor(paths) + : paths.join(packageRoot, "..", "..")), + ); + const canonicalRepositoryRoot = yield* withFileSystemError( + repositoryRoot, + "cannot be resolved", + fs.realPath(repositoryRoot), + ); + const checks: TargetConformanceCheck[] = []; + const addCheck = (id: string, title: string, passed: boolean, detail?: string): void => { + checks.push({ id, target: "repository", title, passed, detail }); + }; + + const marketplacePath = paths.join(repositoryRoot, ".agents", "plugins", "marketplace.json"); + const marketplaceExists = yield* withFileSystemError( + marketplacePath, + "cannot be inspected", + fs.exists(marketplacePath), + ); + addCheck("marketplace.exists", ".agents/plugins/marketplace.json exists", marketplaceExists); + if (marketplaceExists) { + const marketplace = yield* readJson(fs, marketplacePath); + const plugins = nested(marketplace, "plugins"); + const plugin = Array.isArray(plugins) && plugins.length === 1 ? plugins[0] : undefined; + const source = nested(plugin, "source"); + const policy = nested(plugin, "policy"); + const marketplaceContract = + hasExactKeys(marketplace, ["name", "interface", "plugins"]) && + nested(marketplace, "name") === "ask-gina-plugins" && + hasExactKeys(nested(marketplace, "interface"), ["displayName"]) && + nested(marketplace, "interface", "displayName") === "Ask Gina Plugins" && + Array.isArray(plugins) && + plugins.length === 1 && + hasExactKeys(plugin, ["name", "source", "policy", "category"]) && + nested(plugin, "name") === "ask-gina" && + hasExactKeys(source, ["source", "path"]) && + nested(source, "source") === "local" && + nested(source, "path") === "./plugins/ask-gina" && + hasExactKeys(policy, ["installation", "authentication"]) && + nested(policy, "installation") === "AVAILABLE" && + nested(policy, "authentication") === "ON_INSTALL" && + nested(plugin, "category") === "Finance"; + addCheck( + "marketplace.schema", + "Repository marketplace has exact schema and values", + marketplaceContract, + marketplaceContract + ? undefined + : "marketplace name, interface, plugin, policy, category, or source is malformed", + ); + const descriptorVersioned = + nested(marketplace, "version") !== undefined || nested(plugin, "version") !== undefined; + addCheck( + "marketplace.no_version", + "Repository marketplace descriptor is unversioned", + !descriptorVersioned, + descriptorVersioned ? "marketplace descriptor must not declare version" : undefined, + ); + + const sourceValue = nested(source, "path"); + let sourceContained = false; + let sourceDetail: string | undefined; + if (typeof sourceValue !== "string") { + sourceDetail = "plugin source path must be a string"; + } else { + const segments = sourceValue.split(/[\\/]/u); + const normalized = paths.normalize(sourceValue); + const resolvedSource = paths.resolve(repositoryRoot, sourceValue); + const normalizedExpected = paths.normalize("plugins/ask-gina"); + if ( + paths.isAbsolute(sourceValue) || + segments.some((segment) => segment === "..") || + normalized !== normalizedExpected || + !isWithin(paths, repositoryRoot, resolvedSource) + ) { + sourceDetail = `plugin source path escapes or is not canonical: ${sourceValue}`; + } else { + const exists = yield* withFileSystemError( + resolvedSource, + "cannot be inspected", + fs.exists(resolvedSource), + ); + if (!exists) { + sourceDetail = `declared plugin source is missing: ${sourceValue}`; + } else { + const sourceIsLink = yield* fs + .readLink(resolvedSource) + .pipe(Effect.match({ onFailure: () => false, onSuccess: () => true })); + if (sourceIsLink) { + sourceDetail = `declared plugin source is a symbolic link: ${sourceValue}`; + } else { + const realSource = yield* withFileSystemError( + resolvedSource, + "cannot be resolved", + fs.realPath(resolvedSource), + ); + sourceContained = isWithin(paths, canonicalRepositoryRoot, realSource); + if (!sourceContained) sourceDetail = `declared plugin source escapes: ${sourceValue}`; + } + } + } + } + addCheck( + "marketplace.source_containment", + "Repository marketplace source exists inside the repository without symlink indirection", + sourceContained, + sourceDetail, + ); + } + + const openAiManifestPath = paths.join(packageRoot, ".codex-plugin", "plugin.json"); + const openAiMcpPath = paths.join(packageRoot, ".mcp.json"); + const openAiIconPath = paths.join(packageRoot, "assets", "icon.svg"); + const canonicalSkillsRoot = paths.join(packageRoot, "skills"); + const requiredOpenAiPaths = [ + [ + openAiManifestPath, + "repository.root_openai.manifest_exists", + "Root OpenAI manifest exists as a regular file", + "File", + ], + [ + openAiMcpPath, + "repository.root_openai.mcp_exists", + "Root OpenAI MCP configuration exists as a regular file", + "File", + ], + [ + openAiIconPath, + "repository.root_openai.icon_exists", + "Root OpenAI icon exists as a regular file", + "File", + ], + [ + canonicalSkillsRoot, + "repository.root_openai.skills_exists", + "Root canonical skills directory exists", + "Directory", + ], + ] as const; + for (const [candidate, id, title, expectedType] of requiredOpenAiPaths) { + const exists = yield* withFileSystemError( + candidate, + "cannot be inspected", + fs.exists(candidate), + ); + const actualType = exists + ? (yield* withFileSystemError(candidate, "cannot be inspected", fs.stat(candidate))).type + : undefined; + addCheck( + id, + title, + exists && actualType === expectedType, + exists && actualType !== expectedType + ? `expected ${expectedType}, found ${actualType ?? "unknown"}` + : undefined, + ); + } + + if ( + yield* withFileSystemError( + openAiManifestPath, + "cannot be inspected", + fs.exists(openAiManifestPath), + ) + ) { + const manifest = yield* readJson(fs, openAiManifestPath); + addCheck( + "repository.root_openai.manifest_contract", + "Root OpenAI manifest has exact contract and release version", + validateManifest("openai", manifest), + ); + } + if ( + yield* withFileSystemError(openAiMcpPath, "cannot be inspected", fs.exists(openAiMcpPath)) + ) { + const mcp = yield* readJson(fs, openAiMcpPath); + addCheck( + "repository.root_openai.mcp_contract", + "Root OpenAI MCP configuration binds the production endpoint", + validateMcp("openai", mcp), + ); + } + + const expectedSkillNames = ASK_GINA_SKILL_DEFINITIONS.map((skill) => skill.name).sort(); + const skillsExist = yield* withFileSystemError( + canonicalSkillsRoot, + "cannot be inspected", + fs.exists(canonicalSkillsRoot), + ); + let actualSkillNames: readonly string[] = []; + let skillFilesComplete = false; + if (skillsExist) { + actualSkillNames = (yield* withFileSystemError( + canonicalSkillsRoot, + "cannot be read", + fs.readDirectory(canonicalSkillsRoot), + )).sort(); + const declaredFiles = yield* Effect.forEach( + expectedSkillNames, + (skill) => + Effect.all( + [ + withFileSystemError( + paths.join(canonicalSkillsRoot, skill, "SKILL.md"), + "cannot be inspected", + fs.exists(paths.join(canonicalSkillsRoot, skill, "SKILL.md")), + ), + withFileSystemError( + paths.join(canonicalSkillsRoot, skill, "agents", "openai.yaml"), + "cannot be inspected", + fs.exists(paths.join(canonicalSkillsRoot, skill, "agents", "openai.yaml")), + ), + ], + { concurrency: "unbounded" }, + ), + { concurrency: "unbounded" }, + ); + skillFilesComplete = declaredFiles.every(([skill, metadata]) => skill && metadata); + } + addCheck( + "repository.root_openai.skills_contract", + "Root OpenAI source contains exact canonical skills and declared files", + sameSortedStrings(actualSkillNames, expectedSkillNames) && skillFilesComplete, + ); + + const legacyOpenAiOverlay = paths.join(packageRoot, "targets", "openai"); + const legacyOpenAiExists = yield* withFileSystemError( + legacyOpenAiOverlay, + "cannot be inspected", + fs.exists(legacyOpenAiOverlay), + ); + addCheck( + "repository.legacy_openai.absent", + "Legacy targets/openai overlay is absent", + !legacyOpenAiExists, + legacyOpenAiExists ? "legacy OpenAI overlay must be removed" : undefined, + ); + + const symlinks = yield* findSymbolicLinks(fs, paths, packageRoot).pipe( + Effect.mapError((cause) => + targetConformanceError(packageRoot, "cannot inspect plugin source links", cause), + ), + ); + addCheck( + "repository.source.no_symlinks", + "Plugin source contains no symbolic links", + symlinks.length === 0, + symlinks.length === 0 ? undefined : `symbolic links: ${symlinks.join(", ")}`, + ); + + const claudeMarketplacePath = paths.join(repositoryRoot, ".claude-plugin", "marketplace.json"); + const claudeMarketplaceExists = yield* withFileSystemError( + claudeMarketplacePath, + "cannot be inspected", + fs.exists(claudeMarketplacePath), + ); + addCheck( + "repository.claude.marketplace_exists", + "Independent Claude marketplace descriptor exists", + claudeMarketplaceExists, + ); + if (claudeMarketplaceExists) { + const claudeMarketplace = yield* readJson(fs, claudeMarketplacePath); + const claudePlugins = nested(claudeMarketplace, "plugins"); + const claudePlugin = + Array.isArray(claudePlugins) && claudePlugins.length === 1 ? claudePlugins[0] : undefined; + const claudeContract = + nested(claudeMarketplace, "name") === "ask-gina-plugins" && + nested(claudePlugin, "name") === "ask-gina" && + nested(claudePlugin, "source") === "./plugins/ask-gina" && + nested(claudePlugin, "skills") === "./skills" && + nested(claudePlugin, "mcpServers") === "./targets/claude/.mcp.json"; + addCheck( + "repository.claude.marketplace_contract", + "Claude marketplace independently references the Claude MCP overlay", + claudeContract, + ); + } + + const claudeManifestPath = paths.join( + packageRoot, + "targets", + "claude", + ".claude-plugin", + "plugin.json", + ); + const claudeMcpPath = paths.join(packageRoot, "targets", "claude", ".mcp.json"); + const [claudeManifestExists, claudeMcpExists] = yield* Effect.all( + [ + withFileSystemError( + claudeManifestPath, + "cannot be inspected", + fs.exists(claudeManifestPath), + ), + withFileSystemError(claudeMcpPath, "cannot be inspected", fs.exists(claudeMcpPath)), + ], + { concurrency: "unbounded" }, + ); + addCheck( + "repository.claude.overlay_exists", + "Claude overlay manifest and MCP configuration exist", + claudeManifestExists && claudeMcpExists, + ); + if (claudeManifestExists && claudeMcpExists) { + const [claudeManifest, claudeMcp] = yield* Effect.all( + [readJson(fs, claudeManifestPath), readJson(fs, claudeMcpPath)], + { concurrency: "unbounded" }, + ); + addCheck( + "repository.claude.overlay_contract", + "Claude overlay has the release version and production MCP contract", + validateManifest("claude", claudeManifest) && validateMcp("claude", claudeMcp), + ); + } + + return { + target: "repository", + passed: checks.every((check) => check.passed), + checks, + }; + }); export const runTargetConformanceChecks = ( options: TargetConformanceOptions & { readonly target?: TargetName } = {}, ): Effect.Effect< @@ -631,11 +1004,17 @@ export const runTargetConformanceChecks = ( ); const targets: Partial> = {}; for (const summary of summaries) targets[summary.target] = summary; - const checks = summaries.flatMap((summary) => summary.checks); + const shouldCheckRepository = options.checkRepository ?? options.target === undefined; + const repository = shouldCheckRepository + ? yield* checkRepositoryConformance(options) + : undefined; + const allSummaries = repository === undefined ? summaries : [...summaries, repository]; + const checks = allSummaries.flatMap((summary) => summary.checks); const totalPassed = checks.filter((check) => check.passed).length; const totalFailed = checks.length - totalPassed; return { targets, + ...(repository === undefined ? {} : { repository }), totalChecks: checks.length, totalPassed, totalFailed, @@ -683,6 +1062,14 @@ const printHumanReport = (report: TargetConformanceReport): void => { process.stdout.write(` ${check.passed ? "PASS" : "FAIL"} ${check.title}\n`); } } + if (report.repository !== undefined) { + process.stdout.write(`\nREPOSITORY: ${report.repository.passed ? "PASS" : "FAIL"}\n`); + for (const check of report.repository.checks) { + process.stdout.write( + ` ${check.passed ? "PASS" : "FAIL"} ${check.title}${check.detail === undefined ? "" : ` (${check.detail})`}\n`, + ); + } + } process.stdout.write( `\n${report.totalPassed}/${report.totalChecks} checks passed; ${report.allPassed ? "all targets conform" : "conformance failed"}.\n`, ); diff --git a/tools/pack-artifacts.ts b/tools/pack-artifacts.ts index 6cd7de8..5a97b7c 100755 --- a/tools/pack-artifacts.ts +++ b/tools/pack-artifacts.ts @@ -9,7 +9,8 @@ import { runHermeticEvalReplay, sanitizeEvalReplay } from "../packages/evals/src import { copyCheckedRegularFile } from "./archive-security"; import { checkGeneratedTargetConformance } from "./check-target-conformance"; -const HOSTS = ["openai", "cursor", "claude", "copilot", "gemini"] as const; +export const HOSTS = ["openai", "cursor", "claude", "copilot", "gemini"] as const; +export type Host = (typeof HOSTS)[number]; const SKILLS = [ "research-hyperliquid", "research-prediction-markets", @@ -69,12 +70,17 @@ const PACKAGES = [ directory: "packages/evals", packageFiles: ["dist", "LICENSE", "README.md"], compiledFiles: [ + /^bin\/check-codex-marketplace\.d\.ts$/u, + /^bin\/check-codex-marketplace\.js$/u, + /^bin\/check-codex-marketplace\.js\.map$/u, /^bin\/live\.d\.ts$/u, /^bin\/live\.js$/u, /^bin\/live\.js\.map$/u, /^bin\/replay\.d\.ts$/u, /^bin\/replay\.js$/u, /^bin\/replay\.js\.map$/u, + /^codex-cli-[A-Za-z0-9_-]+\.js$/u, + /^codex-cli-[A-Za-z0-9_-]+\.js\.map$/u, /^index\.d\.ts$/u, /^index\.js$/u, /^report-[A-Za-z0-9_-]+\.js$/u, @@ -86,7 +92,7 @@ const PACKAGES = [ ], }, ]; -const TARGET_MANIFESTS: Readonly> = { +const TARGET_MANIFESTS: Readonly> = { openai: ".codex-plugin/plugin.json", cursor: ".cursor-plugin/plugin.json", claude: ".claude-plugin/plugin.json", @@ -764,7 +770,21 @@ const assertLiveSourceBoundary = (root: string, snapshot: string) => ), ); }); +type PluginTargetEffect = Effect.Effect; +export const validateTargetVersion: { + (host: Host, version: string): (pluginRoot: string) => PluginTargetEffect; + (pluginRoot: string, host: Host, version: string): PluginTargetEffect; +} = Function.dual(3, (pluginRoot: string, host: Host, version: string): PluginTargetEffect => + Effect.gen(function* () { + const path = yield* Path.Path; + const manifestRoot = host === "openai" ? pluginRoot : path.join(pluginRoot, "targets", host); + const value = yield* readJson(path.join(manifestRoot, TARGET_MANIFESTS[host])); + if (!isObject(value) || value.version !== version) { + return yield* fail(`${host} target version is inconsistent`); + } + }), +); const validateVersions = (root: string) => Effect.gen(function* () { const path = yield* Path.Path; @@ -787,16 +807,8 @@ const validateVersions = (root: string) => if (pluginManifest.match(/^version:\s*([^\s]+)$/mu)?.[1] !== version) { return yield* fail("plugin.yaml version is inconsistent"); } - yield* Effect.forEach(HOSTS, (host) => - Effect.gen(function* () { - const manifest = TARGET_MANIFESTS[host]; - if (manifest === undefined) return yield* fail(`unknown target ${host}`); - const value = yield* readJson(path.join(root, "plugins/ask-gina/targets", host, manifest)); - if (!isObject(value) || value.version !== version) { - return yield* fail(`${host} target version is inconsistent`); - } - }), - ); + const pluginRoot = path.join(root, "plugins/ask-gina"); + yield* Effect.forEach(HOSTS, (host) => validateTargetVersion(pluginRoot, host, version)); return version; }); @@ -819,6 +831,57 @@ const buildEvalReceipt = (root: string, version: string, sourceCommit: string) = return { releaseVersion: version, sourceCommit, aggregate }; }); +export const stagePluginTarget: { + (plugin: string, stage: string): (host: Host) => PluginTargetEffect; + (host: Host, plugin: string, stage: string): PluginTargetEffect; +} = Function.dual(3, (host: Host, plugin: string, stage: string): PluginTargetEffect => + Effect.gen(function* () { + const path = yield* Path.Path; + const fs = yield* FileSystem.FileSystem; + + if (host === "openai") { + yield* fs + .makeDirectory(stage, { recursive: true }) + .pipe(Effect.mapError((cause) => fail("cannot create openai target stage", cause))); + yield* Effect.forEach([".codex-plugin", "assets"] as const, (entry) => + Effect.gen(function* () { + const source = path.join(plugin, entry); + yield* filesBelow(source); + yield* fs + .copy(source, path.join(stage, entry), { overwrite: true }) + .pipe(Effect.mapError((cause) => fail(`cannot stage openai ${entry}`, cause))); + }), + ); + yield* copyCheckedRegularFile( + path.join(plugin, ".mcp.json"), + path.join(stage, ".mcp.json"), + ).pipe(Effect.mapError((cause) => fail("cannot stage openai .mcp.json", cause))); + } else { + const sourceOverlay = path.join(plugin, "targets", host); + yield* filesBelow(sourceOverlay); + yield* fs + .copy(sourceOverlay, stage, { overwrite: true }) + .pipe(Effect.mapError((cause) => fail(`cannot stage ${host} target`, cause))); + } + + yield* fs + .makeDirectory(path.join(stage, "skills"), { recursive: true }) + .pipe(Effect.mapError((cause) => fail(`cannot create ${host} skills`, cause))); + yield* Effect.forEach(SKILLS, (skill) => + Effect.gen(function* () { + const destination = path.join(stage, "skills", skill); + yield* fs + .copy(path.join(plugin, "skills", skill), destination, { overwrite: true }) + .pipe(Effect.mapError((cause) => fail(`cannot stage ${host}:${skill}`, cause))); + if (host !== "openai") { + yield* fs + .remove(path.join(destination, "agents"), { recursive: true, force: true }) + .pipe(Effect.mapError((cause) => fail(`cannot remove ${host} overlay`, cause))); + } + }), + ); + }), +); export interface BuildArtifactsOptions { readonly root: string; readonly dist: string; @@ -899,26 +962,7 @@ export const buildArtifacts = ({ root, dist }: BuildArtifactsOptions) => const targetReceipts = yield* Effect.forEach(HOSTS, (host) => Effect.gen(function* () { const stage = path.join(temporary, `target-${host}`); - yield* filesBelow(path.join(plugin, "targets", host)); - yield* fs - .copy(path.join(plugin, "targets", host), stage, { overwrite: true }) - .pipe(Effect.mapError((cause) => fail(`cannot stage ${host} target`, cause))); - yield* fs - .makeDirectory(path.join(stage, "skills"), { recursive: true }) - .pipe(Effect.mapError((cause) => fail(`cannot create ${host} skills`, cause))); - yield* Effect.forEach(SKILLS, (skill) => - Effect.gen(function* () { - const destination = path.join(stage, "skills", skill); - yield* fs - .copy(path.join(plugin, "skills", skill), destination, { overwrite: true }) - .pipe(Effect.mapError((cause) => fail(`cannot stage ${host}:${skill}`, cause))); - if (host !== "openai") { - yield* fs - .remove(path.join(destination, "agents"), { recursive: true, force: true }) - .pipe(Effect.mapError((cause) => fail(`cannot remove ${host} overlay`, cause))); - } - }), - ); + yield* stagePluginTarget(host, plugin, stage); const skills = (yield* fs .readDirectory(path.join(stage, "skills")) .pipe(Effect.mapError((cause) => fail(`cannot list ${host} skills`, cause)))).sort(); diff --git a/tools/sync-plugin-skills.ts b/tools/sync-plugin-skills.ts index 32f7c3e..65324bb 100755 --- a/tools/sync-plugin-skills.ts +++ b/tools/sync-plugin-skills.ts @@ -10,6 +10,7 @@ export type TargetName = (typeof TARGET_NAMES)[number]; export const SKILL_NAMES = ASK_GINA_SKILL_DEFINITIONS.map((skill) => skill.name); const OPENAI_METADATA_PATH = ["agents", "openai.yaml"] as const; +const OPENAI_SOURCE_ENTRIES = [".codex-plugin", ".mcp.json", "assets"] as const; const here = fileURLToPath(new URL(".", import.meta.url)); type SyncEnvironment = FileSystem.FileSystem | Path.Path; @@ -151,6 +152,80 @@ const assertSourceIsPortable = ( yield* assertDirectoryNames(canonicalSkills, SKILL_NAMES); yield* assertNoSymbolicLinks(canonicalSkills); + const legacyOpenAiOverlay = paths.join(packageRoot, "targets", "openai"); + const legacyOpenAiOverlayExists = yield* withFileSystemError( + legacyOpenAiOverlay, + "cannot be inspected", + fs.exists(legacyOpenAiOverlay), + ); + if (legacyOpenAiOverlayExists) { + return yield* pluginSkillSyncError( + legacyOpenAiOverlay, + "legacy OpenAI target overlay must not exist", + ); + } + + const openAiFiles = [ + paths.join(packageRoot, ".codex-plugin", "plugin.json"), + paths.join(packageRoot, ".mcp.json"), + paths.join(packageRoot, "assets", "icon.svg"), + ] as const; + yield* Effect.forEach(openAiFiles, (candidate) => + Effect.gen(function* () { + const symbolicLink = yield* fs + .readLink(candidate) + .pipe(Effect.match({ onFailure: () => false, onSuccess: () => true })); + if (symbolicLink) { + return yield* pluginSkillSyncError( + candidate, + "plugin source must not contain symbolic links", + ); + } + const info = yield* withFileSystemError( + candidate, + "cannot be inspected", + fs.stat(candidate), + ); + if (info.type !== "File") { + return yield* pluginSkillSyncError( + candidate, + "required OpenAI source file is not a file", + ); + } + if (candidate.endsWith(".json")) { + const source = yield* withFileSystemError( + candidate, + "cannot be read", + fs.readFileString(candidate), + ); + yield* parseJson(candidate, source); + } + }), + ); + yield* assertNoSymbolicLinks(paths.join(packageRoot, ".codex-plugin")); + yield* assertNoSymbolicLinks(paths.join(packageRoot, "assets")); + yield* Effect.forEach( + TARGET_NAMES.filter((target) => target !== "openai"), + (target) => + Effect.gen(function* () { + const overlay = paths.join(packageRoot, "targets", target); + const exists = yield* withFileSystemError( + overlay, + "cannot be inspected", + fs.exists(overlay), + ); + if (!exists) { + return yield* pluginSkillSyncError(overlay, "missing target overlay"); + } + const info = yield* withFileSystemError(overlay, "cannot be inspected", fs.stat(overlay)); + if (info.type !== "Directory") { + return yield* pluginSkillSyncError(overlay, "target overlay is not a directory"); + } + yield* assertNoSymbolicLinks(overlay); + }), + { concurrency: "unbounded" }, + ); + yield* Effect.forEach( TARGET_NAMES, (target) => { @@ -227,6 +302,23 @@ const copyTargetOverlay = ( ); }); +const copyOpenAiSourceSurfaces = ( + packageRoot: string, + destination: string, +): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const paths = yield* Path.Path; + yield* Effect.forEach(OPENAI_SOURCE_ENTRIES, (entry) => { + const source = paths.join(packageRoot, entry); + return withFileSystemError( + source, + "cannot be copied", + fs.copy(source, paths.join(destination, entry), { overwrite: false }), + ); + }); + }); + const copyCanonicalSkills = ( packageRoot: string, target: TargetName, @@ -281,22 +373,17 @@ const materializeTarget = ( const packageRoot = packageRootFor(paths, options.packageRoot); yield* assertSourceIsPortable(packageRoot); - const sourceOverlay = paths.join(packageRoot, "targets", target); - const overlayExists = yield* withFileSystemError( - sourceOverlay, - "cannot be inspected", - fs.exists(sourceOverlay), - ); - if (!overlayExists) { - return yield* pluginSkillSyncError(sourceOverlay, "missing target overlay"); - } - yield* withFileSystemError( destination, "cannot be created", fs.makeDirectory(destination, { recursive: true }), ); - yield* copyTargetOverlay(sourceOverlay, destination); + + if (target === "openai") { + yield* copyOpenAiSourceSurfaces(packageRoot, destination); + } else { + yield* copyTargetOverlay(paths.join(packageRoot, "targets", target), destination); + } yield* copyCanonicalSkills(packageRoot, target, destination); }); diff --git a/tools/verify-artifacts.ts b/tools/verify-artifacts.ts index 32a39db..ea5d648 100755 --- a/tools/verify-artifacts.ts +++ b/tools/verify-artifacts.ts @@ -457,6 +457,52 @@ const exactDirectory = (directory: string, expected: readonly string[], label: s } }); +export const verifyOpenAiArchivePayload = (directory: string) => + Effect.gen(function* () { + const path = yield* Path.Path; + yield* exactDirectory( + directory, + [".codex-plugin", ".mcp.json", "assets", "skills"], + "OpenAI archive root", + ); + yield* exactDirectory( + path.join(directory, ".codex-plugin"), + ["plugin.json"], + "OpenAI manifest directory", + ); + yield* exactDirectory(path.join(directory, "assets"), ["icon.svg"], "OpenAI assets directory"); + yield* exactDirectory(path.join(directory, "skills"), SKILLS, "OpenAI skills directory"); + yield* Effect.forEach(SKILLS, (skill) => + Effect.all( + [ + exactDirectory( + path.join(directory, "skills", skill), + ["SKILL.md", "agents"], + `OpenAI ${skill} skill directory`, + ), + exactDirectory( + path.join(directory, "skills", skill, "agents"), + ["openai.yaml"], + `OpenAI ${skill} agents directory`, + ), + ], + { concurrency: "unbounded" }, + ), + ); + const expected = [ + ".codex-plugin/plugin.json", + ".mcp.json", + "assets/icon.svg", + ...SKILLS.flatMap((skill) => [ + `skills/${skill}/SKILL.md`, + `skills/${skill}/agents/openai.yaml`, + ]), + ].sort(); + const actual = [...(yield* filesBelow(directory))].sort(); + if (stableJson(actual) !== stableJson(expected)) { + return yield* fail("OpenAI archive payload is not the exact lean host payload"); + } + }); const compareBytes = (actual: string, expected: string, label: string) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -820,6 +866,9 @@ const verifyPackages = ( ); const entries = yield* requiredArray(receipt.packages, "packages receipt packages"); yield* verifySourceDirty(receipt.sourceDirty, sourceDirty, "packages receipt sourceDirty"); + if (receipt.schemaVersion !== "v1" || receipt.releaseVersion !== version) { + return yield* fail("packages receipt release version is stale"); + } if (entries.length !== PACKAGES.length) return yield* fail("packages receipt has the wrong package count"); yield* Effect.forEach(PACKAGES, (definition) => @@ -944,6 +993,9 @@ const verifyTargets = ( ); const targets = yield* requiredArray(receipt.targets, "targets receipt targets"); yield* verifySourceDirty(receipt.sourceDirty, sourceDirty, "targets receipt sourceDirty"); + if (receipt.schemaVersion !== "v1" || receipt.releaseVersion !== version) { + return yield* fail("targets receipt release version is stale"); + } if (targets.length !== HOSTS.length) return yield* fail("targets receipt has the wrong host count"); yield* Effect.forEach(HOSTS, (host) => @@ -978,6 +1030,7 @@ const verifyTargets = ( } const stage = path.join(temporary, `target-${host}`); yield* extract(archive, stage); + if (host === "openai") yield* verifyOpenAiArchivePayload(stage); yield* verifyProofs( yield* parseProofs(entry.files, `${host} receipt files`), yield* fileProofs(stage), diff --git a/vite.config.ts b/vite.config.ts index fdfdead..11060a8 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -50,6 +50,7 @@ export default defineConfig({ ...packDefaults, cwd: fileURLToPath(new URL("packages/evals/", import.meta.url)), entry: { + "bin/check-codex-marketplace": "src/bin/check-codex-marketplace.ts", "bin/live": "src/bin/live.ts", "bin/replay": "src/bin/replay.ts", index: "src/index.ts", From e2d468c17fdf2f5f9ff01a57ffd933fba24cd99c Mon Sep 17 00:00:00 2001 From: Eric Juta Date: Wed, 26 Aug 2026 21:00:08 +0000 Subject: [PATCH 2/2] fix: authenticate private marketplace smoke --- .github/workflows/ci.yml | 23 +--- docs/architecture.md | 12 +- .../__tests__/check-codex-marketplace.test.ts | 21 ++++ .../evals/src/bin/check-codex-marketplace.ts | 108 ++++++++++++++++-- specs/codex-repository-marketplace-plan.md | 12 +- 5 files changed, 142 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b51abad..335d0d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,10 +41,7 @@ jobs: - run: bun run smoke:install - run: bun run check:public-boundary - name: Install pinned Codex marketplace CLI - if: >- - github.event_name == 'push' || - (github.event.pull_request.head.repo.full_name != '' && - github.event.pull_request.head.sha != '') + if: github.event_name == 'push' env: BUN_INSTALL_CACHE_DIR: ${{ runner.temp }}/codex-cli-cache CODEX_CLI_ROOT: ${{ runner.temp }}/codex-cli @@ -53,27 +50,19 @@ jobs: printf '{"private":true}\n' > "$CODEX_CLI_ROOT/package.json" bun add --cwd "$CODEX_CLI_ROOT" --exact @openai/codex@0.149.0-alpha.4.3 - name: Smoke Codex marketplace at published revision - if: >- - github.event_name == 'push' || - (github.event.pull_request.head.repo.full_name != '' && - github.event.pull_request.head.sha != '') + if: github.event_name == 'push' env: CODEX_MARKETPLACE_EXECUTABLE: ${{ runner.temp }}/codex-cli/node_modules/.bin/codex - CODEX_MARKETPLACE_REF: >- - ${{ github.event.pull_request.head.sha || github.sha }} - CODEX_MARKETPLACE_REPOSITORY: >- - ${{ github.event.pull_request.head.repo.full_name || github.repository }} + CODEX_MARKETPLACE_REPOSITORY_TOKEN: ${{ github.token }} + CODEX_MARKETPLACE_REF: ${{ github.sha }} + CODEX_MARKETPLACE_REPOSITORY: ${{ github.repository }} run: >- bun run check:marketplace:codex -- --executable "$CODEX_MARKETPLACE_EXECUTABLE" --repository "$CODEX_MARKETPLACE_REPOSITORY" --ref "$CODEX_MARKETPLACE_REF" - name: Remove temporary Codex CLI state - if: >- - always() && - (github.event_name == 'push' || - (github.event.pull_request.head.repo.full_name != '' && - github.event.pull_request.head.sha != '')) + if: always() && github.event_name == 'push' run: rm -rf "$RUNNER_TEMP/codex-cli" "$RUNNER_TEMP/codex-cli-cache" - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: diff --git a/docs/architecture.md b/docs/architecture.md index 9503af2..8ad79c0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -21,6 +21,12 @@ CommonJS, browser, edge, or subpath entrypoints. They support Node >=24 and Bun points directly at `plugins/ask-gina/`, so a clean checkout exposes the OpenAI source without a generation step. The directly loadable OpenAI files live at the plugin root: `.codex-plugin/plugin.json`, `.mcp.json`, `assets/icon.svg`, and `skills/`. +The repository is private. Automated remote marketplace proof runs only on a trusted +same-repository push and uses an ephemeral, read-only GitHub token only while Codex +clones the marketplace. Pull-request jobs never receive this token. The smoke harness +deletes its isolated token and token-free askpass helper before plugin installation, +and neither plugin installation nor runtime receives the repository token. Marketplace consumers need +GitHub read access independently of Ask Gina MCP authentication. `plugins/ask-gina/skills/` is the only authored skill tree. OpenAI-specific skill metadata remains beside each canonical `SKILL.md` under `agents/openai.yaml`. @@ -67,7 +73,9 @@ authoring source. Fork-safe GitHub Actions run formatting, lint, compiler, test, audit, target conformance, artifact clean-install/runtime, and public-boundary gates on pull requests. These workflows use `pull_request` with `contents: read`, but `main` -has no protected required checks, so they do not gate merge. A separate manual -Responses live-smoke workflow may read two protected secrets and uploads only the +has no protected required checks, so they do not gate merge. The authenticated Codex +remote smoke runs separately on same-repository pushes at the immutable pushed SHA; +it never runs in a pull-request job. A separate manual Responses live-smoke workflow +may read two protected secrets and uploads only the sanitized aggregate. There is no release job, OIDC, package writer, remote cache, or publication authority. diff --git a/packages/evals/__tests__/check-codex-marketplace.test.ts b/packages/evals/__tests__/check-codex-marketplace.test.ts index 2f0e6bf..151b85a 100644 --- a/packages/evals/__tests__/check-codex-marketplace.test.ts +++ b/packages/evals/__tests__/check-codex-marketplace.test.ts @@ -112,6 +112,20 @@ const makeFakeRunner = ( detail: "fake runner did not receive CODEX_HOME", }); } + const isolatedHome = input.environment.HOME; + const repositoryTokenExists = + isolatedHome !== undefined && + (yield* fs.exists(path.join(isolatedHome, ".repository-token")).pipe(Effect.orDie)); + const gitAskpassExists = + isolatedHome !== undefined && + (yield* fs.exists(path.join(isolatedHome, "git-askpass")).pipe(Effect.orDie)); + if (input.stage !== "marketplace-add" && (repositoryTokenExists || gitAskpassExists)) { + return yield* new CodexMarketplaceSmokeError({ + stage: input.stage, + reason: "command-failed", + detail: "repository credential remained after marketplace clone", + }); + } if (input.stage === failStage) { return yield* new CodexMarketplaceSmokeError({ stage: input.stage, @@ -265,6 +279,7 @@ describe("Codex marketplace environment isolation", () => { OPENAI_API_KEY: "provider-secret", ASK_GINA_ACCESS_TOKEN: "gina-secret", GITHUB_TOKEN: "github-secret", + CODEX_MARKETPLACE_REPOSITORY_TOKEN: "repository-secret", SSH_AUTH_SOCK: "/tmp/agent.sock", HTTPS_PROXY: "http://proxy-secret", }); @@ -282,6 +297,7 @@ describe("Codex marketplace environment isolation", () => { assert.notProperty(environment, "OPENAI_API_KEY"); assert.notProperty(environment, "ASK_GINA_ACCESS_TOKEN"); assert.notProperty(environment, "GITHUB_TOKEN"); + assert.notProperty(environment, "CODEX_MARKETPLACE_REPOSITORY_TOKEN"); assert.notProperty(environment, "SSH_AUTH_SOCK"); assert.notProperty(environment, "HTTPS_PROXY"); }), @@ -301,6 +317,7 @@ describe("Codex marketplace smoke lifecycle", () => { parentEnvironment: { PATH: "/usr/bin:/bin", OPENAI_API_KEY: "must-not-leak", + CODEX_MARKETPLACE_REPOSITORY_TOKEN: "repository-secret", }, }); @@ -335,6 +352,10 @@ describe("Codex marketplace smoke lifecycle", () => { ), ); assert.notProperty(commands[0]?.environment ?? {}, "OPENAI_API_KEY"); + assert.notProperty(commands[0]?.environment ?? {}, "CODEX_MARKETPLACE_REPOSITORY_TOKEN"); + assert.match(commands[0]?.environment.GIT_ASKPASS ?? "", /git-askpass$/u); + assert.notInclude(commands[0]?.environment.GIT_ASKPASS ?? "", "repository-secret"); + assert.isFalse(commands.slice(1).some((command) => "GIT_ASKPASS" in command.environment)); const codexHome = commands[0]?.environment.CODEX_HOME; if (codexHome === undefined) return yield* Effect.die("fake runner did not capture home"); assert.isFalse(yield* fs.exists(path.dirname(codexHome))); diff --git a/packages/evals/src/bin/check-codex-marketplace.ts b/packages/evals/src/bin/check-codex-marketplace.ts index 3a9aedf..42266d2 100755 --- a/packages/evals/src/bin/check-codex-marketplace.ts +++ b/packages/evals/src/bin/check-codex-marketplace.ts @@ -13,6 +13,7 @@ import { CODEX_CLI_ALLOWED_ENVIRONMENT_NAMES } from "../codex-cli"; const DEFAULT_REPOSITORY = "askgina/plugins"; const DEFAULT_EXECUTABLE = "codex"; const DEFAULT_TIMEOUT_MS = 120_000; +const REPOSITORY_TOKEN_ENVIRONMENT_NAME = "CODEX_MARKETPLACE_REPOSITORY_TOKEN"; const MAXIMUM_OUTPUT_BYTES = 1_048_576; const MARKETPLACE_NAME = "ask-gina-plugins"; const PLUGIN_NAME = "ask-gina"; @@ -215,6 +216,13 @@ export const buildCodexMarketplaceEnvironment: { }; }, ); +const withGitAskpass = ( + environment: Readonly>, + askpassFile: string, +): Record => ({ + ...environment, + GIT_ASKPASS: askpassFile, +}); export const parseMarketplaceAddOutput = ( output: string, @@ -578,6 +586,17 @@ export const runCodexMarketplaceSmoke = ( runtimeDirectory: path.join(root, "xdg", "runtime"), workingDirectory: path.join(root, "work"), }; + const repositoryToken = parentEnvironment[REPOSITORY_TOKEN_ENVIRONMENT_NAME]; + if ( + repositoryToken !== undefined && + (repositoryToken.length === 0 || /[\r\n]/u.test(repositoryToken)) + ) { + return yield* fail( + "environment", + "invalid-arguments", + `${REPOSITORY_TOKEN_ENVIRONMENT_NAME} must be non-empty and single-line when provided`, + ); + } yield* Effect.forEach( [ isolation.home, @@ -617,9 +636,77 @@ export const runCodexMarketplaceSmoke = ( ), ); const environment = buildCodexMarketplaceEnvironment(isolation, parentEnvironment); + const repositoryTokenFile = path.join(isolation.home, ".repository-token"); + const gitAskpassFile = path.join(isolation.home, "git-askpass"); + if (repositoryToken !== undefined) { + yield* fs + .writeFileString(repositoryTokenFile, repositoryToken, { flag: "wx", mode: 0o600 }) + .pipe( + Effect.mapError( + () => + new CodexMarketplaceSmokeError({ + stage: "environment", + reason: "io-failed", + detail: "cannot initialize isolated repository token", + }), + ), + ); + yield* fs + .writeFileString( + gitAskpassFile, + `#!/bin/sh +case "$1" in + *sername*) printf '%s\\n' 'x-access-token' ;; + *assword*) cat "$HOME/.repository-token" ;; + *) exit 1 ;; +esac +`, + { flag: "wx", mode: 0o700 }, + ) + .pipe( + Effect.mapError( + () => + new CodexMarketplaceSmokeError({ + stage: "environment", + reason: "io-failed", + detail: "cannot initialize isolated Git askpass helper", + }), + ), + ); + } + const marketplaceAddEnvironment = + repositoryToken === undefined ? environment : withGitAskpass(environment, gitAskpassFile); + const removeRepositoryCredentials = + repositoryToken === undefined + ? Effect.void + : Effect.gen(function* () { + const credentialFiles = [repositoryTokenFile, gitAskpassFile] as const; + for (const file of credentialFiles) { + if (yield* fs.exists(file)) yield* fs.remove(file); + } + for (const file of credentialFiles) { + if (yield* fs.exists(file)) { + return yield* fail( + "environment", + "io-failed", + "isolated repository credentials remain after marketplace clone", + ); + } + } + }).pipe( + Effect.mapError( + () => + new CodexMarketplaceSmokeError({ + stage: "environment", + reason: "io-failed", + detail: "cannot remove isolated repository credentials", + }), + ), + ); const runCommand = ( stage: MarketplaceCommandInput["stage"], args: readonly string[], + commandEnvironment: Readonly> = environment, ): Effect.Effect< MarketplaceCommandResult, CodexMarketplaceSmokeError, @@ -630,7 +717,7 @@ export const runCodexMarketplaceSmoke = ( executable: options.executable, args, cwd: isolation.workingDirectory, - environment, + environment: commandEnvironment, timeoutMs: options.timeoutMs, }); const cleanup = Effect.gen(function* () { @@ -647,15 +734,16 @@ export const runCodexMarketplaceSmoke = ( }); return yield* Effect.gen(function* () { - const marketplaceResult = yield* runCommand("marketplace-add", [ - "plugin", - "marketplace", - "add", - options.repository, - "--ref", - options.ref, - "--json", - ]); + const marketplaceAttempt = yield* Effect.result( + runCommand( + "marketplace-add", + ["plugin", "marketplace", "add", options.repository, "--ref", options.ref, "--json"], + marketplaceAddEnvironment, + ), + ); + yield* removeRepositoryCredentials; + if (marketplaceAttempt._tag === "Failure") return yield* marketplaceAttempt.failure; + const marketplaceResult = marketplaceAttempt.success; const marketplace = yield* parseMarketplaceAddOutput(marketplaceResult.stdout); const marketplaceRoot = yield* fs.realPath(marketplace.installedRoot).pipe( Effect.mapError( diff --git a/specs/codex-repository-marketplace-plan.md b/specs/codex-repository-marketplace-plan.md index 7770119..1053ee3 100644 --- a/specs/codex-repository-marketplace-plan.md +++ b/specs/codex-repository-marketplace-plan.md @@ -2,7 +2,7 @@ ## Goal -A clean checkout of `askgina/plugins` must install Ask Gina through the Codex and ChatGPT repository marketplace without running Bun, Vite+, or artifact generation. The repository keeps one authored skill tree and continues to produce lean host archives. +A clean checkout by a GitHub principal with read access to the private `askgina/plugins` repository must install Ask Gina through the Codex and ChatGPT repository marketplace without running Bun, Vite+, or artifact generation. The repository keeps one authored skill tree and continues to produce lean host archives. The implementation may be merged to `main` and verified through an isolated repository-marketplace install. This plan does not authorize OpenAI universal-directory submission, npm publication, MCP deployment, or installation for end users. @@ -13,6 +13,7 @@ The implementation may be merged to `main` and verified through an isolated repo - [#20](https://github.com/askgina/plugins/issues/20) locks the source and target-generation migration. - [#24](https://github.com/askgina/plugins/issues/24) locks conformance and install verification. - [#23](https://github.com/askgina/plugins/issues/23) orders the work and defines rollback. +- On 2026-08-26, the delivery owner kept the repository private and authorized least-privilege GitHub authentication for remote marketplace clones. This supersedes the credential-free clause in #24 while preserving isolated plugin and MCP authentication boundaries. ## Resulting source tree @@ -40,8 +41,8 @@ The marketplace descriptor has no version field. Version equality applies to ver 6. Update `tools/verify-artifacts.ts` and existing archive tests so the OpenAI archive remains lean and foreign-host-free. 7. Migrate Codex evaluator consumers from MCP server name `gina` to `ask-gina`. Move installed-plugin preflight from the Claude MCP path to root `.mcp.json`. Update the matching evaluator tests. 8. Update plugin-core, sync, conformance, pack, verifier, and evaluator tests at their existing boundaries. Update `docs/architecture.md` to distinguish root OpenAI source, non-OpenAI overlays, generated targets, and ignored artifacts. -9. Add an isolated Codex marketplace smoke command. It must use a temporary `HOME`, config, and cache, exercise the remote owner/repository plus `--ref` syntax proved by #21, inspect the installed files, remove the plugin and marketplace on every exit, and require no credentials. -10. Add the smoke to CI. Treat `ON_INSTALL` as listing metadata in this automated check. It does not prove OAuth or a live MCP connection. +9. Add an isolated Codex marketplace smoke command. It must use a temporary `HOME`, config, and cache, exercise the remote owner/repository plus `--ref` syntax proved by #21, inspect the installed files, and remove the plugin and marketplace on every exit. For the private repository, accept one dedicated read-only repository token through `CODEX_MARKETPLACE_REPOSITORY_TOKEN`, write it only to a mode-0600 temporary token file, expose a token-free temporary `GIT_ASKPASS` helper only to `marketplace add`, delete both files before plugin installation, and never pass the token itself to Codex's environment or arguments. +10. Add the smoke to CI only on a trusted same-repository `push`, never in a pull-request job that executes PR-controlled code. Pull requests keep all credential-free static and artifact gates. Treat `ON_INSTALL` as listing metadata in this automated check; it does not prove OAuth or a live MCP connection. ## Verification @@ -54,14 +55,15 @@ bun run check:target-conformance bun run artifacts bun run verify:artifacts bun run check:public-boundary +# Provide CODEX_MARKETPLACE_REPOSITORY_TOKEN through the process environment. bun run check:marketplace:codex -- --ref ``` -Run an external review after these commands pass. Fix every accepted finding and rerun the affected proof. Merge only a published PR revision with green required checks. +Run the authenticated remote smoke from the same immutable PR-head SHA's trusted `push` workflow, not its pull-request workflow. Run an external review after these commands pass. Fix every accepted finding and rerun the affected proof. Merge only a published PR revision with green required checks. Before the initial merge, manually verify the supported ChatGPT host flow described in #24. Record the tested commit, client and platform, install or connection evidence, and one successful skill and tool response. Codex CLI installation alone does not prove host authentication. -After merge, repeat the isolated remote Codex sequence against the immutable `main` commit SHA. Remove all temporary plugin, marketplace, home, config, cache, build, and worktree state. +After merge, repeat the isolated authenticated remote Codex sequence against the immutable `main` commit SHA. The `main` push workflow is acceptable proof when it reports that exact SHA. Remove all temporary repository credentials, plugin, marketplace, home, config, cache, build, and worktree state. ## Rollback