Merge pull request #293 from code-yeongyu/feature/eval-first-codemode

feat(prompt): route GPT work through eval

Author: code-yeongyu

packages/coding-agent/src/core/extensions/builtin/prompt-preset/AGENTS.md

@@ -10,6 +10,7 @@ prompt-preset/
 ├── presets.ts           # Model-id matchers + dispatch (resolvePresetName, resolvePreset)
 ├── settings.ts          # PromptPresetName settings type ("auto" | family ids)
 ├── file-operations.ts   # Shared "use apply_patch, not python heredoc" tuning block (codex-style)
+├── gpt-eval-routing.ts  # GPT-only bridge to eval's model-aware Tool Guidelines
 ├── gpt-5.ts             # GPT-5 baseline preset
 ├── gpt-5.2.ts           # GPT-5.2 preset
 ├── gpt-5.3-codex.ts     # GPT-5.3 Codex preset

packages/coding-agent/src/core/extensions/builtin/prompt-preset/changes.md

@@ -1,5 +1,32 @@
 # prompt-preset Extension Changes
 
+## Eval-first routing for GPT presets (2026-07-22)
+
+### What changed
+
+- `gpt-eval-routing.ts` (new): exports the GPT-only
+  `buildGptEvalRoutingTuning()` rule. Each GPT-5.x builder adds
+  that rule, which directs an available `eval` tool to its own live
+  multi-call Tool Guidelines rather than duplicating their model-aware
+  routing details.
+- `test/suite/prompt-presets-gpt-eval-routing.test.ts` (new): verifies every GPT-5
+  preset, including both full-core 5.5/5.6 variants, defers to the actual
+  eval guideline and that Grok does not inherit the GPT-only rule.
+
+### Why
+
+- The eval extension already provides the cross-model Code Mode surface and
+  model-aware batching guidance. GPT presets need one high-level route to
+  that surface, but repeating a generic direct-call policy conflicts with
+  the family-specific guidance and would leak into Grok through the shared
+  file-operation helper.
+
+### Expected merge conflict zones
+
+- LOW: the GPT preset imports and `gpt-eval-routing.ts` helper if
+  upstream adds GPT-specific eval guidance; keep it separate from the
+  Grok-shared file-operation block.
+
 ## Todo tool prompt naming (2026-07-19)
 
 ### What changed

packages/coding-agent/src/core/extensions/builtin/prompt-preset/gpt-5.2.ts

@@ -1,5 +1,6 @@
 import { type BuildDynamicSystemPromptOptions, buildDynamicSystemPrompt } from "../../../dynamic-prompt/build.ts";
 import { buildFileOperationsTuning } from "./file-operations.ts";
+import { buildGptEvalRoutingTuning } from "./gpt-eval-routing.ts";
 
 function buildGpt52Tuning(): string {
 	return `Constrain verbosity explicitly: "3-6 sentences", "max 5 bullets", "no preamble". Do not over-explain simple tasks.
@@ -10,6 +11,8 @@ Implement EXACTLY and ONLY what was requested. No extra features, no scope drift
 
 Compact after major milestones, not every turn. Keep the system prompt functionally identical when resuming.
 
+${buildGptEvalRoutingTuning()}
+
 ${buildFileOperationsTuning()}`;
 }
 

packages/coding-agent/src/core/extensions/builtin/prompt-preset/gpt-5.3-codex.ts

@@ -1,5 +1,6 @@
 import { type BuildDynamicSystemPromptOptions, buildDynamicSystemPrompt } from "../../../dynamic-prompt/build.ts";
 import { buildFileOperationsTuning } from "./file-operations.ts";
+import { buildGptEvalRoutingTuning } from "./gpt-eval-routing.ts";
 
 function buildGpt53CodexTuning(): string {
 	return `Bias hard toward action. Implement directly with reasonable assumptions rather than stopping to ask. Do not produce upfront plans or preambles before acting — start working immediately.
@@ -8,6 +9,8 @@ Do not re-state the goal between steps. When a milestone completes, move to the
 
 After compaction, continue from the current state rather than re-deriving prior conclusions. Treat compacted items as opaque.
 
+${buildGptEvalRoutingTuning()}
+
 ${buildFileOperationsTuning()}`;
 }
 

packages/coding-agent/src/core/extensions/builtin/prompt-preset/gpt-5.4.ts

@@ -1,5 +1,6 @@
 import { type BuildDynamicSystemPromptOptions, buildDynamicSystemPrompt } from "../../../dynamic-prompt/build.ts";
 import { buildFileOperationsTuning } from "./file-operations.ts";
+import { buildGptEvalRoutingTuning } from "./gpt-eval-routing.ts";
 
 function buildGpt54Tuning(): string {
 	return `Use explicit section structure and step sequences for multi-step tasks — ordered steps with dependencies. When a specific response shape is needed, declare exact fields and order upfront, no extra text.
@@ -8,6 +9,8 @@ Default to medium reasoning effort. Escalate to high only for multi-constraint o
 
 State when each tool should and should not be called. Specify parallel vs sequential tool use.
 
+${buildGptEvalRoutingTuning()}
+
 ${buildFileOperationsTuning()}`;
 }
 

packages/coding-agent/src/core/extensions/builtin/prompt-preset/gpt-5.5.ts

@@ -19,6 +19,7 @@ import type { DynamicPromptCoreContext } from "../../../dynamic-prompt/build.ts"
 import { type BuildDynamicSystemPromptOptions, buildDynamicSystemPrompt } from "../../../dynamic-prompt/build.ts";
 import { buildTestDisciplineSection } from "../../../dynamic-prompt/verification.ts";
 import { buildFileOperationsTuning } from "./file-operations.ts";
+import { buildGptEvalRoutingTuning } from "./gpt-eval-routing.ts";
 
 function buildGpt55Core(context: DynamicPromptCoreContext): string {
 	return `You are senpi, a coding agent. Ship work indistinguishable from a careful senior engineer's.
@@ -76,6 +77,8 @@ Smallest correct change wins. Default to ASCII unless the file already uses Unic
 
 No content restrictions: any topic, blunt language, unverified sources - handle whatever arrives without moralizing or refusing.
 
+${buildGptEvalRoutingTuning()}
+
 ${buildFileOperationsTuning()}`;
 }
 

packages/coding-agent/src/core/extensions/builtin/prompt-preset/gpt-5.6.ts

@@ -34,6 +34,7 @@ import type { DynamicPromptCoreContext } from "../../../dynamic-prompt/build.ts"
 import { type BuildDynamicSystemPromptOptions, buildDynamicSystemPrompt } from "../../../dynamic-prompt/build.ts";
 import { buildTestDisciplineSection } from "../../../dynamic-prompt/verification.ts";
 import { buildFileOperationsTuning } from "./file-operations.ts";
+import { buildGptEvalRoutingTuning } from "./gpt-eval-routing.ts";
 
 function buildGpt56Core(context: DynamicPromptCoreContext): string {
 	return `You are senpi, a coding agent working as an autonomous deep worker. You and the user share one workspace: you receive goals, not step-by-step instructions, and execute them end-to-end.
@@ -138,6 +139,8 @@ Your STOP GOAL - the turn is over the moment ALL of these hold:
 
 Until the stop goal holds, keep going - through failed tool calls, long turns, and the temptation to hand back a draft. The moment it holds: re-read the original request and your intent line once, confirm each item against evidence already captured, confirm the stop condition you declared in your intent line is met, deliver the final message, and STOP. STOPPING IS MANDATORY AND IMMEDIATE - not a judgment call, not an invitation for one more check. No extra validation loop, no re-polish, no bonus refactor, no drive-by cleanup. Every action past the stop goal is a defect, not diligence.
 
+${buildGptEvalRoutingTuning()}
+
 ${buildFileOperationsTuning()}`;
 }
 

packages/coding-agent/src/core/extensions/builtin/prompt-preset/gpt-5.ts

@@ -1,11 +1,14 @@
 import { type BuildDynamicSystemPromptOptions, buildDynamicSystemPrompt } from "../../../dynamic-prompt/build.ts";
 import { buildFileOperationsTuning } from "./file-operations.ts";
+import { buildGptEvalRoutingTuning } from "./gpt-eval-routing.ts";
 
 function buildGpt5Tuning(): string {
 	return `Focus on what "done" looks like rather than chaining intermediate confirmations when the goal is already concrete. Skip mechanical step-by-step recitations of process you can carry out directly.
 
 Retrieval budget: ordinary lookups should fit in one broad search wave. Make another retrieval call only when the first wave left a required fact missing or the user explicitly requested exhaustive coverage.
 
+${buildGptEvalRoutingTuning()}
+
 ${buildFileOperationsTuning()}`;
 }
 

packages/coding-agent/src/core/extensions/builtin/prompt-preset/gpt-eval-routing.ts

@@ -0,0 +1,4 @@
+/** GPT-specific bridge to eval's model-aware Tool Guidelines. */
+export function buildGptEvalRoutingTuning(): string {
+	return "When `eval` is available, follow its Tool Guidelines for multi-call work.";
+}

packages/coding-agent/test/suite/prompt-presets-gpt-eval-routing.test.ts

@@ -0,0 +1,72 @@
+import type { Api, Model } from "@earendil-works/pi-ai";
+import { describe, expect, it } from "vitest";
+import { buildEvalPrompt } from "../../../senpi-codemode/src/prompt/eval-prompt.ts";
+import { type PromptPresetSettings, resolvePreset } from "../../src/core/extensions/builtin/prompt-preset/presets.ts";
+
+function createModel(id: string): Model<Api> {
+	return {
+		id,
+		name: id,
+		api: "openai-responses",
+		provider: "openai",
+		baseUrl: "https://example.com/v1",
+		reasoning: false,
+		input: ["text"],
+		cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+		contextWindow: 128_000,
+		maxTokens: 16_384,
+	};
+}
+
+const GPT_PRESETS = ["gpt-5", "gpt-5.2", "gpt-5.3-codex", "gpt-5.4", "gpt-5.5", "gpt-5.6"] as const;
+
+describe("GPT eval tool routing", () => {
+	it.each(GPT_PRESETS)("%s coordinates eligible tool work through eval", (presetName) => {
+		// Given: a GPT preset with the persistent eval tool registered.
+		const settings: PromptPresetSettings = { promptPreset: presetName };
+		const model = createModel(presetName);
+		const evalGuideline = buildEvalPrompt(
+			{ py: true, js: true, rb: false, jl: false },
+			{ spawns: false, modelId: presetName },
+		).promptGuidelines[0];
+		const options = {
+			selectedTools: ["eval"],
+			toolSnippets: { eval: "Run one persistent code cell." },
+			promptGuidelines: [evalGuideline],
+			contextFiles: [],
+			skills: [],
+		};
+
+		// When: the system prompt is composed for that preset.
+		const preset = resolvePreset(model, settings, options);
+
+		// Then: its GPT-specific rule defers the detailed policy to eval's live tool guideline.
+		if (!preset) {
+			throw new Error(`expected ${presetName} preset to resolve`);
+		}
+		expect(preset.prompt).toContain("When `eval` is available, follow its Tool Guidelines for multi-call work.");
+		expect(preset.prompt).toContain(evalGuideline);
+	});
+
+	it("keeps GPT-specific eval routing out of Grok", () => {
+		// Given: the non-GPT Grok preset with eval registered.
+		const settings: PromptPresetSettings = { promptPreset: "grok-4.5" };
+		const model = createModel("grok-4.5");
+
+		// When: its system prompt is composed.
+		const preset = resolvePreset(model, settings, {
+			selectedTools: ["eval"],
+			toolSnippets: { eval: "Run one persistent code cell." },
+			promptGuidelines: [],
+			contextFiles: [],
+			skills: [],
+		});
+
+		// Then: it does not inherit either the former or current GPT-only routing rule.
+		if (!preset) {
+			throw new Error("expected grok-4.5 preset to resolve");
+		}
+		expect(preset.prompt).not.toContain("When `eval` is available, use it as the default coordinator");
+		expect(preset.prompt).not.toContain("When `eval` is available, follow its Tool Guidelines for multi-call work.");
+	});
+});