refactor: moved testing stage to module

Author: jahilldev

api/src/agent/agent.model.ts

@@ -89,22 +89,3 @@ export interface PrBodyPromptContext {
   baseBranch: string;
   branchName: string;
 }
-
-export interface TestPromptContext {
-  repoFullName: string;
-  issueTitle: string;
-  plan: string;
-  hasBrowser: boolean;
-  priorOutput?: string;
-}
-
-export interface TestFailure {
-  name: string;
-  detail: string;
-}
-
-export interface TestResult {
-  passed: boolean;
-  summary: string;
-  failures: TestFailure[];
-}

api/src/agent/agent.prompts.ts

@@ -3,7 +3,6 @@ import {
   type PlanPromptContext,
   type PrBodyPromptContext,
   type RevisePromptContext,
-  type TestPromptContext,
 } from './agent.model.js';
 
 const PLAN_OUTPUT_CONTRACT = `Your response MUST be the complete, detailed implementation plan — not a statement of intent and not a preamble. Use your tools to fully explore the codebase first, then output the full plan in a single response.
@@ -117,46 +116,3 @@ export function buildPrBodyPrompt(ctx: PrBodyPromptContext): string {
     `Output ONLY the PR body as GitHub-flavored Markdown. No preamble, no explanation — just the content.`,
   ].join('\n\n');
 }
-
-const TEST_OUTPUT_CONTRACT = `Output your verdict as the FIRST thing in your response — a \`\`\`json block before any other text:
-\`\`\`json
-{
-  "passed": true | false,
-  "summary": "<one-paragraph summary of what ran and the overall outcome>",
-  "failures": [
-    { "name": "<test name, command, or category that failed>", "detail": "<what went wrong and any relevant output>" }
-  ]
-}
-\`\`\`
-Set "passed" to true only if ALL tests pass AND the application starts and runs correctly. Use an empty array for "failures" when passing. You may include detailed output after the JSON block.`;
-
-export function buildTestPrompt(ctx: TestPromptContext): string {
-  const parts: string[] = [
-    `You are Hermes, an autonomous engineer working in a clone of \`${ctx.repoFullName}\`. Your task is to verify the implementation for: **${ctx.issueTitle}**.`,
-    `--- APPROVED PLAN ---\n${ctx.plan}\n--- END PLAN ---`,
-    `Instructions:
-1. Discover the test suite by inspecting the project structure (look for \`package.json\` test scripts, \`pytest.ini\`, \`jest.config.*\`, \`vitest.config.*\`, \`go.mod\`, \`Makefile\`, etc.).
-2. Run the tests and capture the full output.
-3. **Try to run the application itself** — start the dev server, CLI, or process and verify it launches without errors. For web/browser applications, open the running app in the browser and exercise the key user flows from the acceptance criteria. For CLI tools, invoke the main commands and check the output.
-4. Do NOT modify any source files — if tests fail or the application errors, document what went wrong. Fixes are handled in a separate step.`,
-  ];
-
-  if (ctx.hasBrowser) {
-    parts.push(
-      `A Camofox browser is available. Use it to open the running application and manually verify the acceptance criteria — click through real user flows, not just check that the page loads.`,
-    );
-  }
-
-  if (ctx.priorOutput) {
-    parts.push(
-      `The previous test run ended with this output — use it as your starting point:\n\`\`\`\n${ctx.priorOutput.slice(0, 4000)}\n\`\`\``,
-    );
-  }
-
-  parts.push(
-    `Use \`.olympian/\` as a scratch directory for any temporary files (diffs, logs, etc.) — it is excluded from commits automatically. Do not run git yourself.`,
-    TEST_OUTPUT_CONTRACT,
-  );
-
-  return parts.join('\n\n');
-}

api/src/agent/agent.utility.ts

@@ -2,8 +2,7 @@ import { randomUUID } from 'node:crypto';
 import { writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { spawn } from 'node:child_process';
-import { z } from 'zod';
-import { STDOUT_CAP, type RawSpawnResult, type SpawnSpec, type TestResult } from './agent.model.js';
+import { STDOUT_CAP, type RawSpawnResult, type SpawnSpec } from './agent.model.js';
 
 const HERMES_CONTAINER_HOME = '/root/.hermes';
 const CONTAINER_WORKDIR = '/workspace';
@@ -261,29 +260,3 @@ export function extractJsonBlock(text: string): unknown | null {
 
   return null;
 }
-
-const testFailureSchema = z.object({
-  name: z.string().default(''),
-  detail: z.string().default(''),
-});
-
-const testResultSchema = z.object({
-  passed: z.boolean(),
-  summary: z.string().default(''),
-  failures: z.array(testFailureSchema).default([]),
-});
-
-/**
- * Parses the test agent's stdout into a structured result. Returns null when the
- * agent did not emit a valid JSON verdict (caller treats that as a failing run).
- */
-export function parseTestResult(stdout: string): TestResult | null {
-  const raw = extractJsonBlock(stdout);
-  const parsed = testResultSchema.safeParse(raw);
-
-  if (!raw || !parsed.success) {
-    return null;
-  }
-
-  return parsed.data;
-}

api/src/orchestrator/orchestrator.module.ts

@@ -4,11 +4,20 @@ import { QueueModule } from '../queue/queue.module.js';
 import { AgentModule } from '../agent/agent.module.js';
 import { WorkspaceModule } from '../workspace/workspace.module.js';
 import { ReviewModule } from '../review/review.module.js';
+import { TestingModule } from '../testing/testing.module.js';
 import { GithubModule } from '../github/github.module.js';
 import { OrchestratorService } from './orchestrator.service.js';
 
 @Module({
-  imports: [JobModule, QueueModule, AgentModule, WorkspaceModule, ReviewModule, GithubModule],
+  imports: [
+    JobModule,
+    QueueModule,
+    AgentModule,
+    WorkspaceModule,
+    ReviewModule,
+    TestingModule,
+    GithubModule,
+  ],
   providers: [OrchestratorService],
   exports: [OrchestratorService],
 })

api/src/orchestrator/orchestrator.service.ts

@@ -13,8 +13,8 @@ import {
   buildPlanPrompt,
   buildPrBodyPrompt,
   buildRevisePrompt,
-  buildTestPrompt,
 } from '../agent/agent.prompts.js';
+import { buildTestPrompt } from '../testing/testing.prompts.js';
 import { WorkspaceService } from '../workspace/workspace.service.js';
 import { ReviewService } from '../review/review.service.js';
 import { buildReviewPrompt } from '../review/review.prompts.js';
@@ -23,7 +23,7 @@ import { formatIssues, formatIssuesMarkdown, parseReview } from '../review/revie
 import { GithubService } from '../github/github.service.js';
 import { APPROVAL_PERMISSIONS, type RepoRef, type ReviewFeedback } from '../github/github.model.js';
 import { extractAttachmentUrls } from '../github/github.utility.js';
-import { parseTestResult } from '../agent/agent.utility.js';
+import { TestingService } from '../testing/testing.service.js';
 import {
   type IssueCommentEvent,
   type IssueLabeledEvent,
@@ -59,6 +59,7 @@ export class OrchestratorService {
     private readonly agent: HermesAgentService,
     private readonly workspace: WorkspaceService,
     private readonly review: ReviewService,
+    private readonly testing: TestingService,
     private readonly github: GithubService,
   ) {}
 
@@ -653,7 +654,7 @@ export class OrchestratorService {
     });
     await this.workspace.commitAll(ws.dir, 'test: run test suite');
 
-    const testResult = res.status === 'SUCCEEDED' ? parseTestResult(res.stdout) : null;
+    const testResult = res.status === 'SUCCEEDED' ? this.testing.parse(res.stdout) : null;
 
     if (res.status === 'SUCCEEDED' && testResult?.passed === true) {
       await this.jobs.transition(jobId, 'SELF_REVIEWING', {
@@ -707,14 +708,9 @@ export class OrchestratorService {
         return (lastTestRun.stderr || lastTestRun.stdout || '').slice(0, 4000);
       }
       // Process exited cleanly but the structured verdict said tests failed.
-      const result = parseTestResult(lastTestRun.stdout ?? '');
+      const result = this.testing.parse(lastTestRun.stdout ?? '');
       if (!result || result.passed) return undefined;
-      const lines = [`Summary: ${result.summary}`];
-      if (result.failures.length > 0) {
-        lines.push('Failures:');
-        result.failures.forEach((f, i) => lines.push(`${i + 1}. ${f.name}\n   ${f.detail}`));
-      }
-      return lines.join('\n');
+      return this.testing.formatFailures(result);
     })();
 
     const issues = lastFailedReview ? (JSON.parse(lastFailedReview.issues) as ReviewIssue[]) : [];

api/src/testing/testing.model.ts

@@ -0,0 +1,18 @@
+export interface TestFailure {
+  name: string;
+  detail: string;
+}
+
+export interface TestResult {
+  passed: boolean;
+  summary: string;
+  failures: TestFailure[];
+}
+
+export interface TestPromptContext {
+  repoFullName: string;
+  issueTitle: string;
+  plan: string;
+  hasBrowser: boolean;
+  priorOutput?: string;
+}

api/src/testing/testing.module.ts

@@ -0,0 +1,8 @@
+import { Module } from '@nestjs/common';
+import { TestingService } from './testing.service.js';
+
+@Module({
+  providers: [TestingService],
+  exports: [TestingService],
+})
+export class TestingModule {}

api/src/testing/testing.prompts.ts

@@ -0,0 +1,44 @@
+import { type TestPromptContext } from './testing.model.js';
+
+const TEST_OUTPUT_CONTRACT = `Output your verdict as the FIRST thing in your response — a \`\`\`json block before any other text:
+\`\`\`json
+{
+  "passed": true | false,
+  "summary": "<one-paragraph summary of what ran and the overall outcome>",
+  "failures": [
+    { "name": "<test name, command, or category that failed>", "detail": "<what went wrong and any relevant output>" }
+  ]
+}
+\`\`\`
+Set "passed" to true only if ALL tests pass AND the application starts and runs correctly. Use an empty array for "failures" when passing. You may include detailed output after the JSON block.`;
+
+export function buildTestPrompt(ctx: TestPromptContext): string {
+  const parts: string[] = [
+    `You are Hermes, an autonomous engineer working in a clone of \`${ctx.repoFullName}\`. Your task is to verify the implementation for: **${ctx.issueTitle}**.`,
+    `--- APPROVED PLAN ---\n${ctx.plan}\n--- END PLAN ---`,
+    `Instructions:
+1. Discover the test suite by inspecting the project structure (look for \`package.json\` test scripts, \`pytest.ini\`, \`jest.config.*\`, \`vitest.config.*\`, \`go.mod\`, \`Makefile\`, etc.).
+2. Run the tests and capture the full output.
+3. **Try to run the application itself** — start the dev server, CLI, or process and verify it launches without errors. For web/browser applications, open the running app in the browser and exercise the key user flows from the acceptance criteria. For CLI tools, invoke the main commands and check the output.
+4. Do NOT modify any source files — if tests fail or the application errors, document what went wrong. Fixes are handled in a separate step.`,
+  ];
+
+  if (ctx.hasBrowser) {
+    parts.push(
+      `A Camofox browser is available. Use it to open the running application and manually verify the acceptance criteria — click through real user flows, not just check that the page loads.`,
+    );
+  }
+
+  if (ctx.priorOutput) {
+    parts.push(
+      `The previous test run ended with this output — use it as your starting point:\n\`\`\`\n${ctx.priorOutput.slice(0, 4000)}\n\`\`\``,
+    );
+  }
+
+  parts.push(
+    `Use \`.olympian/\` as a scratch directory for any temporary files (diffs, logs, etc.) — it is excluded from commits automatically. Do not run git yourself.`,
+    TEST_OUTPUT_CONTRACT,
+  );
+
+  return parts.join('\n\n');
+}

api/src/testing/testing.service.ts

@@ -0,0 +1,25 @@
+import { Injectable } from '@nestjs/common';
+import { AppConfigService } from '../config/config.service.js';
+import { type TestResult } from './testing.model.js';
+import { formatTestFailures, parseTestResult } from './testing.utility.js';
+
+/**
+ * Owns the TEST phase policy: structured verdict parsing, failure formatting, and
+ * the iteration cap that prevents the TEST→REVISE loop from running indefinitely.
+ */
+@Injectable()
+export class TestingService {
+  constructor(private readonly config: AppConfigService) {}
+
+  get maxIterations(): number {
+    return this.config.get('MAX_TEST_ITERATIONS');
+  }
+
+  parse(stdout: string): TestResult | null {
+    return parseTestResult(stdout);
+  }
+
+  formatFailures(result: TestResult): string {
+    return formatTestFailures(result);
+  }
+}

api/src/testing/testing.utility.ts

@@ -0,0 +1,39 @@
+import { z } from 'zod';
+import { extractJsonBlock } from '../agent/agent.utility.js';
+import { type TestFailure, type TestResult } from './testing.model.js';
+
+const testFailureSchema = z.object({
+  name: z.string().default(''),
+  detail: z.string().default(''),
+});
+
+const testResultSchema = z.object({
+  passed: z.boolean(),
+  summary: z.string().default(''),
+  failures: z.array(testFailureSchema).default([]),
+});
+
+/**
+ * Parses the test agent's stdout into a structured result. Returns null when the
+ * agent did not emit a valid JSON verdict (caller treats that as a failing run).
+ */
+export function parseTestResult(stdout: string): TestResult | null {
+  const raw = extractJsonBlock(stdout);
+  const parsed = testResultSchema.safeParse(raw);
+  if (!raw || !parsed.success) {
+    return null;
+  }
+  return parsed.data;
+}
+
+/** Formats a parsed TestResult's failures as a human-readable string for the REVISE prompt. */
+export function formatTestFailures(result: TestResult): string {
+  const lines = [`Summary: ${result.summary}`];
+  if (result.failures.length > 0) {
+    lines.push('Failures:');
+    result.failures.forEach((f: TestFailure, i: number) =>
+      lines.push(`${i + 1}. ${f.name}\n   ${f.detail}`),
+    );
+  }
+  return lines.join('\n');
+}