fix: corrected test stage to structured output

Author: jahilldev

api/src/agent/agent.model.ts

@@ -97,3 +97,14 @@ export interface TestPromptContext {
   hasBrowser: boolean;
   priorOutput?: string;
 }
+
+export interface TestFailure {
+  name: string;
+  detail: string;
+}
+
+export interface TestResult {
+  passed: boolean;
+  summary: string;
+  failures: TestFailure[];
+}

api/src/agent/agent.prompts.ts

@@ -118,20 +118,32 @@ export function buildPrBodyPrompt(ctx: PrBodyPromptContext): string {
   ].join('\n\n');
 }
 
+const TEST_OUTPUT_CONTRACT = `Output your verdict as the FIRST thing in your response — a \`\`\`json block before any other text:
+\`\`\`json
+{
+  "passed": true | false,
+  "summary": "<one-paragraph summary of what ran and the overall outcome>",
+  "failures": [
+    { "name": "<test name, command, or category that failed>", "detail": "<what went wrong and any relevant output>" }
+  ]
+}
+\`\`\`
+Set "passed" to true only if ALL tests pass AND the application starts and runs correctly. Use an empty array for "failures" when passing. You may include detailed output after the JSON block.`;
+
 export function buildTestPrompt(ctx: TestPromptContext): string {
   const parts: string[] = [
-    `You are Hermes, an autonomous engineer working in a clone of \`${ctx.repoFullName}\`. Your task is to ensure the test suite passes for the changes made to resolve: **${ctx.issueTitle}**.`,
+    `You are Hermes, an autonomous engineer working in a clone of \`${ctx.repoFullName}\`. Your task is to verify the implementation for: **${ctx.issueTitle}**.`,
     `--- APPROVED PLAN ---\n${ctx.plan}\n--- END PLAN ---`,
     `Instructions:
 1. Discover the test suite by inspecting the project structure (look for \`package.json\` test scripts, \`pytest.ini\`, \`jest.config.*\`, \`vitest.config.*\`, \`go.mod\`, \`Makefile\`, etc.).
 2. Run the tests and capture the full output.
 3. **Try to run the application itself** — start the dev server, CLI, or process and verify it launches without errors. For web/browser applications, open the running app in the browser and exercise the key user flows from the acceptance criteria. For CLI tools, invoke the main commands and check the output.
-4. Report the results. Do NOT modify any source files — if tests fail or the application errors, document what went wrong and stop. Fixes are handled in a separate step.`,
+4. Do NOT modify any source files — if tests fail or the application errors, document what went wrong. Fixes are handled in a separate step.`,
   ];
 
   if (ctx.hasBrowser) {
     parts.push(
-      `A Camofox browser is available for step 5. Use it to open the running application and manually verify the acceptance criteria — click through real user flows, not just check that the page loads.`,
+      `A Camofox browser is available. Use it to open the running application and manually verify the acceptance criteria — click through real user flows, not just check that the page loads.`,
     );
   }
 
@@ -142,7 +154,8 @@ export function buildTestPrompt(ctx: TestPromptContext): string {
   }
 
   parts.push(
-    `When done, write a brief summary of what ran, what passed, and what failed or errored (if anything). Use \`.olympian/\` as a scratch directory for any temporary files (diffs, logs, etc.) — it is excluded from commits automatically. Do not run git yourself.`,
+    `Use \`.olympian/\` as a scratch directory for any temporary files (diffs, logs, etc.) — it is excluded from commits automatically. Do not run git yourself.`,
+    TEST_OUTPUT_CONTRACT,
   );
 
   return parts.join('\n\n');

api/src/agent/agent.utility.ts

@@ -2,7 +2,8 @@ import { randomUUID } from 'node:crypto';
 import { writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { spawn } from 'node:child_process';
-import { STDOUT_CAP, type RawSpawnResult, type SpawnSpec } from './agent.model.js';
+import { z } from 'zod';
+import { STDOUT_CAP, type RawSpawnResult, type SpawnSpec, type TestResult } from './agent.model.js';
 
 const HERMES_CONTAINER_HOME = '/root/.hermes';
 const CONTAINER_WORKDIR = '/workspace';
@@ -260,3 +261,29 @@ export function extractJsonBlock(text: string): unknown | null {
 
   return null;
 }
+
+const testFailureSchema = z.object({
+  name: z.string().default(''),
+  detail: z.string().default(''),
+});
+
+const testResultSchema = z.object({
+  passed: z.boolean(),
+  summary: z.string().default(''),
+  failures: z.array(testFailureSchema).default([]),
+});
+
+/**
+ * Parses the test agent's stdout into a structured result. Returns null when the
+ * agent did not emit a valid JSON verdict (caller treats that as a failing run).
+ */
+export function parseTestResult(stdout: string): TestResult | null {
+  const raw = extractJsonBlock(stdout);
+  const parsed = testResultSchema.safeParse(raw);
+
+  if (!raw || !parsed.success) {
+    return null;
+  }
+
+  return parsed.data;
+}

api/src/orchestrator/orchestrator.service.ts

@@ -23,6 +23,7 @@ import { formatIssues, formatIssuesMarkdown, parseReview } from '../review/revie
 import { GithubService } from '../github/github.service.js';
 import { APPROVAL_PERMISSIONS, type RepoRef, type ReviewFeedback } from '../github/github.model.js';
 import { extractAttachmentUrls } from '../github/github.utility.js';
+import { parseTestResult } from '../agent/agent.utility.js';
 import {
   type IssueCommentEvent,
   type IssueLabeledEvent,
@@ -652,15 +653,20 @@ export class OrchestratorService {
     });
     await this.workspace.commitAll(ws.dir, 'test: run test suite');
 
-    if (res.status === 'SUCCEEDED') {
+    const testResult = res.status === 'SUCCEEDED' ? parseTestResult(res.stdout) : null;
+
+    if (res.status === 'SUCCEEDED' && testResult?.passed === true) {
       await this.jobs.transition(jobId, 'SELF_REVIEWING', {
         reason: 'tests passed',
         actor: 'AGENT',
       });
       await this.queue.enqueue({ jobId, kind: 'REVIEW' });
     } else {
-      this.logger.warn(`[job ${jobId}] test ${res.status}; routing to revise`);
-      await this.jobs.transition(jobId, 'REVISING', { reason: 'tests failed', actor: 'AGENT' });
+      const reason = testResult
+        ? `tests failed: ${testResult.failures.map((f) => f.name).join(', ') || 'see summary'}`
+        : `test agent ${res.status}`;
+      this.logger.warn(`[job ${jobId}] ${reason}; routing to revise`);
+      await this.jobs.transition(jobId, 'REVISING', { reason, actor: 'AGENT' });
       await this.queue.enqueue({ jobId, kind: 'REVISE' });
     }
   }
@@ -695,10 +701,18 @@ export class OrchestratorService {
       }),
     ]);
 
-    const testOutput =
-      lastTestRun && lastTestRun.status !== 'SUCCEEDED'
-        ? (lastTestRun.stderr || lastTestRun.stdout || '').slice(0, 4000)
-        : undefined;
+    const testOutput = (() => {
+      if (!lastTestRun) return undefined;
+      if (lastTestRun.status !== 'SUCCEEDED') {
+        return (lastTestRun.stderr || lastTestRun.stdout || '').slice(0, 4000);
+      }
+      // Process exited cleanly but the structured verdict said tests failed.
+      const result = parseTestResult(lastTestRun.stdout ?? '');
+      if (result && !result.passed) {
+        return (lastTestRun.stdout ?? '').slice(0, 4000);
+      }
+      return undefined;
+    })();
 
     const issues = lastFailedReview ? (JSON.parse(lastFailedReview.issues) as ReviewIssue[]) : [];
     const issuesText = issues.length > 0 ? formatIssues(issues) : undefined;