feat: introduced deterministic and browser based testing

Author: jahilldev

api/src/agent/agent.model.ts

@@ -1,4 +1,4 @@
-export const AGENT_PHASES = ['PLAN', 'IMPLEMENT', 'REVIEW', 'REVISE', 'PR_BODY'] as const;
+export const AGENT_PHASES = ['PLAN', 'IMPLEMENT', 'TEST', 'REVIEW', 'REVISE', 'PR_BODY'] as const;
 export type AgentPhase = (typeof AGENT_PHASES)[number];
 
 export type AgentRunStatus = 'RUNNING' | 'SUCCEEDED' | 'FAILED' | 'TIMED_OUT';
@@ -83,3 +83,11 @@ export interface PrBodyPromptContext {
   baseBranch: string;
   branchName: string;
 }
+
+export interface TestPromptContext {
+  repoFullName: string;
+  issueTitle: string;
+  plan: string;
+  hasBrowser: boolean;
+  priorOutput?: string;
+}

api/src/agent/agent.prompts.ts

@@ -3,6 +3,7 @@ import {
   type PlanPromptContext,
   type PrBodyPromptContext,
   type RevisePromptContext,
+  type TestPromptContext,
 } from './agent.model.js';
 
 const PLAN_OUTPUT_CONTRACT = `Respond with ONLY the implementation plan as GitHub-flavored Markdown.
@@ -75,3 +76,29 @@ export function buildPrBodyPrompt(ctx: PrBodyPromptContext): string {
     `Output ONLY the PR body as GitHub-flavored Markdown. No preamble, no explanation — just the content.`,
   ].join('\n\n');
 }
+
+export function buildTestPrompt(ctx: TestPromptContext): string {
+  const parts: string[] = [
+    `You are Hermes, an autonomous engineer working in a clone of \`${ctx.repoFullName}\`. Your task is to ensure the test suite passes for the changes made to resolve: **${ctx.issueTitle}**.`,
+    `--- APPROVED PLAN ---\n${ctx.plan}\n--- END PLAN ---`,
+    `Instructions:
+1. Discover the test suite by inspecting the project structure (look for \`package.json\` test scripts, \`pytest.ini\`, \`jest.config.*\`, \`vitest.config.*\`, \`go.mod\`, \`Makefile\`, etc.).
+2. Run the tests and capture the output.
+3. If tests fail, diagnose the root cause from the output and fix the code.
+4. Re-run until all tests pass or you have exhausted your budget.`,
+  ];
+  if (ctx.hasBrowser) {
+    parts.push(
+      `A browser is available. Use browser tools to manually exercise the key user flows described in the plan's acceptance criteria.`,
+    );
+  }
+  if (ctx.priorOutput) {
+    parts.push(
+      `The previous test run ended with this output — use it as your starting point:\n\`\`\`\n${ctx.priorOutput.slice(0, 4000)}\n\`\`\``,
+    );
+  }
+  parts.push(
+    `When done, write a brief summary: which tests ran, which passed, which failed (if any), and what fixes you applied. The orchestrator will commit any file changes — do not run git yourself.`,
+  );
+  return parts.join('\n\n');
+}

api/src/config/config.model.ts

@@ -37,6 +37,7 @@ export const envSchema = z.object({
   MAX_PLAN_REVISIONS: intFromString(10),
   MAX_IMPLEMENTATION_ITERATIONS: intFromString(5),
   MAX_REVIEW_PASSES: intFromString(5),
+  MAX_TEST_ITERATIONS: intFromString(3),
   COMMAND_PREFIX: z.string().default('/hermes'),
 
   // queue / worker

api/src/job/job.model.ts

@@ -6,6 +6,7 @@ export const JOB_STATES = [
   'PLANNING',
   'AWAITING_PLAN_APPROVAL',
   'IMPLEMENTING',
+  'TESTING',
   'SELF_REVIEWING',
   'REVISING',
   'OPENING_PR',
@@ -28,7 +29,8 @@ export const ALLOWED_TRANSITIONS: Record<JobState, JobState[]> = {
   TRIAGED: ['PLANNING'],
   PLANNING: ['AWAITING_PLAN_APPROVAL'],
   AWAITING_PLAN_APPROVAL: ['PLANNING', 'IMPLEMENTING'],
-  IMPLEMENTING: ['SELF_REVIEWING'],
+  IMPLEMENTING: ['TESTING'],
+  TESTING: ['SELF_REVIEWING'],
   SELF_REVIEWING: ['REVISING', 'OPENING_PR'],
   REVISING: ['SELF_REVIEWING'],
   OPENING_PR: ['AWAITING_PR_APPROVAL'],

api/src/metrics/metrics.model.ts

@@ -1,4 +1,4 @@
 export const METRIC_PREFIX = 'hermes_';
 
-export type AgentPhaseLabel = 'PLAN' | 'IMPLEMENT' | 'REVIEW' | 'REVISE' | 'PR_BODY';
+export type AgentPhaseLabel = 'PLAN' | 'IMPLEMENT' | 'TEST' | 'REVIEW' | 'REVISE' | 'PR_BODY';
 export type AgentRunStatusLabel = 'SUCCEEDED' | 'FAILED' | 'TIMED_OUT';

api/src/orchestrator/orchestrator.service.ts

@@ -13,6 +13,7 @@ import {
   buildPlanPrompt,
   buildPrBodyPrompt,
   buildRevisePrompt,
+  buildTestPrompt,
 } from '../agent/agent.prompts.js';
 import { WorkspaceService } from '../workspace/workspace.service.js';
 import { ReviewService } from '../review/review.service.js';
@@ -22,7 +23,6 @@ import { formatIssues, parseReview } from '../review/review.utility.js';
 import { GithubService } from '../github/github.service.js';
 import {
   APPROVAL_PERMISSIONS,
-  type AttachmentRef,
   type RepoRef,
   type ReviewFeedback,
 } from '../github/github.model.js';
@@ -364,6 +364,8 @@ export class OrchestratorService {
         return this.handlePlan(task.jobId);
       case 'IMPLEMENT':
         return this.handleImplement(task.jobId);
+      case 'TEST':
+        return this.handleTest(task.jobId);
       case 'REVIEW':
         return this.handleReview(task.jobId);
       case 'OPEN_PR':
@@ -545,11 +547,58 @@ export class OrchestratorService {
     if (!committedSomething && !(await this.workspace.hasCommitsAhead(ws.dir, ws.baseBranch))) {
       throw new Error('agent produced no changes');
     }
-    await this.jobs.transition(jobId, 'SELF_REVIEWING', {
+    await this.jobs.transition(jobId, 'TESTING', {
       reason: 'implementation complete',
       actor: 'AGENT',
     });
     await this.jobs.update(jobId, { reviewCycle: { increment: 1 } });
+    await this.queue.enqueue({ jobId, kind: 'TEST' });
+  }
+
+  private async handleTest(jobId: string): Promise<void> {
+    const { job, ref } = await this.context(jobId);
+    const ws = await this.workspace.prepare({
+      jobId,
+      installationId: this.ghIdFromRef(ref),
+      owner: job.repoOwner,
+      repo: job.repoName,
+      branchName:
+        job.branchName ?? branchNameFor(this.config.get('BRANCH_PREFIX'), job.issueNumber),
+    });
+    const plan = await this.approvedPlan(jobId);
+    const hasBrowser = !!process.env.CAMOFOX_URL;
+    const maxIters = this.config.get('MAX_TEST_ITERATIONS');
+
+    let priorOutput: string | undefined;
+    for (let attempt = 1; attempt <= maxIters; attempt++) {
+      const prompt = buildTestPrompt({
+        repoFullName: job.repoFullName,
+        issueTitle: job.issueTitle,
+        plan,
+        hasBrowser,
+        priorOutput,
+      });
+      const res = await this.agent.run({
+        jobId,
+        phase: 'TEST',
+        cwd: ws.dir,
+        prompt,
+        toolsets: hasBrowser ? 'browser' : undefined,
+      });
+      await this.workspace.commitAll(ws.dir, `test: fix failing tests (attempt ${attempt})`);
+      if (res.status === 'SUCCEEDED') {
+        break;
+      }
+      priorOutput = (res.stderr || res.stdout).slice(0, 4000);
+      this.logger.warn(
+        `[job ${jobId}] test attempt ${attempt} ${res.status}; ${attempt < maxIters ? 'retrying' : 'proceeding to review'}`,
+      );
+    }
+
+    await this.jobs.transition(jobId, 'SELF_REVIEWING', {
+      reason: 'testing complete',
+      actor: 'AGENT',
+    });
     await this.queue.enqueue({ jobId, kind: 'REVIEW' });
   }
 
@@ -575,6 +624,25 @@ export class OrchestratorService {
 
     for (let iter = 1; iter <= maxPasses; iter++) {
       const pass = priorPasses + iter;
+      // Run tests before each review pass. The test agent fixes what it can; any
+      // remaining failures become evidence for the review agent.
+      const hasBrowser = !!process.env.CAMOFOX_URL;
+      const testRes = await this.agent.run({
+        jobId,
+        phase: 'TEST',
+        cwd: ws.dir,
+        prompt: buildTestPrompt({
+          repoFullName: job.repoFullName,
+          issueTitle: job.issueTitle,
+          plan,
+          hasBrowser,
+        }),
+        toolsets: hasBrowser ? 'browser' : undefined,
+      });
+      await this.workspace.commitAll(ws.dir, `test: fix failing tests (review pass ${pass})`);
+      if (testRes.status !== 'SUCCEEDED') {
+        this.logger.warn(`[job ${jobId}] inline test pass ${pass} ${testRes.status}; proceeding to review`);
+      }
       // Idempotent: a no-op when already SELF_REVIEWING, REVISING -> SELF_REVIEWING otherwise.
       await this.jobs.transition(jobId, 'SELF_REVIEWING', {
         reason: `review pass ${pass}`,

api/src/queue/queue.model.ts

@@ -1,4 +1,4 @@
-export const TASK_KINDS = ['PLAN', 'IMPLEMENT', 'REVIEW', 'REVISE', 'OPEN_PR'] as const;
+export const TASK_KINDS = ['PLAN', 'IMPLEMENT', 'TEST', 'REVIEW', 'REVISE', 'OPEN_PR'] as const;
 export type TaskKind = (typeof TASK_KINDS)[number];
 
 export const TASK_STATUSES = ['PENDING', 'RUNNING', 'DONE', 'FAILED'] as const;