fix(benchmark): bound the LLM-judge call with --timeout

docs/commands/benchmark.md

@@ -32,7 +32,7 @@ Use `--preset <name>` to quickly configure for your hardware:
 | `--preset <name>` | Use a hardware profile: `low`, `medium`, `high`, `ultra` |
 | `-m, --model <path>` | Path to model file |
 | `-d, --dataset <path>` | Path to benchmark dataset |
-| `-t, --timeout <ms>` | Timeout per test (default: 30000) |
+| `-t, --timeout <ms>` | Timeout per model call — generation, and the LLM-judge call that scores it (default: 30000) |
 | `--threads <n>` | CPU threads (default: auto) |
 | `--gpu-layers <n>` | GPU layers to offload (default: max) |
 | `--ctx-size <n>` | Context size in tokens (default: 4096) |

src/commands/benchmark.tsx

@@ -662,19 +662,45 @@ export function BenchmarkCommand({options}: Props) {
 								const judgePrompt = test.messages
 									? formatConversationForJudge(test.messages, contextMsg)
 									: (test.prompt as string);
-								const judgeResult = await callJudge(
-									judgePrompt,
-									sampleResponse.trim(),
-									criteria,
-									judgeConfig,
-									threshold,
-									test.acceptable,
-								);
-								samplePassed = judgeResult.pass;
-								if (sample === 0) {
-									judgeScore = judgeResult.score;
-									judgeReasoning = judgeResult.reasoning;
-									judgeCriteriaScores = judgeResult.criteriaScores;
+								// The judge decides pass/fail, so a provider that stalls hangs
+								// the run exactly as a stalled generation would — and the
+								// inference timer above has already been cleared by the time we
+								// reach here. Give judging its own budget rather than extending
+								// that one: this is a second call, and a slow-but-fine generation
+								// must not eat the time the judge needs and flip a pass to a fail.
+								const judgeController = new AbortController();
+								const judgeTimeoutId = setTimeout(() => {
+									judgeController.abort();
+								}, timeout);
+								try {
+									const judgeResult = await callJudge(
+										judgePrompt,
+										sampleResponse.trim(),
+										criteria,
+										judgeConfig,
+										threshold,
+										test.acceptable,
+										judgeController.signal,
+									);
+									samplePassed = judgeResult.pass;
+									if (sample === 0) {
+										judgeScore = judgeResult.score;
+										judgeReasoning = judgeResult.reasoning;
+										judgeCriteriaScores = judgeResult.criteriaScores;
+									}
+								} catch (err) {
+									// A judge that never answered is not a verdict. Fail the
+									// sample and move on — the same way an inference timeout is
+									// handled above — and leave judgeScore unset so the report
+									// says why rather than reading as the model scoring 0.
+									samplePassed = false;
+									if (sample === 0) {
+										judgeReasoning = judgeController.signal.aborted
+											? `Judge timed out after ${timeout}ms`
+											: `Judge call failed: ${err instanceof Error ? err.message : 'Unknown error'}`;
+									}
+								} finally {
+									clearTimeout(judgeTimeoutId);
 								}
 							} else {
 								// Use string matching

src/lib/judge.spec.ts

@@ -7,11 +7,14 @@ import {
 	statSync,
 	writeFileSync,
 } from 'node:fs';
+import {createServer, type Server} from 'node:http';
+import type {AddressInfo, Socket} from 'node:net';
 import {tmpdir} from 'node:os';
 import {join} from 'node:path';
 import test from 'ava';
 import {
 	buildJudgePrompt,
+	callJudge,
 	getJudgeConfigPath,
 	JUDGE_CRITERIA,
 	parseJudgeResponse,
@@ -334,3 +337,115 @@ test.serial('saveJudgeConfig - recovers from a stale temp file', t => {
 		rmSync(dir, {recursive: true, force: true});
 	}
 });
+
+// callJudge
+
+/**
+ * Start a local OpenAI-compatible endpoint and hand back its base URL plus a
+ * teardown. `respond` is left undefined to model the failure this guards
+ * against: a server that accepts the connection and never answers, which is
+ * what a hung model server or a slow rate-limit backoff looks like from here.
+ */
+async function startJudgeEndpoint(respond?: (content: string) => string) {
+	const sockets: Socket[] = [];
+	const server: Server = createServer((_req, res) => {
+		if (!respond) return;
+		res.writeHead(200, {'content-type': 'application/json'});
+		res.end(respond(''));
+	});
+	server.on('connection', socket => sockets.push(socket));
+	await new Promise<void>(resolve => {
+		server.listen(0, '127.0.0.1', resolve);
+	});
+	const {port} = server.address() as AddressInfo;
+	return {
+		baseUrl: `http://127.0.0.1:${port}/v1`,
+		async close() {
+			// A never-answered request holds its socket open, so the server would
+			// never finish closing and the test worker would never exit.
+			for (const socket of sockets) socket.destroy();
+			await new Promise<void>(resolve => {
+				server.close(() => resolve());
+			});
+		},
+	};
+}
+
+test.serial('callJudge - an abort signal cancels a judge that never replies', async t => {
+	// Before the fix callJudge took no signal and forwarded none into
+	// generateText, so this await never settled and `benchmark --timeout` had
+	// nothing to say about it.
+	const endpoint = await startJudgeEndpoint();
+	const controller = new AbortController();
+	const timeoutId = setTimeout(() => controller.abort(), 250);
+	const startedAt = Date.now();
+	try {
+		await t.throwsAsync(
+			callJudge(
+				'What is 2+2?',
+				'4',
+				resolveCriteria(['helpful']),
+				{
+					name: 'Stalled',
+					baseUrl: endpoint.baseUrl,
+					model: 'test-model',
+				},
+				7,
+				undefined,
+				controller.signal,
+			),
+		);
+		// Not just "it threw": it threw on the budget. The AI SDK retries a
+		// failed call by default, so an unaborted attempt would still be
+		// waiting here rather than having given up.
+		t.true(
+			Date.now() - startedAt < 3000,
+			'judge call outlived the abort budget',
+		);
+	} finally {
+		clearTimeout(timeoutId);
+		await endpoint.close();
+	}
+});
+
+test.serial('callJudge - returns the judge verdict when the provider answers', async t => {
+	// The signal is optional and must stay out of the way: a judge that
+	// replies inside its budget still scores the response normally.
+	const verdict =
+		'{"scores": {"helpful": 9}, "overall": 9, "reasoning": "Correct.", "pass": true}';
+	const endpoint = await startJudgeEndpoint(() =>
+		JSON.stringify({
+			id: 'chatcmpl-test',
+			object: 'chat.completion',
+			created: 0,
+			model: 'test-model',
+			choices: [
+				{
+					index: 0,
+					message: {role: 'assistant', content: verdict},
+					finish_reason: 'stop',
+				},
+			],
+			usage: {prompt_tokens: 1, completion_tokens: 1, total_tokens: 2},
+		}),
+	);
+	const controller = new AbortController();
+	const timeoutId = setTimeout(() => controller.abort(), 5000);
+	try {
+		const result = await callJudge(
+			'What is 2+2?',
+			'4',
+			resolveCriteria(['helpful']),
+			{name: 'Local', baseUrl: endpoint.baseUrl, model: 'test-model'},
+			7,
+			undefined,
+			controller.signal,
+		);
+		t.true(result.pass);
+		t.is(result.score, 9);
+		t.is(result.criteriaScores.helpful, 9);
+	} finally {
+		clearTimeout(timeoutId);
+		await endpoint.close();
+	}
+});

src/lib/judge.ts

@@ -214,14 +214,24 @@ export function parseJudgeResponse(
 	};
 }
 
-/** Call the LLM judge to evaluate a response */
+/**
+ * Call the LLM judge to evaluate a response.
+ *
+ * `abortSignal` is forwarded into `generateText` so the caller can bound the
+ * call. Without one there is nothing stopping a provider that accepts the
+ * connection and never replies from hanging the caller forever — and the AI
+ * SDK's own retry policy sits on top of that, re-issuing a request nobody is
+ * timing. The signal covers the retries too: it cancels the in-flight fetch
+ * and interrupts the backoff delay between attempts.
+ */
 export async function callJudge(
 	prompt: string,
 	response: string,
 	criteria: JudgeCriteria[],
 	config: JudgeProviderConfig,
 	passThreshold = 7,
 	referenceAnswers?: string[],
+	abortSignal?: AbortSignal,
 ): Promise<JudgeResult> {
 	const provider = createJudgeProvider(config);
 	const model = provider(config.model);
@@ -237,6 +247,7 @@ export async function callJudge(
 	const result = await generateText({
 		model,
 		messages: [{role: 'user', content: judgePrompt}],
+		abortSignal,
 	});
 
 	try {