use vitest cli instead of node for evals

Author: JReinhold

eval/lib/agents/claude-code-cli.ts

@@ -285,8 +285,7 @@ function getTodoProgress(
 	messages: ClaudeCodeStreamMessage[],
 ): TodoProgress | null {
 	// Find the most recent TodoWrite message
-	for (let i = messages.length - 1; i >= 0; i--) {
-		const message = messages[i];
+	for (const message of messages.toReversed()) {
 		if (message.type === 'assistant') {
 			const todoWrite = message.message.content.find(
 				(c): c is ToolUseContent =>

eval/lib/collect-args.ts

@@ -362,7 +362,7 @@ export async function collectArgs() {
 							const config: McpServerConfig = {
 								[mcpServerName]: {
 									type: 'stdio',
-									command,
+									command: command!,
 									args: argsParts.length > 0 ? argsParts : undefined,
 								},
 							};

eval/lib/evaluations/prepare-evaluations.ts

@@ -28,6 +28,20 @@ export async function prepareEvaluations({
 		filter: (source) =>
 			!source.includes('node_modules') && !source.includes('dist'),
 	});
+
+	const { default: pkgJson } = await import(
+		path.join(projectPath, 'package.json'),
+		{
+			with: { type: 'json' },
+		}
+	);
+	// add the storybook script after agent execution, so it does not taint the experiment
+	pkgJson.scripts.storybook = 'storybook dev --port 6006';
+	await fs.writeFile(
+		path.join(projectPath, 'package.json'),
+		JSON.stringify(pkgJson, null, 2),
+	);
+
 	await fs
 		.cp(
 			path.join(evalPath, 'expected', 'stories'),

eval/lib/evaluations/test-stories.ts

@@ -1,24 +1,41 @@
-import { startVitest } from 'vitest/node';
 import * as path from 'node:path';
 import * as fs from 'node:fs/promises';
 import type { EvaluationSummary, ExperimentArgs } from '../../types';
 import type { JsonTestResults } from 'vitest/reporters';
+import { x } from 'tinyexec';
+import { dedent } from 'ts-dedent';
 
 export async function testStories({
 	projectPath,
 	resultsPath,
 }: ExperimentArgs): Promise<Pick<EvaluationSummary, 'test' | 'a11y'>> {
 	const testResultsPath = path.join(resultsPath, 'tests.json');
 
-	const vitest = await startVitest('test', undefined, {
-		root: projectPath,
-		watch: false,
-		silent: true,
-		reporters: ['json'],
-		outputFile: testResultsPath,
+	const result = await x('pnpm', ['test'], {
+		nodeOptions: {
+			cwd: projectPath,
+		},
 	});
 
-	await vitest.close();
+	await fs.writeFile(
+		path.join(resultsPath, 'tests.md'),
+		dedent`# Test Results
+	
+	**Exit Code:** ${result.exitCode}
+
+	## stdout
+	
+	\`\`\`sh
+	${result.stdout}
+	\`\`\`
+	
+	## stderr
+	
+	\`\`\`
+	${result.stderr}
+	\`\`\`
+	`,
+	);
 
 	const { default: jsonTestResults } = (await import(testResultsPath, {
 		with: { type: 'json' },

eval/lib/evaluations/typecheck.ts

@@ -109,8 +109,8 @@ if (import.meta.main) {
 	}
 	console.log({
 		typeErrors: await checkTypes({
-			projectPath: path.join(experimentPath[0], 'project'),
-			resultsPath: path.join(experimentPath[0], 'results'),
+			projectPath: path.join(experimentPath[0]!, 'project'),
+			resultsPath: path.join(experimentPath[0]!, 'results'),
 		} as ExperimentArgs),
 	});
 }

eval/lib/save/google-sheet.ts

@@ -44,8 +44,9 @@ function getContextDetails(context: Context): string {
 			const mcpConfig = Object.values(context.mcpServerConfig)[0];
 			if (mcpConfig?.type === 'stdio' && mcpConfig.args) {
 				const manifestIndex = mcpConfig.args.indexOf('--manifestPath');
-				if (manifestIndex !== -1 && mcpConfig.args[manifestIndex + 1]) {
-					return path.basename(mcpConfig.args[manifestIndex + 1]);
+				const manifestIndexValue = mcpConfig.args[manifestIndex + 1];
+				if (manifestIndex !== -1 && manifestIndexValue) {
+					return path.basename(manifestIndexValue);
 				}
 			}
 			return 'unknown manifest name';

eval/package.json

@@ -7,13 +7,13 @@
 	"scripts": {
 		"build-storybook": "storybook build",
 		"eval": "node eval.ts",
-		"storybook": "storybook dev -p 6006"
+		"storybook": "storybook dev -p 6006",
+		"typecheck": "tsc"
 	},
 	"devDependencies": {
 		"@anthropic-ai/claude-agent-sdk": "^0.1.30",
 		"@clack/prompts": "1.0.0-alpha.6",
 		"@radix-ui/colors": "^3.0.0",
-		"chromatic": "^13.3.3",
 		"@radix-ui/react-popover": "^1.1.15",
 		"@radix-ui/react-toggle": "^1.1.10",
 		"@radix-ui/react-toggle-group": "^1.1.11",
@@ -24,9 +24,11 @@
 		"@tsconfig/node24": "^24.0.1",
 		"@types/envinfo": "^7.8.4",
 		"@types/eslint": "^9.6.1",
+		"@types/node": "^24.10.1",
 		"@types/react": "^18.3.26",
 		"@vitejs/plugin-react-swc": "^4.2.0",
 		"ai-tokenizer": "^1.0.3",
+		"chromatic": "^13.3.3",
 		"envinfo": "^7.20.0",
 		"eslint": "^9.36.0",
 		"globals": "^16.4.0",

eval/templates/evaluation/eslint.config.js

@@ -12,7 +12,7 @@ export default defineConfig([
 		extends: [
 			js.configs.recommended,
 			tseslint.configs.recommendedTypeChecked,
-			reactHooks.configs['recommended-latest'],
+			reactHooks.configs.flat['recommended-latest'],
 			reactRefresh.configs.vite,
 		],
 		languageOptions: {

eval/templates/project/package.json

@@ -9,8 +9,7 @@
 		"dev": "vite",
 		"lint": "eslint .",
 		"preview": "vite preview",
-		"test": "vitest",
-		"storybook": "storybook dev --port 6006",
+		"test": "vitest run --reporter json --outputFile ../results/tests.json",
 		"typecheck": "tsc --noEmit --project ./tsconfig.app.json"
 	},
 	"dependencies": {

eval/templates/result-docs/conversation.tsx

@@ -172,8 +172,8 @@ const CodeBlock = ({
 	const codeRef = useRef<HTMLElement>(null);
 
 	useEffect(() => {
-		if (codeRef.current && (window as any).hljs) {
-			(window as any).hljs.highlightElement(codeRef.current);
+		if (codeRef.current && (globalThis as any).hljs) {
+			(globalThis as any).hljs.highlightElement(codeRef.current);
 		}
 	}, [content, isTruncated]);
 
@@ -410,6 +410,15 @@ const ElapsedTime = ({
 	</div>
 );
 
+const TYPE_COLORS = {
+	assistant: { bg: '#dbeafe', text: '#1e40af' },
+	user: { bg: '#f3e8ff', text: '#6b21a8' },
+	system: { bg: '#e0e7ff', text: '#3730a3' },
+	result: { bg: '#dcfce7', text: '#166534' },
+	tool: { bg: '#fef3c7', text: '#92400e' },
+	prompt: { bg: '#fce7f3', text: '#9f1239' },
+} as const;
+
 const Turn = ({
 	children,
 	type,
@@ -420,7 +429,7 @@ const Turn = ({
 	isMCP = false,
 }: {
 	children: React.ReactNode;
-	type: string;
+	type: keyof typeof TYPE_COLORS;
 	title: string;
 	subtitle?: string;
 	tokenCount?: string;
@@ -429,16 +438,7 @@ const Turn = ({
 }) => {
 	const [isExpanded, setIsExpanded] = useState(false);
 
-	const typeColors: Record<string, { bg: string; text: string }> = {
-		assistant: { bg: '#dbeafe', text: '#1e40af' },
-		user: { bg: '#f3e8ff', text: '#6b21a8' },
-		system: { bg: '#e0e7ff', text: '#3730a3' },
-		result: { bg: '#dcfce7', text: '#166534' },
-		tool: { bg: '#fef3c7', text: '#92400e' },
-		prompt: { bg: '#fce7f3', text: '#9f1239' },
-	};
-
-	const colors = typeColors[type] || typeColors.assistant;
+	const colors = TYPE_COLORS[type] ?? TYPE_COLORS.assistant;
 
 	return (
 		<div
@@ -771,10 +771,9 @@ function groupToolCallsWithResults(turns: ConversationMessage[]): Array<{
 	}> = [];
 	const usedResultIndices = new Set<number>();
 
-	for (let i = 0; i < turns.length; i++) {
+	for (const [i, turn] of turns.entries()) {
 		if (usedResultIndices.has(i)) continue;
 
-		const turn = turns[i];
 		const toolUseContent =
 			turn.type === 'assistant' &&
 			'message' in turn &&