Fix evals (#81)

* cleanup

* get-component-documentation only accepts a single component id

* fix versions

* use vitest cli instead of node for evals

* prefix experiment scripts so they are not picked up by turborepo

Author: JReinhold

eval/lib/agents/claude-code-cli.ts

@@ -285,8 +285,7 @@ function getTodoProgress(
 	messages: ClaudeCodeStreamMessage[],
 ): TodoProgress | null {
 	// Find the most recent TodoWrite message
-	for (let i = messages.length - 1; i >= 0; i--) {
-		const message = messages[i];
+	for (const message of messages.toReversed()) {
 		if (message.type === 'assistant') {
 			const todoWrite = message.message.content.find(
 				(c): c is ToolUseContent =>

eval/lib/collect-args.ts

@@ -362,7 +362,7 @@ export async function collectArgs() {
 							const config: McpServerConfig = {
 								[mcpServerName]: {
 									type: 'stdio',
-									command,
+									command: command!,
 									args: argsParts.length > 0 ? argsParts : undefined,
 								},
 							};

eval/lib/evaluations/build.ts

@@ -8,7 +8,7 @@ export async function build({
 	projectPath,
 	resultsPath,
 }: ExperimentArgs): Promise<boolean> {
-	const result = await x('pnpm', ['build'], {
+	const result = await x('pnpm', ['eval:build'], {
 		nodeOptions: {
 			cwd: projectPath,
 		},

eval/lib/evaluations/prepare-evaluations.ts

@@ -9,14 +9,14 @@ export async function prepareEvaluations({
 }: ExperimentArgs) {
 	await addDevDependency(
 		[
-			'vitest@catalog:',
-			'@vitest/browser-playwright@catalog:',
-			'storybook@catalog:',
-			'@storybook/addon-docs@catalog:',
-			'@storybook/addon-a11y@catalog:',
-			'@storybook/addon-vitest@catalog:',
-			'@storybook/react-vite@catalog:',
-			'eslint-plugin-storybook@catalog:',
+			'vitest@catalog:experiments',
+			'@vitest/browser-playwright@catalog:experiments',
+			'storybook@catalog:experiments',
+			'@storybook/addon-docs@catalog:experiments',
+			'@storybook/addon-a11y@catalog:experiments',
+			'@storybook/addon-vitest@catalog:experiments',
+			'@storybook/react-vite@catalog:experiments',
+			'eslint-plugin-storybook@catalog:experiments',
 		],
 		{ cwd: projectPath, silent: true },
 	);
@@ -28,6 +28,20 @@ export async function prepareEvaluations({
 		filter: (source) =>
 			!source.includes('node_modules') && !source.includes('dist'),
 	});
+
+	const { default: pkgJson } = await import(
+		path.join(projectPath, 'package.json'),
+		{
+			with: { type: 'json' },
+		}
+	);
+	// add the storybook script after agent execution, so it does not taint the experiment
+	pkgJson.scripts.storybook = 'storybook dev --port 6006';
+	await fs.writeFile(
+		path.join(projectPath, 'package.json'),
+		JSON.stringify(pkgJson, null, 2),
+	);
+
 	await fs
 		.cp(
 			path.join(evalPath, 'expected', 'stories'),

eval/lib/evaluations/test-stories.ts

@@ -1,24 +1,41 @@
-import { startVitest } from 'vitest/node';
 import * as path from 'node:path';
 import * as fs from 'node:fs/promises';
 import type { EvaluationSummary, ExperimentArgs } from '../../types';
 import type { JsonTestResults } from 'vitest/reporters';
+import { x } from 'tinyexec';
+import { dedent } from 'ts-dedent';
 
 export async function testStories({
 	projectPath,
 	resultsPath,
 }: ExperimentArgs): Promise<Pick<EvaluationSummary, 'test' | 'a11y'>> {
 	const testResultsPath = path.join(resultsPath, 'tests.json');
 
-	const vitest = await startVitest('test', undefined, {
-		root: projectPath,
-		watch: false,
-		silent: true,
-		reporters: ['json'],
-		outputFile: testResultsPath,
+	const result = await x('pnpm', ['eval:test'], {
+		nodeOptions: {
+			cwd: projectPath,
+		},
 	});
 
-	await vitest.close();
+	await fs.writeFile(
+		path.join(resultsPath, 'tests.md'),
+		dedent`# Test Results
+	
+	**Exit Code:** ${result.exitCode}
+
+	## stdout
+	
+	\`\`\`sh
+	${result.stdout}
+	\`\`\`
+	
+	## stderr
+	
+	\`\`\`
+	${result.stderr}
+	\`\`\`
+	`,
+	);
 
 	const { default: jsonTestResults } = (await import(testResultsPath, {
 		with: { type: 'json' },

eval/lib/evaluations/typecheck.ts

@@ -109,8 +109,8 @@ if (import.meta.main) {
 	}
 	console.log({
 		typeErrors: await checkTypes({
-			projectPath: path.join(experimentPath[0], 'project'),
-			resultsPath: path.join(experimentPath[0], 'results'),
+			projectPath: path.join(experimentPath[0]!, 'project'),
+			resultsPath: path.join(experimentPath[0]!, 'results'),
 		} as ExperimentArgs),
 	});
 }

eval/lib/save/chromatic.ts

@@ -22,7 +22,7 @@ export async function buildStorybook(
 	let buildError = '';
 
 	try {
-		await runScript('build-storybook', {
+		await runScript('eval:build-storybook', {
 			cwd: projectPath,
 			silent: true,
 		});

eval/lib/save/google-sheet.ts

@@ -44,8 +44,9 @@ function getContextDetails(context: Context): string {
 			const mcpConfig = Object.values(context.mcpServerConfig)[0];
 			if (mcpConfig?.type === 'stdio' && mcpConfig.args) {
 				const manifestIndex = mcpConfig.args.indexOf('--manifestPath');
-				if (manifestIndex !== -1 && mcpConfig.args[manifestIndex + 1]) {
-					return path.basename(mcpConfig.args[manifestIndex + 1]);
+				const manifestIndexValue = mcpConfig.args[manifestIndex + 1];
+				if (manifestIndex !== -1 && manifestIndexValue) {
+					return path.basename(manifestIndexValue);
 				}
 			}
 			return 'unknown manifest name';

eval/package.json

@@ -5,15 +5,14 @@
 	"description": "The project for evaluating UI component development with and without Storybook MCP",
 	"type": "module",
 	"scripts": {
-		"build-storybook": "storybook build",
 		"eval": "node eval.ts",
-		"storybook": "storybook dev -p 6006"
+		"storybook": "storybook dev -p 6006",
+		"typecheck": "tsc"
 	},
 	"devDependencies": {
 		"@anthropic-ai/claude-agent-sdk": "^0.1.30",
 		"@clack/prompts": "1.0.0-alpha.6",
 		"@radix-ui/colors": "^3.0.0",
-		"chromatic": "^13.3.3",
 		"@radix-ui/react-popover": "^1.1.15",
 		"@radix-ui/react-toggle": "^1.1.10",
 		"@radix-ui/react-toggle-group": "^1.1.11",
@@ -24,9 +23,11 @@
 		"@tsconfig/node24": "^24.0.1",
 		"@types/envinfo": "^7.8.4",
 		"@types/eslint": "^9.6.1",
+		"@types/node": "^24.10.1",
 		"@types/react": "^18.3.26",
 		"@vitejs/plugin-react-swc": "^4.2.0",
 		"ai-tokenizer": "^1.0.3",
+		"chromatic": "^13.3.3",
 		"envinfo": "^7.20.0",
 		"eslint": "^9.36.0",
 		"globals": "^16.4.0",

eval/templates/evaluation/eslint.config.js

@@ -12,7 +12,7 @@ export default defineConfig([
 		extends: [
 			js.configs.recommended,
 			tseslint.configs.recommendedTypeChecked,
-			reactHooks.configs['recommended-latest'],
+			reactHooks.configs.flat['recommended-latest'],
 			reactRefresh.configs.vite,
 		],
 		languageOptions: {