diff --git a/eval/eval.ts b/eval/eval.ts index b34db2f5..8721deb8 100644 --- a/eval/eval.ts +++ b/eval/eval.ts @@ -2,6 +2,7 @@ import * as p from '@clack/prompts'; import { claudeCodeCli } from './lib/agents/claude-code-cli.ts'; import * as path from 'node:path'; import * as fs from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; import type { ExperimentArgs } from './types.ts'; import { prepareExperiment } from './lib/prepare-experiment.ts'; import { evaluate } from './lib/evaluations/evaluate.ts'; @@ -75,7 +76,7 @@ const experimentArgs: ExperimentArgs = { evalName: args.eval, context: args.context, agent: args.agent, - hooks: await import(path.join(evalPath, 'hooks.ts')) + hooks: await import(pathToFileURL(path.join(evalPath, 'hooks.ts')).href) .then((mod) => mod.default) .catch(() => ({})), }; diff --git a/eval/lib/agents/claude-code-cli.ts b/eval/lib/agents/claude-code-cli.ts index 832ca140..a7b3191a 100644 --- a/eval/lib/agents/claude-code-cli.ts +++ b/eval/lib/agents/claude-code-cli.ts @@ -354,7 +354,6 @@ export const claudeCodeCli: Agent = { '--dangerously-skip-permissions', '--output-format=stream-json', '--verbose', - prompt, ]; if (mcpServerConfig) { @@ -364,12 +363,15 @@ export const claudeCodeCli: Agent = { const claudeProcess = x('claude', args, { nodeOptions: { cwd: projectPath, - stdio: ['pipe', 'pipe', 'pipe'], // pipe stdin to send "yes" for MCP prompts + stdio: ['pipe', 'pipe', 'pipe'], // pipe stdin to send prompt and "yes" for MCP prompts }, }); // Auto-approve MCP server trust prompt by sending "1" (Yes, proceed) + // Then send the prompt through stdin if (claudeProcess.process?.stdin) { claudeProcess.process?.stdin.write('1\n'); + claudeProcess.process?.stdin.write(prompt); + claudeProcess.process?.stdin.write('\n'); claudeProcess.process?.stdin.end(); } const messages: ClaudeCodeStreamMessage[] = []; diff --git a/eval/lib/collect-args.ts b/eval/lib/collect-args.ts index 188bcbbd..4868b241 100644 --- a/eval/lib/collect-args.ts +++ b/eval/lib/collect-args.ts @@ -3,6 +3,7 @@ import * as v from 'valibot'; import * as p from '@clack/prompts'; import * as path from 'node:path'; import * as fs from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; import { McpServerConfigSchema, type Context, @@ -65,9 +66,13 @@ export async function collectArgs() { ); } return ( - await import(path.join(EVALS_DIR, parsedEvalPath, filePath), { - with: { type: 'json' }, - }) + await import( + pathToFileURL(path.join(EVALS_DIR, parsedEvalPath, filePath)) + .href, + { + with: { type: 'json' }, + } + ) ).default; }), McpServerConfigSchema, @@ -209,7 +214,7 @@ export async function collectArgs() { } if (dirent.name.endsWith('.json') && !dirent.name.includes('mcp')) { const { default: manifestContent } = await import( - path.join(evalPath, dirent.name), + pathToFileURL(path.join(evalPath, dirent.name)).href, { with: { type: 'json' }, } diff --git a/eval/lib/evaluations/prepare-evaluations.ts b/eval/lib/evaluations/prepare-evaluations.ts index 55218542..e2cdf36b 100644 --- a/eval/lib/evaluations/prepare-evaluations.ts +++ b/eval/lib/evaluations/prepare-evaluations.ts @@ -1,5 +1,6 @@ import * as path from 'node:path'; import * as fs from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; import { addDevDependency } from 'nypm'; import type { ExperimentArgs } from '../../types.ts'; @@ -27,7 +28,7 @@ export async function prepareEvaluations({ projectPath }: ExperimentArgs) { }); const { default: pkgJson } = await import( - path.join(projectPath, 'package.json'), + pathToFileURL(path.join(projectPath, 'package.json')).href, { with: { type: 'json' }, } diff --git a/eval/lib/evaluations/test-stories.ts b/eval/lib/evaluations/test-stories.ts index c128fd2f..3273bbb0 100644 --- a/eval/lib/evaluations/test-stories.ts +++ b/eval/lib/evaluations/test-stories.ts @@ -1,5 +1,6 @@ import * as path from 'node:path'; import * as fs from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; import type { EvaluationSummary, ExperimentArgs } from '../../types'; import type { JsonTestResults } from 'vitest/reporters'; import { x } from 'tinyexec'; @@ -37,9 +38,12 @@ export async function testStories({ `, ); - const { default: jsonTestResults } = (await import(testResultsPath, { - with: { type: 'json' }, - })) as { default: JsonTestResults }; + const { default: jsonTestResults } = (await import( + pathToFileURL(testResultsPath).href, + { + with: { type: 'json' }, + } + )) as { default: JsonTestResults }; // write the file again to pretty-print it await fs.writeFile(testResultsPath, JSON.stringify(jsonTestResults, null, 2)); diff --git a/eval/lib/prepare-experiment.ts b/eval/lib/prepare-experiment.ts index 838f61d4..39f66f37 100644 --- a/eval/lib/prepare-experiment.ts +++ b/eval/lib/prepare-experiment.ts @@ -1,6 +1,7 @@ import type { ExperimentArgs } from '../types.ts'; import * as path from 'node:path'; import * as fs from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; import { installDependencies } from 'nypm'; import { taskLog } from '@clack/prompts'; import { runHook } from './run-hook.ts'; @@ -26,9 +27,12 @@ export async function prepareExperiment(experimentArgs: ExperimentArgs) { }); const packageJsonPath = path.join(experimentArgs.projectPath, 'package.json'); - const { default: packageJson } = await import(packageJsonPath, { - with: { type: 'json' }, - }); + const { default: packageJson } = await import( + pathToFileURL(packageJsonPath).href, + { + with: { type: 'json' }, + } + ); packageJson.name = `@storybook/mcp-eval--${experimentArgs.evalName}--${path.basename(experimentArgs.experimentPath)}`.toLowerCase(); diff --git a/eval/lib/save/chromatic.ts b/eval/lib/save/chromatic.ts index 34261058..2312f7b3 100644 --- a/eval/lib/save/chromatic.ts +++ b/eval/lib/save/chromatic.ts @@ -2,6 +2,7 @@ import type { ExperimentArgs } from '../../types.ts'; import { runScript } from 'nypm'; import * as fs from 'node:fs/promises'; import * as path from 'node:path'; +import { pathToFileURL } from 'node:url'; import { dedent } from 'ts-dedent'; import { x } from 'tinyexec'; @@ -94,9 +95,12 @@ export async function uploadToChromatic( experimentArgs.projectPath, 'chromatic-diagnostics.json', ); - const { default: diagnostics } = (await import(diagnosticsPath, { - with: { type: 'json' }, - })) as { default: ChromaticDiagnostics }; + const { default: diagnostics } = (await import( + pathToFileURL(diagnosticsPath).href, + { + with: { type: 'json' }, + } + )) as { default: ChromaticDiagnostics }; return diagnostics.storybookUrl; }