feat(core): Add introspection diagnostic tool for AI workflow builder (#25172)

Author: burivuhster

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/cli.test.ts

@@ -64,15 +64,31 @@ jest.mock('../cli/webhook', () => ({
 jest.mock('../harness/evaluation-helpers', () => ({
 	consumeGenerator: (...args: unknown[]): unknown => mockConsumeGenerator(...args),
 	getChatPayload: (...args: unknown[]): unknown => mockGetChatPayload(...args),
+	createWorkflowGenerator: () =>
+		jest.fn().mockResolvedValue({ name: 'Test', nodes: [], connections: {} }),
+}));
+
+jest.mock('../lifecycles/introspection-analysis', () => ({
+	createIntrospectionAnalysisLifecycle: () => ({}),
 }));
 
 jest.mock('../index', () => ({
 	runEvaluation: (...args: unknown[]): unknown => mockRunEvaluation(...args),
 	createConsoleLifecycle: (...args: unknown[]): unknown => mockCreateConsoleLifecycle(...args),
+	mergeLifecycles: (...lifecycles: unknown[]): unknown => {
+		// Simple merge implementation for tests - just return the first non-empty lifecycle
+		const valid = (lifecycles as Array<Record<string, unknown> | undefined>).filter(
+			(lc) => lc !== undefined,
+		);
+		if (valid.length === 0) return {};
+		// Return a merged object
+		return Object.assign({}, ...valid);
+	},
 	createLLMJudgeEvaluator: (...args: unknown[]): unknown => mockCreateLLMJudgeEvaluator(...args),
 	createProgrammaticEvaluator: (...args: unknown[]): unknown =>
 		mockCreateProgrammaticEvaluator(...args),
 	createPairwiseEvaluator: (...args: unknown[]): unknown => mockCreatePairwiseEvaluator(...args),
+	createSimilarityEvaluator: () => ({ name: 'similarity', evaluate: jest.fn() }),
 }));
 
 /** Helper to create a minimal valid workflow for tests */

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/lifecycle.test.ts

@@ -128,7 +128,7 @@ describe('Console Lifecycle', () => {
 			lifecycle.onStart(config);
 			mockConsole.log.mockClear();
 
-			lifecycle.onEnd({
+			await lifecycle.onEnd({
 				totalExamples: 0,
 				passed: 0,
 				failed: 0,
@@ -538,7 +538,7 @@ describe('Console Lifecycle', () => {
 				totalDurationMs: 30000,
 			};
 
-			lifecycle.onEnd(summary);
+			await lifecycle.onEnd(summary);
 
 			expect(mockConsole.log).toHaveBeenCalled();
 			const logOutput = mockConsole.log.mock.calls.flat().join(' ');
@@ -601,7 +601,7 @@ describe('Console Lifecycle', () => {
 
 			lifecycle.onStart(config);
 			lifecycle.onExampleStart(1, 1, 'Test');
-			lifecycle.onEnd({
+			await lifecycle.onEnd({
 				totalExamples: 1,
 				passed: 1,
 				failed: 0,
@@ -809,10 +809,44 @@ describe('Console Lifecycle', () => {
 				averageScore: 0.85,
 				totalDurationMs: 5000,
 			};
-			merged.onEnd(summary);
+			await merged.onEnd(summary);
 
 			expect(hook1).toHaveBeenCalledWith(summary);
 			expect(hook2).toHaveBeenCalledWith(summary);
 		});
+
+		it('should properly await async onEnd hooks in mergeLifecycles', async () => {
+			const { mergeLifecycles } = await import('../harness/lifecycle');
+
+			const callOrder: string[] = [];
+
+			const asyncHook = jest.fn(async () => {
+				await new Promise((resolve) => setTimeout(resolve, 50));
+				callOrder.push('async');
+			});
+			const syncHook = jest.fn(() => {
+				callOrder.push('sync');
+			});
+
+			const lifecycle1: Partial<EvaluationLifecycle> = { onEnd: asyncHook };
+			const lifecycle2: Partial<EvaluationLifecycle> = { onEnd: syncHook };
+
+			const merged = mergeLifecycles(lifecycle1, lifecycle2);
+			const summary: RunSummary = {
+				totalExamples: 1,
+				passed: 1,
+				failed: 0,
+				errors: 0,
+				averageScore: 1,
+				totalDurationMs: 100,
+			};
+
+			await merged.onEnd(summary);
+
+			expect(asyncHook).toHaveBeenCalledWith(summary);
+			expect(syncHook).toHaveBeenCalledWith(summary);
+			// Async hook should complete before sync hook starts (sequential await)
+			expect(callOrder).toEqual(['async', 'sync']);
+		});
 	});
 });

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/runner-langsmith.test.ts

@@ -33,7 +33,6 @@ jest.mock('langsmith/traceable', () => ({
 	),
 }));
 
-// Mock core/environment module (dynamically imported in runner.ts)
 function createMockWorkflow(name = 'Test Workflow'): SimpleWorkflow {
 	return { name, nodes: [], connections: {} };
 }

packages/@n8n/ai-workflow-builder.ee/evaluations/cli/argument-parser.ts

@@ -8,7 +8,12 @@ import type { LangsmithExampleFilters } from '../harness/harness-types';
 import { DEFAULTS } from '../support/constants';
 import type { StageModels } from '../support/environment';
 
-export type EvaluationSuite = 'llm-judge' | 'pairwise' | 'programmatic' | 'similarity';
+export type EvaluationSuite =
+	| 'llm-judge'
+	| 'pairwise'
+	| 'programmatic'
+	| 'similarity'
+	| 'introspection';
 export type EvaluationBackend = 'local' | 'langsmith';
 export type AgentType = 'multi-agent' | 'code-builder';
 
@@ -79,7 +84,9 @@ const modelIdSchema = z.enum(AVAILABLE_MODELS as [ModelId, ...ModelId[]]);
 
 const cliSchema = z
 	.object({
-		suite: z.enum(['llm-judge', 'pairwise', 'programmatic', 'similarity']).default('llm-judge'),
+		suite: z
+			.enum(['llm-judge', 'pairwise', 'programmatic', 'similarity', 'introspection'])
+			.default('llm-judge'),
 		backend: z.enum(['local', 'langsmith']).default('local'),
 		agent: z.enum(['code-builder', 'multi-agent']).default('code-builder'),
 
@@ -152,7 +159,7 @@ const FLAG_DEFS: Record<string, FlagDef> = {
 		key: 'suite',
 		kind: 'string',
 		group: 'eval',
-		desc: 'Evaluation suite (llm-judge|pairwise|programmatic|similarity)',
+		desc: 'Evaluation suite (llm-judge|pairwise|programmatic|similarity|introspection)',
 	},
 	'--backend': { key: 'backend', kind: 'string', group: 'eval', desc: 'Backend (local|langsmith)' },
 	'--agent': {
@@ -446,14 +453,19 @@ function parseCli(argv: string[]): {
 
 function parseFeatureFlags(args: {
 	templateExamples: boolean;
+	suite: EvaluationSuite;
 }): BuilderFeatureFlags | undefined {
 	const templateExamplesFromEnv = process.env.EVAL_FEATURE_TEMPLATE_EXAMPLES === 'true';
 	const templateExamples = templateExamplesFromEnv || args.templateExamples;
 
-	if (!templateExamples) return undefined;
+	// Auto-enable introspection for introspection suite
+	const enableIntrospection = args.suite === 'introspection';
+
+	if (!templateExamples && !enableIntrospection) return undefined;
 
 	return {
 		templateExamples: templateExamples || undefined,
+		enableIntrospection: enableIntrospection || undefined,
 	};
 }
 
@@ -521,6 +533,7 @@ export function parseEvaluationArgs(argv: string[] = process.argv.slice(2)): Eva
 
 	const featureFlags = parseFeatureFlags({
 		templateExamples: parsed.templateExamples,
+		suite: parsed.suite,
 	});
 
 	const filters = parseFilters({

packages/@n8n/ai-workflow-builder.ee/evaluations/cli/index.ts

@@ -45,11 +45,12 @@ import {
 	getChatPayload,
 } from '../harness/evaluation-helpers';
 import { createLogger } from '../harness/logger';
-import type { GenerationCollectors } from '../harness/runner';
+import type { GenerationCollectors, SubgraphMetricsCollector } from '../harness/runner';
 import { TokenUsageTrackingHandler } from '../harness/token-tracking-handler';
 import {
 	runEvaluation,
 	createConsoleLifecycle,
+	mergeLifecycles,
 	createLLMJudgeEvaluator,
 	createProgrammaticEvaluator,
 	createPairwiseEvaluator,
@@ -61,6 +62,7 @@ import {
 	type GenerationResult,
 } from '../index';
 import { generateRunId, isWorkflowStateValues } from '../langsmith/types';
+import { createIntrospectionAnalysisLifecycle } from '../lifecycles/introspection-analysis';
 import { AGENT_TYPES, EVAL_TYPES, EVAL_USERS } from '../support/constants';
 import { setupTestEnvironment, createAgent, type ResolvedStageLLMs } from '../support/environment';
 
@@ -82,6 +84,28 @@ function hasCoordinationLog(
 	return Array.isArray(obj.coordinationLog);
 }
 
+/**
+ * Report subgraph metrics from coordination log and workflow.
+ */
+function reportSubgraphMetrics(
+	collector: SubgraphMetricsCollector,
+	stateValues: unknown,
+	workflow: SimpleWorkflow,
+): void {
+	const coordinationLog = hasCoordinationLog(stateValues) ? stateValues.coordinationLog : undefined;
+	const nodeCount = workflow.nodes?.length;
+	const metrics = extractSubgraphMetrics(coordinationLog, nodeCount);
+
+	if (
+		metrics.discoveryDurationMs !== undefined ||
+		metrics.builderDurationMs !== undefined ||
+		metrics.responderDurationMs !== undefined ||
+		metrics.nodeCount !== undefined
+	) {
+		collector(metrics);
+	}
+}
+
 /**
  * Create a workflow generator function for the multi-agent system.
  * LangSmith tracing is handled via traceable() in the runner.
@@ -93,6 +117,7 @@ function createWorkflowGenerator(
 	llms: ResolvedStageLLMs,
 	featureFlags?: BuilderFeatureFlags,
 ): (prompt: string, collectors?: GenerationCollectors) => Promise<SimpleWorkflow> {
+
 	return async (prompt: string, collectors?: GenerationCollectors): Promise<SimpleWorkflow> => {
 		const runId = generateRunId();
 
@@ -138,26 +163,48 @@ function createWorkflowGenerator(
 
 		// Extract and report subgraph metrics from coordination log
 		if (collectors?.subgraphMetrics) {
-			const coordinationLog = hasCoordinationLog(state.values)
-				? state.values.coordinationLog
-				: undefined;
-			const nodeCount = workflow.nodes?.length;
-			const metrics = extractSubgraphMetrics(coordinationLog, nodeCount);
-
-			if (
-				metrics.discoveryDurationMs !== undefined ||
-				metrics.builderDurationMs !== undefined ||
-				metrics.responderDurationMs !== undefined ||
-				metrics.nodeCount !== undefined
-			) {
-				collectors.subgraphMetrics(metrics);
-			}
+			reportSubgraphMetrics(collectors.subgraphMetrics, state.values, workflow);
 		}
 
+		// Report introspection events
+		collectors?.introspectionEvents?.(state.values.introspectionEvents ?? []);
+
 		return workflow;
 	};
 }
 
+/**
+ * Create evaluators based on suite type.
+ */
+function createEvaluators(params: {
+	suite: string;
+	judgeLlm: ResolvedStageLLMs['judge'];
+	parsedNodeTypes: Parameters<typeof createProgrammaticEvaluator>[0];
+	numJudges: number;
+}): Array<Evaluator<EvaluationContext>> {
+	const { suite, judgeLlm, parsedNodeTypes, numJudges } = params;
+	const evaluators: Array<Evaluator<EvaluationContext>> = [];
+
+	switch (suite) {
+		case 'llm-judge':
+			evaluators.push(createLLMJudgeEvaluator(judgeLlm, parsedNodeTypes));
+			evaluators.push(createProgrammaticEvaluator(parsedNodeTypes));
+			break;
+		case 'pairwise':
+			evaluators.push(createPairwiseEvaluator(judgeLlm, { numJudges }));
+			evaluators.push(createProgrammaticEvaluator(parsedNodeTypes));
+			break;
+		case 'programmatic':
+			evaluators.push(createProgrammaticEvaluator(parsedNodeTypes));
+			break;
+		case 'similarity':
+			evaluators.push(createSimilarityEvaluator());
+			break;
+	}
+
+	return evaluators;
+}
+
 /**
  * Create a CodeWorkflowBuilder generator function.
  * Uses the CodeWorkflowBuilder which coordinates planning and coding agents to generate
@@ -314,7 +361,6 @@ export async function runV2Evaluation(): Promise<void> {
 
 	// Setup environment with per-stage model configuration
 	const logger = createLogger(args.verbose);
-	const lifecycle = createConsoleLifecycle({ verbose: args.verbose, logger });
 	const stageModels = argsToStageModels(args);
 	const env = await setupTestEnvironment(stageModels, logger);
 
@@ -324,7 +370,6 @@ export async function runV2Evaluation(): Promise<void> {
 	}
 
 	// Create workflow generator based on agent type
-	// CODE_BUILDER uses CodeWorkflowBuilder
 	const generateWorkflow =
 		args.agent === AGENT_TYPES.CODE_BUILDER
 			? createCodeWorkflowBuilderGenerator(
@@ -335,43 +380,39 @@ export async function runV2Evaluation(): Promise<void> {
 				)
 			: createWorkflowGenerator(env.parsedNodeTypes, env.llms, args.featureFlags);
 
-	// Create evaluators based on suite (using judge LLM for evaluation)
-	// The --suite flag is always respected, regardless of agent type
-	const evaluators: Array<Evaluator<EvaluationContext>> = [];
-
-	switch (args.suite) {
-		case 'llm-judge':
-			evaluators.push(createLLMJudgeEvaluator(env.llms.judge, env.parsedNodeTypes));
-			evaluators.push(createProgrammaticEvaluator(env.parsedNodeTypes));
-			break;
-		case 'pairwise':
-			evaluators.push(
-				createPairwiseEvaluator(env.llms.judge, {
-					numJudges: args.numJudges,
-				}),
-			);
-			evaluators.push(createProgrammaticEvaluator(env.parsedNodeTypes));
-			break;
-		case 'programmatic':
-			evaluators.push(createProgrammaticEvaluator(env.parsedNodeTypes));
-			break;
-		case 'similarity':
-			evaluators.push(createSimilarityEvaluator());
-			break;
-	}
+	// Create evaluators based on suite type
+	const evaluators = createEvaluators({
+		suite: args.suite,
+		judgeLlm: env.llms.judge,
+		parsedNodeTypes: env.parsedNodeTypes,
+		numJudges: args.numJudges,
+	});
 
 	const llmCallLimiter = pLimit(args.concurrency);
 
+	// Merge console lifecycle with optional introspection analysis lifecycle
+	const mergedLifecycle = mergeLifecycles(
+		createConsoleLifecycle({ verbose: args.verbose, logger }),
+		args.suite === 'introspection'
+			? createIntrospectionAnalysisLifecycle({
+					judgeLlm: env.llms.judge,
+					outputDir: args.outputDir,
+					logger,
+				})
+			: undefined,
+	);
+
 	const baseConfig = {
 		generateWorkflow,
 		evaluators,
-		lifecycle,
+		lifecycle: mergedLifecycle,
 		logger,
 		outputDir: args.outputDir,
 		outputCsv: args.outputCsv,
 		suite: args.suite,
 		timeoutMs: args.timeoutMs,
 		context: { llmCallLimiter },
+		passThreshold: args.suite === 'introspection' ? 0 : undefined,
 	};
 
 	const config: RunConfig =
@@ -400,6 +441,7 @@ export async function runV2Evaluation(): Promise<void> {
 					...baseConfig,
 					mode: 'local',
 					dataset: loadTestCases(args),
+					concurrency: args.concurrency,
 				};
 
 	// Run evaluation