feat(ai-builder): Planning mode (#25498)

Signed-off-by: Oleg Ivaniv <me@olegivaniv.com>

Author: OlegIvaniv

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/feedback.test.ts

@@ -1,4 +1,4 @@
-import { langsmithMetricKey } from '../harness/feedback';
+import { langsmithMetricKey, toLangsmithEvaluationResult } from '../harness/feedback';
 import type { Feedback } from '../harness/harness-types';
 
 describe('langsmithMetricKey()', () => {
@@ -41,20 +41,20 @@ describe('langsmithMetricKey()', () => {
 	it('should prefix metrics evaluator with evaluator name', () => {
 		const discoveryLatency: Feedback = {
 			evaluator: 'metrics',
-			metric: 'discovery_latency_ms',
-			score: 500,
+			metric: 'discovery_latency_s',
+			score: 0.5,
 			kind: 'metric',
 		};
 		const builderLatency: Feedback = {
 			evaluator: 'metrics',
-			metric: 'builder_latency_ms',
-			score: 1500,
+			metric: 'builder_latency_s',
+			score: 1.5,
 			kind: 'metric',
 		};
 		const responderLatency: Feedback = {
 			evaluator: 'metrics',
-			metric: 'responder_latency_ms',
-			score: 200,
+			metric: 'responder_latency_s',
+			score: 0.2,
 			kind: 'metric',
 		};
 		const nodeCount: Feedback = {
@@ -64,9 +64,9 @@ describe('langsmithMetricKey()', () => {
 			kind: 'metric',
 		};
 
-		expect(langsmithMetricKey(discoveryLatency)).toBe('metrics.discovery_latency_ms');
-		expect(langsmithMetricKey(builderLatency)).toBe('metrics.builder_latency_ms');
-		expect(langsmithMetricKey(responderLatency)).toBe('metrics.responder_latency_ms');
+		expect(langsmithMetricKey(discoveryLatency)).toBe('metrics.discovery_latency_s');
+		expect(langsmithMetricKey(builderLatency)).toBe('metrics.builder_latency_s');
+		expect(langsmithMetricKey(responderLatency)).toBe('metrics.responder_latency_s');
 		expect(langsmithMetricKey(nodeCount)).toBe('metrics.node_count');
 	});
 
@@ -79,11 +79,74 @@ describe('langsmithMetricKey()', () => {
 			{ evaluator: 'pairwise', metric: 'pairwise_primary', score: 1, kind: 'score' },
 			{ evaluator: 'pairwise', metric: 'pairwise_total_violations', score: 1, kind: 'detail' },
 			{ evaluator: 'pairwise', metric: 'judge1', score: 0, kind: 'detail' },
-			{ evaluator: 'metrics', metric: 'discovery_latency_ms', score: 500, kind: 'metric' },
+			{ evaluator: 'metrics', metric: 'discovery_latency_s', score: 0.5, kind: 'metric' },
 			{ evaluator: 'metrics', metric: 'node_count', score: 5, kind: 'metric' },
 		];
 
 		const keys = feedback.map(langsmithMetricKey);
 		expect(new Set(keys).size).toBe(keys.length);
 	});
 });
+
+describe('toLangsmithEvaluationResult()', () => {
+	it('should clamp scores exceeding LangSmith max limit (safety net)', () => {
+		const fb: Feedback = {
+			evaluator: 'metrics',
+			metric: 'some_metric',
+			score: 150000, // Exceeds 99999.9999
+			kind: 'metric',
+		};
+
+		const result = toLangsmithEvaluationResult(fb);
+		expect(result.score).toBe(99999.9999);
+	});
+
+	it('should clamp scores below LangSmith min limit (safety net)', () => {
+		const fb: Feedback = {
+			evaluator: 'metrics',
+			metric: 'some_metric',
+			score: -200000,
+			kind: 'metric',
+		};
+
+		const result = toLangsmithEvaluationResult(fb);
+		expect(result.score).toBe(-99999.9999);
+	});
+
+	it('should preserve scores within valid range', () => {
+		const fb: Feedback = {
+			evaluator: 'metrics',
+			metric: 'discovery_latency_s',
+			score: 161.288, // 161 seconds, well within limits
+			kind: 'metric',
+		};
+
+		const result = toLangsmithEvaluationResult(fb);
+		expect(result.score).toBe(161.288);
+	});
+
+	it('should include comment when present', () => {
+		const fb: Feedback = {
+			evaluator: 'llm-judge',
+			metric: 'overallScore',
+			score: 0.85,
+			kind: 'score',
+			comment: 'Good workflow',
+		};
+
+		const result = toLangsmithEvaluationResult(fb);
+		expect(result.comment).toBe('Good workflow');
+	});
+
+	it('should not include comment when absent', () => {
+		const fb: Feedback = {
+			evaluator: 'llm-judge',
+			metric: 'overallScore',
+			score: 0.85,
+			kind: 'score',
+		};
+
+		const result = toLangsmithEvaluationResult(fb);
+		expect(result.comment).toBeUndefined();
+	});
+});

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/evaluation-helpers.ts

@@ -86,7 +86,7 @@ export function getChatPayload(options: GetChatPayloadOptions): ChatPayload {
  * Matches the CoordinationLogEntry type from src/types/coordination.ts
  */
 interface CoordinationLogEntry {
-	phase: 'discovery' | 'builder' | 'state_management' | 'responder';
+	phase: 'discovery' | 'builder' | 'state_management' | 'responder' | 'planner';
 	status: 'completed' | 'in_progress' | 'error';
 	timestamp: number;
 }

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/feedback.ts

@@ -21,7 +21,7 @@ function isPairwiseV1Metric(metric: string): boolean {
  * - Programmatic: keep evaluator prefix (e.g. `programmatic.trigger`)
  * - LLM-judge: keep metrics unprefixed (e.g. `overallScore`, `connections`, `maintainability.nodeNamingQuality`)
  * - Pairwise: keep v1 metrics unprefixed (e.g. `pairwise_primary`), but namespace non-v1 details.
- * - Metrics: keep evaluator prefix (e.g. `metrics.discovery_latency_ms`, `metrics.node_count`)
+ * - Metrics: keep evaluator prefix (e.g. `metrics.discovery_latency_s`, `metrics.node_count`)
  */
 export function langsmithMetricKey(feedback: Feedback): string {
 	if (feedback.evaluator === 'pairwise') {
@@ -44,10 +44,24 @@ export function langsmithMetricKey(feedback: Feedback): string {
 	return feedbackKey(feedback);
 }
 
+/**
+ * LangSmith score limits.
+ */
+const LANGSMITH_SCORE_MIN = -99999.9999;
+const LANGSMITH_SCORE_MAX = 99999.9999;
+
+/**
+ * Clamp a score to LangSmith's valid range.
+ * LangSmith rejects scores outside [-99999.9999, 99999.9999].
+ */
+function clampScoreForLangsmith(score: number): number {
+	return Math.max(LANGSMITH_SCORE_MIN, Math.min(LANGSMITH_SCORE_MAX, score));
+}
+
 export function toLangsmithEvaluationResult(feedback: Feedback): LangsmithEvaluationResultLike {
 	return {
 		key: langsmithMetricKey(feedback),
-		score: feedback.score,
+		score: clampScoreForLangsmith(feedback.score),
 		...(feedback.comment ? { comment: feedback.comment } : {}),
 	};
 }

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/runner.ts

@@ -117,9 +117,21 @@ function hasErrorFeedback(feedback: Feedback[]): boolean {
 	return feedback.some((f) => f.metric === 'error');
 }
 
+/**
+ * Convert milliseconds to seconds for LangSmith metrics.
+ * LangSmith scores must be within [-99999.9999, 99999.9999], so we store
+ * latencies in seconds (supporting up to ~27 hours) instead of milliseconds.
+ */
+function msToSeconds(ms: number): number {
+	return ms / 1000;
+}
+
 /**
  * Create feedback items for subgraph metrics.
  * These are reported to LangSmith as 'metrics' evaluator feedback.
+ *
+ * Note: Latencies are converted from milliseconds to seconds to stay within
+ * LangSmith's score limits while preserving precision for long-running operations.
  */
 function createMetricsFeedback(args: {
 	discoveryDurationMs?: number;
@@ -132,26 +144,26 @@ function createMetricsFeedback(args: {
 	if (args.discoveryDurationMs !== undefined) {
 		feedback.push({
 			evaluator: 'metrics',
-			metric: 'discovery_latency_ms',
-			score: args.discoveryDurationMs,
+			metric: 'discovery_latency_s',
+			score: msToSeconds(args.discoveryDurationMs),
 			kind: 'metric',
 		});
 	}
 
 	if (args.builderDurationMs !== undefined) {
 		feedback.push({
 			evaluator: 'metrics',
-			metric: 'builder_latency_ms',
-			score: args.builderDurationMs,
+			metric: 'builder_latency_s',
+			score: msToSeconds(args.builderDurationMs),
 			kind: 'metric',
 		});
 	}
 
 	if (args.responderDurationMs !== undefined) {
 		feedback.push({
 			evaluator: 'metrics',
-			metric: 'responder_latency_ms',
-			score: args.responderDurationMs,
+			metric: 'responder_latency_s',
+			score: msToSeconds(args.responderDurationMs),
 			kind: 'metric',
 		});
 	}

packages/@n8n/ai-workflow-builder.ee/evaluations/support/environment.ts

@@ -40,6 +40,8 @@ export interface StageModels {
 	builder?: ModelId;
 	/** Model for parameter updater (within builder) */
 	parameterUpdater?: ModelId;
+	/** Model for planner stage (plan mode) */
+	planner?: ModelId;
 	/** Model for LLM judge evaluation */
 	judge?: ModelId;
 }
@@ -55,6 +57,7 @@ export interface ResolvedStageLLMs {
 	discovery: BaseChatModel;
 	builder: BaseChatModel;
 	parameterUpdater: BaseChatModel;
+	planner: BaseChatModel;
 	judge: BaseChatModel;
 }
 
@@ -106,6 +109,7 @@ export async function resolveStageModels(stageModels: StageModels): Promise<Reso
 		parameterUpdater: stageModels.parameterUpdater
 			? await setupLLM(stageModels.parameterUpdater)
 			: builderLLM,
+		planner: stageModels.planner ? await setupLLM(stageModels.planner) : defaultLLM,
 		judge: stageModels.judge ? await setupLLM(stageModels.judge) : defaultLLM,
 	};
 }
@@ -221,6 +225,7 @@ export function createAgent(options: CreateAgentOptions): WorkflowBuilderAgent {
 			discovery: llms.discovery,
 			builder: llms.builder,
 			parameterUpdater: llms.parameterUpdater,
+			planner: llms.planner,
 		},
 		checkpointer: new MemorySaver(),
 		tracer,

packages/@n8n/ai-workflow-builder.ee/package.json

@@ -56,6 +56,7 @@
     "@langchain/core": "catalog:",
     "@langchain/langgraph": "1.0.2",
     "@langchain/openai": "catalog:",
+    "langchain": "catalog:",
     "@n8n/backend-common": "workspace:*",
     "@n8n/config": "workspace:*",
     "@n8n/di": "workspace:*",