diff --git a/x-pack/platform/plugins/shared/agent_builder_workflows/server/tools/generate_workflow.test.ts b/x-pack/platform/plugins/shared/agent_builder_workflows/server/tools/generate_workflow.test.ts index 6d79f8bc20ab9..7f41e5da42780 100644 --- a/x-pack/platform/plugins/shared/agent_builder_workflows/server/tools/generate_workflow.test.ts +++ b/x-pack/platform/plugins/shared/agent_builder_workflows/server/tools/generate_workflow.test.ts @@ -21,8 +21,10 @@ jest.mock('@kbn/agent-builder-workflow-gen', () => ({ const generateWorkflowMock = generateWorkflow as jest.MockedFunction; describe('generateWorkflowTool', () => { + const validateWorkflowMock = jest.fn().mockResolvedValue({ valid: true, diagnostics: [] }); + const workflowsManagement = { - management: { __mock: 'workflowsApi' }, + management: { validateWorkflow: validateWorkflowMock }, } as any; const aiTelemetryClient = { @@ -73,6 +75,8 @@ describe('generateWorkflowTool', () => { beforeEach(() => { generateWorkflowMock.mockReset(); aiTelemetryClient.reportEditResult.mockReset(); + validateWorkflowMock.mockReset(); + validateWorkflowMock.mockResolvedValue({ valid: true, diagnostics: [] }); }); it('creates a new workflow: adds diff attachment, adds workflow attachment, sends UI event, reports telemetry', async () => { @@ -245,6 +249,76 @@ describe('generateWorkflowTool', () => { ); }); + it('passes a compact validation result to telemetry when the generated YAML is valid', async () => { + generateWorkflowMock.mockResolvedValueOnce({ + workflow: generatedWorkflow, + response: 'created', + } as any); + validateWorkflowMock.mockResolvedValueOnce({ valid: true, diagnostics: [] }); + + const context = buildContext(); + const tool = generateWorkflowTool({ workflowsManagement, aiTelemetryClient }); + await tool.handler({ query: 'q' } as any, context); + + expect(validateWorkflowMock).toHaveBeenCalledWith( + expect.stringContaining('name: foo'), + 'default', + expect.objectContaining({ __mock: 'request' }) + ); + expect(aiTelemetryClient.reportEditResult).toHaveBeenCalledWith( + expect.objectContaining({ + editSuccess: true, + isCreation: true, + validation: { valid: true }, + }) + ); + }); + + it('passes compact errors to telemetry when the generated YAML fails validation', async () => { + generateWorkflowMock.mockResolvedValueOnce({ + workflow: generatedWorkflow, + response: 'created', + } as any); + validateWorkflowMock.mockResolvedValueOnce({ + valid: false, + diagnostics: [ + { severity: 'error', source: 'schema', message: 'missing field', path: ['steps', 0] }, + { severity: 'warning', source: 'schema', message: 'soft warning' }, + { severity: 'error', source: 'liquid', message: 'bad template' }, + ], + }); + + const context = buildContext(); + const tool = generateWorkflowTool({ workflowsManagement, aiTelemetryClient }); + await tool.handler({ query: 'q' } as any, context); + + expect(aiTelemetryClient.reportEditResult).toHaveBeenCalledWith( + expect.objectContaining({ + editSuccess: true, + validation: { + valid: false, + errors: ['[schema] missing field (at steps.0)', '[liquid] bad template'], + }, + }) + ); + }); + + it('omits validation in telemetry when validateWorkflow throws', async () => { + generateWorkflowMock.mockResolvedValueOnce({ + workflow: generatedWorkflow, + response: 'created', + } as any); + validateWorkflowMock.mockRejectedValueOnce(new Error('validator down')); + + const context = buildContext(); + const tool = generateWorkflowTool({ workflowsManagement, aiTelemetryClient }); + await tool.handler({ query: 'q' } as any, context); + + expect(aiTelemetryClient.reportEditResult).toHaveBeenCalledWith( + expect.objectContaining({ editSuccess: true, validation: undefined }) + ); + }); + it('returns an errorResult and reports failed telemetry when generateWorkflow throws', async () => { generateWorkflowMock.mockRejectedValueOnce(new Error('boom')); diff --git a/x-pack/platform/plugins/shared/agent_builder_workflows/server/tools/generate_workflow.ts b/x-pack/platform/plugins/shared/agent_builder_workflows/server/tools/generate_workflow.ts index ec91efe3a74b8..7bcfa29fb9ef0 100644 --- a/x-pack/platform/plugins/shared/agent_builder_workflows/server/tools/generate_workflow.ts +++ b/x-pack/platform/plugins/shared/agent_builder_workflows/server/tools/generate_workflow.ts @@ -7,18 +7,47 @@ import { v4 } from 'uuid'; import { z } from '@kbn/zod/v4'; +import type { KibanaRequest } from '@kbn/core/server'; import { platformCoreTools, ToolType } from '@kbn/agent-builder-common'; import type { BuiltinToolDefinition } from '@kbn/agent-builder-server'; import { generateWorkflow, type GenerateWorkflowEdit } from '@kbn/agent-builder-workflow-gen'; import { cleanPrompt } from '@kbn/agent-builder-genai-utils/prompts'; import { errorResult, otherResult } from '@kbn/agent-builder-genai-utils/tools/utils/results'; -import type { WorkflowsServerPluginSetup } from '@kbn/workflows-management-plugin/server'; +import type { + WorkflowsManagementApi, + WorkflowsServerPluginSetup, +} from '@kbn/workflows-management-plugin/server'; import { workflowIdSchema } from '@kbn/workflows-management-plugin/common/lib/workflow_id_schema'; import { WORKFLOW_YAML_ATTACHMENT_TYPE } from '@kbn/workflows/common/constants'; import { stringifyWorkflowDefinition } from '@kbn/workflows-yaml'; import type { WorkflowsAiTelemetryClient } from '../telemetry/workflows_ai_telemetry_client'; import { emitWorkflowDiff, extractConversationId } from './utils/workflow_attachments'; +interface CompactValidation { + valid: boolean; + errors?: string[]; +} + +const runCompactValidation = async ( + yaml: string, + api: WorkflowsManagementApi, + spaceId: string, + request: KibanaRequest +): Promise => { + try { + const result = await api.validateWorkflow(yaml, spaceId, request); + if (result.valid) { + return { valid: true }; + } + const errors = result.diagnostics + .filter((d) => d.severity === 'error') + .map((d) => `[${d.source}] ${d.message}${d.path ? ` (at ${d.path.join('.')})` : ''}`); + return { valid: false, errors }; + } catch { + return undefined; + } +}; + const generateWorkflowSchema = z.object({ query: z .string() @@ -162,11 +191,14 @@ And you should **not**: toolId: platformCoreTools.generateWorkflow, }); + const validation = await runCompactValidation(afterYaml, workflowsApi, spaceId, request); + aiTelemetryClient.reportEditResult({ toolId: platformCoreTools.generateWorkflow, conversationId: extractConversationId(toolContext), editSuccess: true, isCreation: !sourceAttachment, + validation, }); return {