[Agent Builder] use getAgentDescription from attachment types (elastic#243541)

## Summary

We forgot to take into account instructions from attachment type
(`getAgentDescription`) in the agent's description.

This PR addresses the problem

Author: pgayvallet

x-pack/platform/plugins/shared/onechat/server/services/agents/modes/default/graph.ts

@@ -36,6 +36,7 @@ import {
   errorAction,
   handoverAction,
 } from './actions';
+import type { ProcessedConversation } from '../utils/prepare_conversation';
 
 // number of successive recoverable errors we try to recover from before throwing
 const MAX_ERROR_COUNT = 2;
@@ -47,13 +48,15 @@ export const createAgentGraph = ({
   capabilities,
   logger,
   events,
+  processedConversation,
 }: {
   chatModel: InferenceChatModel;
   tools: StructuredTool[];
   capabilities: ResolvedAgentCapabilities;
   configuration: ResolvedConfiguration;
   logger: Logger;
   events: AgentEventEmitter;
+  processedConversation: ProcessedConversation;
 }) => {
   const toolNode = new ToolNode<BaseMessage[]>(tools);
 
@@ -72,6 +75,7 @@ export const createAgentGraph = ({
           capabilities,
           initialMessages: state.initialMessages,
           actions: state.mainActions,
+          attachmentTypes: processedConversation.attachmentTypes,
         })
       );
 

x-pack/platform/plugins/shared/onechat/server/services/agents/modes/default/prompts.ts

@@ -17,6 +17,7 @@ import { ChartType } from '@kbn/visualization-utils';
 import { customInstructionsBlock, formatDate } from './prompts/prompt_helpers';
 import type { ResearchAgentAction, AnswerAgentAction } from './actions';
 import { formatResearcherActionHistory, formatAnswerActionHistory } from './prompts/format_actions';
+import type { ProcessedAttachmentType } from '../utils/prepare_conversation';
 
 const tools = {
   indexExplorer: sanitizeToolId(platformCoreTools.indexExplorer),
@@ -29,11 +30,13 @@ export const getActPrompt = ({
   capabilities,
   initialMessages,
   actions,
+  attachmentTypes,
 }: {
   customInstructions?: string;
   capabilities: ResolvedAgentCapabilities;
   initialMessages: BaseMessageLike[];
   actions: ResearchAgentAction[];
+  attachmentTypes: ProcessedAttachmentType[];
 }): BaseMessageLike[] => {
   return [
     [
@@ -126,6 +129,8 @@ Constraints:
 
 ${customInstructionsBlock(customInstructions)}
 
+${renderAttachmentTypeInstructions(attachmentTypes)}
+
 ## ADDITIONAL INFO
 - Current date: ${formatDate()}
 
@@ -243,3 +248,23 @@ function renderVisualizationPrompt() {
       To visualize this response as a bar chart your reply should be:
       <${tagName} ${attributes.toolResultId}="LiDoF1" ${attributes.chartType}="${ChartType.Bar}"/>`;
 }
+
+const renderAttachmentTypeInstructions = (attachmentTypes: ProcessedAttachmentType[]): string => {
+  if (attachmentTypes.length === 0) {
+    return '';
+  }
+
+  const perTypeInstructions = attachmentTypes.map(({ type, agentDescription }) => {
+    return `### ${type} attachments
+
+${agentDescription ?? 'No instructions available.'}
+`;
+  });
+
+  return `## ATTACHMENT TYPES
+
+  The current conversation contains attachments. Here is the list of attachment types present in the conversation and their corresponding instructions:
+
+${perTypeInstructions.join('\n\n')}
+  `;
+};

x-pack/platform/plugins/shared/onechat/server/services/agents/modes/default/run_chat_agent.ts

@@ -112,6 +112,7 @@ export const runDefaultAgentMode: RunChatAgentFn = async (
     tools: allTools,
     configuration: resolvedConfiguration,
     capabilities: resolvedCapabilities,
+    processedConversation,
   });
 
   logger.debug(`Running chat agent with graph: ${chatAgentGraphName}, runId: ${runId}`);

x-pack/platform/plugins/shared/onechat/server/services/agents/modes/utils/prepare_conversation.test.ts

@@ -83,6 +83,7 @@ describe('prepareConversation', () => {
       });
 
       expect(result).toEqual({
+        attachmentTypes: [],
         nextInput: {
           message: 'Hello',
           attachments: [],
@@ -145,7 +146,7 @@ describe('prepareConversation', () => {
       });
 
       expect(mockGetToolResultId).toHaveBeenCalledTimes(1);
-      expect(mockAttachmentsService.getTypeDefinition).toHaveBeenCalledTimes(1);
+      expect(mockAttachmentsService.getTypeDefinition).toHaveBeenCalledTimes(2);
     });
 
     it('should preserve existing ID for attachment with ID', async () => {
@@ -182,7 +183,7 @@ describe('prepareConversation', () => {
       });
 
       expect(mockGetToolResultId).not.toHaveBeenCalled();
-      expect(mockAttachmentsService.getTypeDefinition).toHaveBeenCalledTimes(1);
+      expect(mockAttachmentsService.getTypeDefinition).toHaveBeenCalledTimes(2);
     });
 
     it('should process multiple attachments', async () => {
@@ -371,7 +372,7 @@ describe('prepareConversation', () => {
         representation: mockRepresentation,
       });
 
-      expect(mockAttachmentsService.getTypeDefinition).toHaveBeenCalledTimes(1);
+      expect(mockAttachmentsService.getTypeDefinition).toHaveBeenCalledTimes(2);
     });
 
     it('should process multiple previous rounds', async () => {

x-pack/platform/plugins/shared/onechat/server/services/agents/modes/utils/prepare_conversation.ts

@@ -17,6 +17,11 @@ export interface ProcessedAttachment {
   representation: AttachmentRepresentation;
 }
 
+export interface ProcessedAttachmentType {
+  type: string;
+  agentDescription?: string;
+}
+
 export interface ProcessedRoundInput {
   message: string;
   attachments: ProcessedAttachment[];
@@ -29,6 +34,7 @@ export type ProcessedConversationRound = Omit<ConversationRound, 'input'> & {
 export interface ProcessedConversation {
   previousRounds: ProcessedConversationRound[];
   nextInput: ProcessedRoundInput;
+  attachmentTypes: ProcessedAttachmentType[];
 }
 
 export const prepareConversation = async ({
@@ -41,13 +47,37 @@ export const prepareConversation = async ({
   attachmentsService: AttachmentsService;
 }): Promise<ProcessedConversation> => {
   const processedNextInput = await prepareRoundInput({ input: nextInput, attachmentsService });
+  const processedRounds = await Promise.all(
+    previousRounds.map((round) => {
+      return prepareRound({ round, attachmentsService });
+    })
+  );
+
+  const attachmentTypeIds = [
+    ...new Set<string>([
+      ...processedNextInput.attachments.map((attachment) => attachment.attachment.type),
+      ...processedRounds.flatMap((round) =>
+        round.input.attachments.map((attachment) => attachment.attachment.type)
+      ),
+    ]),
+  ];
+
+  const attachmentTypes = await Promise.all(
+    attachmentTypeIds.map<Promise<ProcessedAttachmentType>>(async (type) => {
+      const definition = attachmentsService.getTypeDefinition(type);
+      const description = definition?.getAgentDescription?.() ?? '';
+
+      return {
+        type,
+        description,
+      };
+    })
+  );
+
   return {
     nextInput: processedNextInput,
-    previousRounds: await Promise.all(
-      previousRounds.map((round) => {
-        return prepareRound({ round, attachmentsService });
-      })
-    ),
+    previousRounds: processedRounds,
+    attachmentTypes,
   };
 };
 

x-pack/platform/plugins/shared/onechat/server/services/agents/modes/utils/to_langchain_messages.test.ts

@@ -91,7 +91,7 @@ describe('conversationLangchainMessages', () => {
   it('returns only the user message if no previous rounds', () => {
     const nextInput = makeRoundInput('hello');
     const result = conversationToLangchainMessages({
-      conversation: { previousRounds: [], nextInput },
+      conversation: { previousRounds: [], nextInput, attachmentTypes: [] },
     });
     expect(result).toHaveLength(1);
     expect(isHumanMessage(result[0])).toBe(true);
@@ -112,7 +112,7 @@ describe('conversationLangchainMessages', () => {
     ];
     const nextInput = makeRoundInput('how are you?');
     const result = conversationToLangchainMessages({
-      conversation: { previousRounds, nextInput },
+      conversation: { previousRounds, nextInput, attachmentTypes: [] },
     });
 
     expect(result).toHaveLength(3);
@@ -149,7 +149,7 @@ describe('conversationLangchainMessages', () => {
     ];
     const nextInput = makeRoundInput('next');
     const result = conversationToLangchainMessages({
-      conversation: { previousRounds, nextInput },
+      conversation: { previousRounds, nextInput, attachmentTypes: [] },
     });
     // 1 user + 1 tool call (AI + Tool) + 1 assistant + 1 user
     expect(result).toHaveLength(5);
@@ -215,7 +215,7 @@ describe('conversationLangchainMessages', () => {
     ];
     const nextInput = makeRoundInput('bye');
     const result = conversationToLangchainMessages({
-      conversation: { previousRounds, nextInput },
+      conversation: { previousRounds, nextInput, attachmentTypes: [] },
     });
     // 1 user + 1 assistant + 1 user + 1 tool call (AI + Tool) + 1 assistant + 1 user
     expect(result).toHaveLength(7);
@@ -274,7 +274,7 @@ describe('conversationLangchainMessages', () => {
     ];
     const nextInput = makeRoundInput('next');
     const result = conversationToLangchainMessages({
-      conversation: { previousRounds, nextInput },
+      conversation: { previousRounds, nextInput, attachmentTypes: [] },
     });
     // 1 user + 1 tool call (AI + Tool) + 1 assistant + 1 user
     expect(result).toHaveLength(5);
@@ -295,7 +295,7 @@ describe('conversationLangchainMessages', () => {
       );
       const nextInput = makeRoundInput('hello with attachment', [attachment]);
       const result = conversationToLangchainMessages({
-        conversation: { previousRounds: [], nextInput },
+        conversation: { previousRounds: [], nextInput, attachmentTypes: [] },
       });
 
       expect(result).toHaveLength(1);
@@ -326,7 +326,7 @@ describe('conversationLangchainMessages', () => {
         attachment2,
       ]);
       const result = conversationToLangchainMessages({
-        conversation: { previousRounds: [], nextInput },
+        conversation: { previousRounds: [], nextInput, attachmentTypes: [] },
       });
 
       expect(result).toHaveLength(1);
@@ -361,7 +361,7 @@ describe('conversationLangchainMessages', () => {
       ];
       const nextInput = makeRoundInput('next message');
       const result = conversationToLangchainMessages({
-        conversation: { previousRounds, nextInput },
+        conversation: { previousRounds, nextInput, attachmentTypes: [] },
       });
 
       expect(result).toHaveLength(3);

x-pack/platform/plugins/shared/onechat/server/services/attachments/definitions/esql.ts

@@ -10,6 +10,7 @@ import type { EsqlAttachmentData } from '@kbn/onechat-common/attachments';
 import { AttachmentType, esqlAttachmentDataSchema } from '@kbn/onechat-common/attachments';
 import { platformCoreTools } from '@kbn/onechat-common/tools';
 import type { AttachmentTypeDefinition } from '@kbn/onechat-server/attachments';
+import { sanitizeToolId } from '@kbn/onechat-genai-utils/langchain';
 
 /**
  * Creates the definition for the `text` attachment type.
@@ -43,6 +44,11 @@ export const createEsqlAttachmentType = (): AttachmentTypeDefinition<
         },
       };
     },
+    getAgentDescription: () => {
+      return `${AttachmentType.esql} can be executed using the ${sanitizeToolId(
+        platformCoreTools.executeEsql
+      )} tool`;
+    },
     getTools: () => [platformCoreTools.executeEsql, platformCoreTools.generateEsql],
   };
 };