[Agent Builder] built-in agents can clear default system instructions (elastic#244692)

## Summary

Fix elastic/search-team#11964

- Add a new `replace_default_instructions` option to agent
configuration, only usable by built-in agents.
- When set to true, we will remove (most of) our default system prompt
and use the provided custom instructions as a replacement instead of an
extension

- Also do some cleanup in the agent prompts by extracting some things to
utils

---------

Co-authored-by: Peter Pisljar <peter.pisljar@gmail.com>

Author: pgayvallet

x-pack/platform/packages/shared/onechat/onechat-common/agents/definition.ts

@@ -75,6 +75,13 @@ export interface AgentConfiguration {
    * For custom per-step instructions, use the `research` and `answer` configuration fields instead.
    */
   instructions?: string;
+  /**
+   * If set to true, the custom instructions will be used as a replacement for the system prompt instead of extending it.
+   *
+   * This will impact both the research and answer prompts. For custom per-step instructions, use the `research` and `answer` configuration fields instead.
+   */
+  replace_default_instructions?: boolean;
+
   /**
    * List of tools exposed to the agent
    */
@@ -96,13 +103,21 @@ export interface AgentResearchStepConfiguration {
    * Custom instruction for the agent's research step.
    */
   instructions?: string;
+  /**
+   * If set to true, the custom instructions will be used as a replacement for the system prompt instead of extending it.
+   */
+  replace_default_instructions?: boolean;
 }
 
 export interface AgentAnswerStepConfiguration {
   /**
    * Custom instruction for the agent's answer step.
    */
   instructions?: string;
+  /**
+   * If set to true, the custom instructions will be used as a replacement for the system prompt instead of extending it.
+   */
+  replace_default_instructions?: boolean;
 }
 
 /**

x-pack/platform/plugins/shared/onechat/server/services/agents/modes/default/answer_agent_structured.ts

@@ -11,9 +11,10 @@ import type { ResolvedAgentCapabilities } from '@kbn/onechat-common';
 import type { AgentEventEmitter } from '@kbn/onechat-server';
 import { createReasoningEvent } from '@kbn/onechat-genai-utils/langchain';
 import type { Logger } from '@kbn/logging';
-import { errorAction } from './actions';
+import type { ProcessedAttachmentType } from '../utils/prepare_conversation';
 import type { ResolvedConfiguration } from '../types';
 import { convertError, isRecoverableError } from '../utils/errors';
+import { errorAction } from './actions';
 import { getStructuredAnswerPrompt } from './prompts';
 import { getRandomAnsweringMessage } from './i18n';
 import { tags } from './constants';
@@ -38,13 +39,15 @@ export const createAnswerAgentStructured = ({
   capabilities,
   events,
   outputSchema,
+  attachmentTypes,
 }: {
   chatModel: InferenceChatModel;
   configuration: ResolvedConfiguration;
   capabilities: ResolvedAgentCapabilities;
   events: AgentEventEmitter;
   outputSchema?: Record<string, unknown>;
   logger: Logger;
+  attachmentTypes: ProcessedAttachmentType[];
 }) => {
   return async (state: StateType) => {
     if (state.answerActions.length === 0 && state.errorCount === 0) {
@@ -82,6 +85,7 @@ export const createAnswerAgentStructured = ({
         initialMessages: state.initialMessages,
         actions: state.mainActions,
         answerActions: state.answerActions,
+        attachmentTypes,
       });
 
       const response = await structuredModel.invoke(prompt);

x-pack/platform/plugins/shared/onechat/server/services/agents/modes/default/graph.ts

@@ -18,7 +18,7 @@ import type { AgentEventEmitter } from '@kbn/onechat-server';
 import { createReasoningEvent, createToolCallMessage } from '@kbn/onechat-genai-utils/langchain';
 import type { ResolvedConfiguration } from '../types';
 import { convertError, isRecoverableError } from '../utils/errors';
-import { getActPrompt, getAnswerPrompt } from './prompts';
+import { getResearchAgentPrompt, getAnswerAgentPrompt } from './prompts';
 import { getRandomAnsweringMessage, getRandomThinkingMessage } from './i18n';
 import { steps, tags } from './constants';
 import type { StateType } from './state';
@@ -76,8 +76,9 @@ export const createAgentGraph = ({
     }
     try {
       const response = await researcherModel.invoke(
-        getActPrompt({
+        getResearchAgentPrompt({
           customInstructions: configuration.research.instructions,
+          clearSystemMessage: configuration.research.replace_default_instructions,
           capabilities,
           initialMessages: state.initialMessages,
           actions: state.mainActions,
@@ -168,12 +169,14 @@ export const createAgentGraph = ({
     }
     try {
       const response = await answeringModel.invoke(
-        getAnswerPrompt({
+        getAnswerAgentPrompt({
           customInstructions: configuration.answer.instructions,
+          clearSystemMessage: configuration.answer.replace_default_instructions,
           capabilities,
           initialMessages: state.initialMessages,
           actions: state.mainActions,
           answerActions: state.answerActions,
+          attachmentTypes: processedConversation.attachmentTypes,
         })
       );
 
@@ -202,6 +205,7 @@ export const createAgentGraph = ({
     capabilities,
     events,
     outputSchema,
+    attachmentTypes: processedConversation.attachmentTypes,
     logger,
   });