From f45056ec07b22f84b490880996dba23d4ec5c0e6 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Mon, 20 Apr 2026 19:40:50 +0200 Subject: [PATCH 01/81] feat(agentic-ai): extract documents from tool call results into user messages Documents in tool call results were serialized as base64 blobs in the tool result text, which models cannot interpret. This change extracts documents from tool call results into a follow-up UserMessage with proper DocumentContent blocks, making them visible to the LLM via the existing document-to-content conversion path (ImageContent, PdfFileContent, TextContent). - Add ToolCallResultDocumentExtractor to recursively find Document instances in tool call result content trees - Create a document UserMessage (tagged with toolCallDocuments metadata) after the ToolCallResultMessage, before event messages - Extract documents from event messages into DocumentContent blocks - Serialize documents in tool results as document references (default DocumentSerializer) instead of base64 content blocks - Remove stale DocumentToContentSerializer/Module/ResponseModel infrastructure - Simplify ToolCallConverterImpl (remove ContentConverter dependency) - Simplify ContentConverterImpl (remove contentObjectMapper copy) - Update e2e tests to assert new message structure ADR: docs/adr/003-document-handling-in-tool-call-results.md --- .../BaseL4JAiAgentJobWorkerTest.java | 4 +- .../L4JAiAgentJobWorkerToolCallingTests.java | 145 ++++++++---- .../BaseL4JAiAgentConnectorTest.java | 4 +- .../L4JAiAgentConnectorToolCallingTests.java | 144 +++++++---- ...-document-handling-in-tool-call-results.md | 122 ++++++++++ ...ment-handling-in-tool-call-results.plan.md | 168 +++++++++++++ connectors/agentic-ai/pom.xml | 5 + .../agent/AgentMessagesHandlerImpl.java | 42 +++- .../ToolCallResultDocumentExtractor.java | 77 ++++++ .../langchain4j/ContentConverterImpl.java | 8 +- ...icAiLangchain4JFrameworkConfiguration.java | 4 +- .../document/DocumentToContentModule.java | 26 -- .../DocumentToContentResponseModel.java | 19 -- .../document/DocumentToContentSerializer.java | 97 -------- .../tool/ToolCallConverterImpl.java | 17 +- .../AgenticAiConnectorsAutoConfiguration.java | 14 +- .../agent/AgentMessagesHandlerTest.java | 190 ++++++++++++++- .../ToolCallResultDocumentExtractorTest.java | 223 ++++++++++++++++++ .../langchain4j/ContentConverterTest.java | 27 ++- .../DocumentToContentSerializerTest.java | 178 -------------- .../tool/ToolCallConverterTest.java | 48 ++-- 21 files changed, 1104 insertions(+), 458 deletions(-) create mode 100644 connectors/agentic-ai/docs/adr/003-document-handling-in-tool-call-results.md create mode 100644 connectors/agentic-ai/docs/adr/003-document-handling-in-tool-call-results.plan.md create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java delete mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/document/DocumentToContentModule.java delete mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/document/DocumentToContentResponseModel.java delete mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/document/DocumentToContentSerializer.java create mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractorTest.java delete mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/document/DocumentToContentSerializerTest.java diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/BaseL4JAiAgentJobWorkerTest.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/BaseL4JAiAgentJobWorkerTest.java index 59dd1213387..736cb776fff 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/BaseL4JAiAgentJobWorkerTest.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/BaseL4JAiAgentJobWorkerTest.java @@ -38,9 +38,9 @@ import dev.langchain4j.model.output.TokenUsage; import io.camunda.connector.agenticai.adhoctoolsschema.schema.AdHocToolsSchemaResolver; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelFactory; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.document.DocumentToContentResponseModel; import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.agenticai.aiagent.model.JobWorkerAgentResponse; +import io.camunda.connector.document.jackson.DocumentReferenceModel; import io.camunda.connector.e2e.ElementTemplate; import io.camunda.connector.e2e.ZeebeTest; import io.camunda.connector.e2e.agenticai.aiagent.BaseAiAgentJobWorkerTest; @@ -377,5 +377,5 @@ protected static ChatInteraction of( } } - protected record DownloadFileToolResult(int status, DocumentToContentResponseModel document) {} + protected record DownloadFileToolResult(int status, DocumentReferenceModel document) {} } diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java index b856e6d4c37..1ad07ae7c69 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java @@ -18,18 +18,22 @@ import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.FEEDBACK_LOOP_RESPONSE_TEXT; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import dev.langchain4j.agent.tool.ToolExecutionRequest; import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.data.message.Content; +import dev.langchain4j.data.message.ImageContent; +import dev.langchain4j.data.message.PdfFileContent; import dev.langchain4j.data.message.SystemMessage; +import dev.langchain4j.data.message.TextContent; import dev.langchain4j.data.message.ToolExecutionResultMessage; import dev.langchain4j.data.message.UserMessage; import dev.langchain4j.model.chat.response.ChatResponse; import dev.langchain4j.model.chat.response.ChatResponseMetadata; import dev.langchain4j.model.output.FinishReason; import dev.langchain4j.model.output.TokenUsage; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.document.DocumentToContentResponseModel; import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.e2e.agenticai.assertj.JobWorkerAgentResponseAssert; import io.camunda.connector.test.utils.annotation.SlowTest; @@ -77,44 +81,23 @@ void executesAgentWithToolCallingAndUserFeedback() throws Exception { void supportsDocumentResponsesFromToolCalls( String filename, String type, String mimeType, WireMockRuntimeInfo wireMock) throws Exception { - DownloadFileToolResult expectedDownloadFileResult; - if (type.equals("text")) { - expectedDownloadFileResult = - new DownloadFileToolResult( - 200, - new DocumentToContentResponseModel(type, mimeType, testFileContent(filename).get())); - } else { - expectedDownloadFileResult = - new DownloadFileToolResult( - 200, - new DocumentToContentResponseModel( - type, mimeType, testFileContentBase64(filename).get())); - } - final var initialUserPrompt = "Go and download a document!"; - final var expectedConversation = - List.of( - new SystemMessage( - "You are a helpful AI assistant. Answer all the questions, but always be nice. Explain your thinking."), - new UserMessage(initialUserPrompt), - new AiMessage( - "The user asked me to download a document. I will call the Download_A_File tool to do so.", - List.of( - ToolExecutionRequest.builder() - .id("aaa111") - .name("Download_A_File") - .arguments( - "{\"url\": \"%s\"}" - .formatted(wireMock.getHttpBaseUrl() + "/" + filename)) - .build())), - new ToolExecutionResultMessage( - "aaa111", - "Download_A_File", - objectMapper.writeValueAsString(expectedDownloadFileResult)), - new AiMessage( - "I loaded a document and learned that it contains interesting data. Anything specific you want to know?"), - new UserMessage("What is the content type?"), - new AiMessage("The content type is '%s'".formatted(mimeType))); + + // The AI message with tool call and the subsequent responses for the conversation + final var aiToolCallMessage = + new AiMessage( + "The user asked me to download a document. I will call the Download_A_File tool to do so.", + List.of( + ToolExecutionRequest.builder() + .id("aaa111") + .name("Download_A_File") + .arguments( + "{\"url\": \"%s\"}".formatted(wireMock.getHttpBaseUrl() + "/" + filename)) + .build())); + final var aiResponseAfterTool = + new AiMessage( + "I loaded a document and learned that it contains interesting data. Anything specific you want to know?"); + final var aiFinalResponse = new AiMessage("The content type is '%s'".formatted(mimeType)); mockChatInteractions( ChatInteraction.of( @@ -124,7 +107,7 @@ void supportsDocumentResponsesFromToolCalls( .finishReason(FinishReason.TOOL_EXECUTION) .tokenUsage(new TokenUsage(10, 20)) .build()) - .aiMessage((AiMessage) expectedConversation.get(2)) + .aiMessage(aiToolCallMessage) .build()), ChatInteraction.of( ChatResponse.builder() @@ -133,7 +116,7 @@ void supportsDocumentResponsesFromToolCalls( .finishReason(FinishReason.STOP) .tokenUsage(new TokenUsage(100, 200)) .build()) - .aiMessage((AiMessage) expectedConversation.get(4)) + .aiMessage(aiResponseAfterTool) .build(), userFollowUpFeedback("What is the content type?")), ChatInteraction.of( @@ -143,7 +126,7 @@ void supportsDocumentResponsesFromToolCalls( .finishReason(FinishReason.STOP) .tokenUsage(new TokenUsage(11, 22)) .build()) - .aiMessage((AiMessage) expectedConversation.get(6)) + .aiMessage(aiFinalResponse) .build(), userSatisfiedFeedback())); @@ -154,18 +137,90 @@ void supportsDocumentResponsesFromToolCalls( Map.of("userPrompt", initialUserPrompt)) .waitForProcessCompletion(); - assertLastChatRequest(expectedConversation); + // Assert the conversation structure with document extraction + await() + .alias("Chat request captured") + .untilAsserted(() -> assertThat(chatRequestCaptor.getValue()).isNotNull()); + + final var lastMessages = chatRequestCaptor.getValue().messages(); + + // Expected message order (8 messages, last AI response not included in request): + // 0: SystemMessage + // 1: UserMessage (initial prompt) + // 2: AiMessage (tool call) + // 3: ToolExecutionResultMessage (document serialized as reference) + // 4: UserMessage (document content extracted from tool result) + // 5: AiMessage (response after tool) + // 6: UserMessage (follow-up question) + assertThat(lastMessages).hasSize(7); + + assertThat(lastMessages.get(0)).isInstanceOf(SystemMessage.class); + assertThat(lastMessages.get(1)).isInstanceOf(UserMessage.class); + assertThat(lastMessages.get(2)).isInstanceOf(AiMessage.class); + + // Tool result: document serialized as document reference + assertThat(lastMessages.get(3)) + .isInstanceOfSatisfying( + ToolExecutionResultMessage.class, + msg -> { + assertThat(msg.id()).isEqualTo("aaa111"); + assertThat(msg.toolName()).isEqualTo("Download_A_File"); + assertThat(msg.text()).contains("camunda.document.type"); + assertThat(msg.text()).contains(mimeType); + }); + + // Document user message: extracted document content + assertThat(lastMessages.get(4)) + .isInstanceOfSatisfying( + UserMessage.class, + msg -> { + List contents = msg.contents(); + assertThat(contents).hasSize(2); + assertThat(contents.get(0)) + .isInstanceOfSatisfying( + TextContent.class, + tc -> + assertThat(tc.text()) + .isEqualTo("Tool call 'Download_A_File' (aaa111) documents:")); + assertDocumentContentBlock(contents.get(1), type, mimeType); + }); + + assertThat(lastMessages.get(5)).isInstanceOf(AiMessage.class); + + assertThat(lastMessages.get(6)) + .isInstanceOfSatisfying( + UserMessage.class, + msg -> assertThat(msg.singleText()).isEqualTo("What is the content type?")); - String expectedResponseText = ((AiMessage) expectedConversation.getLast()).text(); assertAgentResponse( zeebeTest, agentResponse -> JobWorkerAgentResponseAssert.assertThat(agentResponse) .isReady() .hasMetrics(new AgentMetrics(3, new AgentMetrics.TokenUsage(121, 242))) - .hasResponseMessageText(expectedResponseText) - .hasResponseText(expectedResponseText)); + .hasResponseMessageText(aiFinalResponse.text()) + .hasResponseText(aiFinalResponse.text())); assertThat(userFeedbackJobWorkerCounter.get()).isEqualTo(2); } + + private void assertDocumentContentBlock( + Content content, String expectedType, String expectedMimeType) { + if (expectedType.equals("text")) { + assertThat(content).isInstanceOf(TextContent.class); + } else if (expectedMimeType.equals("application/pdf")) { + assertThat(content) + .isInstanceOfSatisfying( + PdfFileContent.class, pdf -> assertThat(pdf.pdfFile().base64Data()).isNotBlank()); + } else { + // image types + assertThat(content) + .isInstanceOfSatisfying( + ImageContent.class, + img -> { + assertThat(img.image().mimeType()).isEqualTo(expectedMimeType); + assertThat(img.image().base64Data()).isNotBlank(); + }); + } + } } diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/BaseL4JAiAgentConnectorTest.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/BaseL4JAiAgentConnectorTest.java index 8217b6a2dc0..d359fa70aa4 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/BaseL4JAiAgentConnectorTest.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/BaseL4JAiAgentConnectorTest.java @@ -37,9 +37,9 @@ import dev.langchain4j.model.output.TokenUsage; import io.camunda.connector.agenticai.adhoctoolsschema.schema.AdHocToolsSchemaResolver; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelFactory; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.document.DocumentToContentResponseModel; import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.agenticai.aiagent.model.AgentResponse; +import io.camunda.connector.document.jackson.DocumentReferenceModel; import io.camunda.connector.e2e.ElementTemplate; import io.camunda.connector.e2e.ZeebeTest; import io.camunda.connector.e2e.agenticai.aiagent.BaseAiAgentConnectorTest; @@ -339,5 +339,5 @@ protected static ChatInteraction of( } } - protected record DownloadFileToolResult(int status, DocumentToContentResponseModel document) {} + protected record DownloadFileToolResult(int status, DocumentReferenceModel document) {} } diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java index 6be1bfd8d53..0c5e3f05ab2 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java @@ -18,18 +18,22 @@ import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.FEEDBACK_LOOP_RESPONSE_TEXT; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import dev.langchain4j.agent.tool.ToolExecutionRequest; import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.data.message.Content; +import dev.langchain4j.data.message.ImageContent; +import dev.langchain4j.data.message.PdfFileContent; import dev.langchain4j.data.message.SystemMessage; +import dev.langchain4j.data.message.TextContent; import dev.langchain4j.data.message.ToolExecutionResultMessage; import dev.langchain4j.data.message.UserMessage; import dev.langchain4j.model.chat.response.ChatResponse; import dev.langchain4j.model.chat.response.ChatResponseMetadata; import dev.langchain4j.model.output.FinishReason; import dev.langchain4j.model.output.TokenUsage; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.document.DocumentToContentResponseModel; import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.e2e.agenticai.assertj.AgentResponseAssert; import io.camunda.connector.test.utils.annotation.SlowTest; @@ -71,44 +75,22 @@ void executesAgentWithToolCallingAndUserFeedback() throws Exception { void supportsDocumentResponsesFromToolCalls( String filename, String type, String mimeType, WireMockRuntimeInfo wireMock) throws Exception { - DownloadFileToolResult expectedDownloadFileResult; - if (type.equals("text")) { - expectedDownloadFileResult = - new DownloadFileToolResult( - 200, - new DocumentToContentResponseModel(type, mimeType, testFileContent(filename).get())); - } else { - expectedDownloadFileResult = - new DownloadFileToolResult( - 200, - new DocumentToContentResponseModel( - type, mimeType, testFileContentBase64(filename).get())); - } - final var initialUserPrompt = "Go and download a document!"; - final var expectedConversation = - List.of( - new SystemMessage( - "You are a helpful AI assistant. Answer all the questions, but always be nice. Explain your thinking."), - new UserMessage(initialUserPrompt), - new AiMessage( - "The user asked me to download a document. I will call the Download_A_File tool to do so.", - List.of( - ToolExecutionRequest.builder() - .id("aaa111") - .name("Download_A_File") - .arguments( - "{\"url\": \"%s\"}" - .formatted(wireMock.getHttpBaseUrl() + "/" + filename)) - .build())), - new ToolExecutionResultMessage( - "aaa111", - "Download_A_File", - objectMapper.writeValueAsString(expectedDownloadFileResult)), - new AiMessage( - "I loaded a document and learned that it contains interesting data. Anything specific you want to know?"), - new UserMessage("What is the content type?"), - new AiMessage("The content type is '%s'".formatted(mimeType))); + + final var aiToolCallMessage = + new AiMessage( + "The user asked me to download a document. I will call the Download_A_File tool to do so.", + List.of( + ToolExecutionRequest.builder() + .id("aaa111") + .name("Download_A_File") + .arguments( + "{\"url\": \"%s\"}".formatted(wireMock.getHttpBaseUrl() + "/" + filename)) + .build())); + final var aiResponseAfterTool = + new AiMessage( + "I loaded a document and learned that it contains interesting data. Anything specific you want to know?"); + final var aiFinalResponse = new AiMessage("The content type is '%s'".formatted(mimeType)); mockChatInteractions( ChatInteraction.of( @@ -118,7 +100,7 @@ void supportsDocumentResponsesFromToolCalls( .finishReason(FinishReason.TOOL_EXECUTION) .tokenUsage(new TokenUsage(10, 20)) .build()) - .aiMessage((AiMessage) expectedConversation.get(2)) + .aiMessage(aiToolCallMessage) .build()), ChatInteraction.of( ChatResponse.builder() @@ -127,7 +109,7 @@ void supportsDocumentResponsesFromToolCalls( .finishReason(FinishReason.STOP) .tokenUsage(new TokenUsage(100, 200)) .build()) - .aiMessage((AiMessage) expectedConversation.get(4)) + .aiMessage(aiResponseAfterTool) .build(), userFollowUpFeedback("What is the content type?")), ChatInteraction.of( @@ -137,7 +119,7 @@ void supportsDocumentResponsesFromToolCalls( .finishReason(FinishReason.STOP) .tokenUsage(new TokenUsage(11, 22)) .build()) - .aiMessage((AiMessage) expectedConversation.get(6)) + .aiMessage(aiFinalResponse) .build(), userSatisfiedFeedback())); @@ -148,9 +130,61 @@ void supportsDocumentResponsesFromToolCalls( Map.of("userPrompt", initialUserPrompt)) .waitForProcessCompletion(); - assertLastChatRequest(3, expectedConversation); + // Assert the conversation structure with document extraction + await() + .alias("Chat request captured") + .untilAsserted(() -> assertThat(chatRequestCaptor.getValue()).isNotNull()); + + final var lastMessages = chatRequestCaptor.getValue().messages(); + + // Expected message order (8 messages, last AI response not included in request): + // 0: SystemMessage + // 1: UserMessage (initial prompt) + // 2: AiMessage (tool call) + // 3: ToolExecutionResultMessage (document serialized as reference) + // 4: UserMessage (document content extracted from tool result) + // 5: AiMessage (response after tool) + // 6: UserMessage (follow-up question) + assertThat(lastMessages).hasSize(7); + + assertThat(lastMessages.get(0)).isInstanceOf(SystemMessage.class); + assertThat(lastMessages.get(1)).isInstanceOf(UserMessage.class); + assertThat(lastMessages.get(2)).isInstanceOf(AiMessage.class); + + // Tool result: document serialized as document reference + assertThat(lastMessages.get(3)) + .isInstanceOfSatisfying( + ToolExecutionResultMessage.class, + msg -> { + assertThat(msg.id()).isEqualTo("aaa111"); + assertThat(msg.toolName()).isEqualTo("Download_A_File"); + assertThat(msg.text()).contains("camunda.document.type"); + assertThat(msg.text()).contains(mimeType); + }); + + // Document user message: extracted document content + assertThat(lastMessages.get(4)) + .isInstanceOfSatisfying( + UserMessage.class, + msg -> { + List contents = msg.contents(); + assertThat(contents).hasSize(2); + assertThat(contents.get(0)) + .isInstanceOfSatisfying( + TextContent.class, + tc -> + assertThat(tc.text()) + .isEqualTo("Tool call 'Download_A_File' (aaa111) documents:")); + assertDocumentContentBlock(contents.get(1), type, mimeType); + }); + + assertThat(lastMessages.get(5)).isInstanceOf(AiMessage.class); + + assertThat(lastMessages.get(6)) + .isInstanceOfSatisfying( + UserMessage.class, + msg -> assertThat(msg.singleText()).isEqualTo("What is the content type?")); - String expectedResponseText = ((AiMessage) expectedConversation.getLast()).text(); assertAgentResponse( zeebeTest, agentResponse -> @@ -158,9 +192,29 @@ void supportsDocumentResponsesFromToolCalls( .isReady() .hasNoToolCalls() .hasMetrics(new AgentMetrics(3, new AgentMetrics.TokenUsage(121, 242))) - .hasResponseMessageText(expectedResponseText) - .hasResponseText(expectedResponseText)); + .hasResponseMessageText(aiFinalResponse.text()) + .hasResponseText(aiFinalResponse.text())); assertThat(userFeedbackJobWorkerCounter.get()).isEqualTo(2); } + + private void assertDocumentContentBlock( + Content content, String expectedType, String expectedMimeType) { + if (expectedType.equals("text")) { + assertThat(content).isInstanceOf(TextContent.class); + } else if (expectedMimeType.equals("application/pdf")) { + assertThat(content) + .isInstanceOfSatisfying( + PdfFileContent.class, pdf -> assertThat(pdf.pdfFile().base64Data()).isNotBlank()); + } else { + // image types + assertThat(content) + .isInstanceOfSatisfying( + ImageContent.class, + img -> { + assertThat(img.image().mimeType()).isEqualTo(expectedMimeType); + assertThat(img.image().base64Data()).isNotBlank(); + }); + } + } } diff --git a/connectors/agentic-ai/docs/adr/003-document-handling-in-tool-call-results.md b/connectors/agentic-ai/docs/adr/003-document-handling-in-tool-call-results.md new file mode 100644 index 00000000000..232b80b2fc9 --- /dev/null +++ b/connectors/agentic-ai/docs/adr/003-document-handling-in-tool-call-results.md @@ -0,0 +1,122 @@ +# Document handling in tool call results + +* Deciders: Agentic AI Team +* Date: Apr 10, 2026 + +## Status + +**Proposed**. + +## Context and Problem Statement + +BPMN tool activities (modeled as ad-hoc subprocess tasks) can return complex data structures containing Camunda +Documents, either at the root level or nested within maps and lists. When these tool call results are passed to the LLM, +the documents need to be represented in a way the model can actually interpret. + +The current implementation converts `ToolCallResult.content()` to a single JSON text string for LangChain4J's +`ToolExecutionResultMessage`. Documents encountered during Jackson serialization are converted to a Claude-specific +content block format (`DocumentToContentSerializer`) containing base64-encoded data embedded within the text. This +approach does not work -- models cannot meaningfully interpret large base64 blobs embedded in tool result text. + +In contrast, documents provided via user messages already work correctly: they are converted to proper LangChain4J +content types (`ImageContent`, `PdfFileContent`, `TextContent`) through `DocumentToContentConverterImpl` and arrive at +the model as native multi-modal content blocks. + +## Decision Drivers + +* **Model compatibility**: Documents must be provided in a format that LLMs can actually process. +* **Provider independence**: The solution must work across all supported providers (Anthropic, OpenAI via Bedrock), + not just those with native multi-content tool result support. +* **Auditability**: Document handling must be visible in the persisted conversation history. +* **Abstraction layer**: Changes should be made in the generic agent layer (internal message model), not deep in the + LangChain4J framework adapter, so they are framework-agnostic and auditable. +* **Consistency**: Reuse existing, proven document handling infrastructure where possible. + +## Considered Options + +### Option 1: Multi-content `ToolExecutionResultMessage` (L4J-native) + +Extract documents from tool call results and add them as separate `Content` blocks on the LangChain4J +`ToolExecutionResultMessage`, which supports `List` since v1.12. + +**Rejected** because: + +- LangChain4J provider adapters have inconsistent support for multi-content tool results across providers. The Anthropic + adapter may handle `ImageContent` in tool results, but Bedrock support is unclear. +- Changes would be invisible in the conversation history (only visible at the L4J wire format level). +- Tightly couples the solution to LangChain4J capabilities. + +### Option 2: Text-only (describe binary documents, inline text documents) + +Serialize all documents as textual descriptions (filename, content type) without providing actual content. + +**Rejected** because the model cannot analyze documents it cannot see. This defeats the purpose for use cases that +require visual/content analysis of tool-returned documents. + +### Option 3: Extract documents into appended user messages (selected) + +Extract all documents from tool call results into a follow-up `UserMessage` with `DocumentContent` blocks. The tool +result text retains a document reference (serialized by the default connectors `DocumentSerializer`) so the model can +correlate the reference with the actual content in the user message. + +## Decision + +**Option 3**: Extract documents from tool call results into appended user messages. + +### Design + +**Generic layer** (`AgentMessagesHandlerImpl`): + +1. After building the `ToolCallResultMessage`, scan each `ToolCallResult.content()` tree for `Document` instances. +2. Build a single `UserMessage` containing `TextContent` separators per tool call (for association) and `DocumentContent` + blocks for each extracted document. +3. Apply the same extraction to event messages: append `DocumentContent` blocks to the event `UserMessage`. +4. Message ordering: `ToolCallResultMessage` -> document `UserMessage` -> event `UserMessage`(s). + +**LangChain4J layer** (`ToolCallConverterImpl`): + +1. Serialize `ToolCallResult.content()` using the default connectors `ObjectMapper`, which serializes `Document` + instances as document references (store ID, document ID, metadata with content type and filename) via the standard + `DocumentSerializer`. +2. Remove the `ContentConverter` dependency; the custom `DocumentToContentModule` / `DocumentToContentSerializer` / + `DocumentToContentResponseModel` infrastructure is deleted. + +**Conversation history**: The `ToolCallResultMessage` retains `Document` objects in its content tree (serialized as +references when persisted). The follow-up `UserMessage` with `DocumentContent` blocks is a regular message in the +conversation. Both are visible and auditable. + +**User message format** (example with two tool calls): + +``` +TextContent: "Tool call 'generate_report' (call_1) documents:" +DocumentContent: report.pdf +DocumentContent: chart.png +TextContent: "Tool call 'fetch_data' (call_2) documents:" +DocumentContent: data.csv +``` + +### Future optimization (out of scope) + +A follow-up optimization can promote specific document types from the user message back into the +`ToolExecutionResultMessage` for providers that support native multi-content tool results (e.g., images on Anthropic). +This would be a post-processing step in the L4J framework adapter, transparent to the generic layer and conversation +history. + +## Consequences + +### Positive + +* Documents in tool call results become visible to the LLM across all providers. +* Reuses the existing, proven `DocumentToContentConverterImpl` path (same as user prompt documents). +* Conversation history shows the full picture: tool results with document references + user message with document + content. +* Removes ~3 classes of stale document-to-base64 serialization infrastructure. +* Text documents (JSON, XML, plain text) rendered as readable text; binary documents (PDF, images) rendered as native + multi-modal content blocks. + +### Negative + +* Adds one extra user message to the conversation when tool results contain documents, consuming additional tokens. +* The model must correlate document references in the tool result text with document content in the follow-up user + message. The text separators with tool call name and ID mitigate this. +* Slight increase in conversation history size due to the additional message. diff --git a/connectors/agentic-ai/docs/adr/003-document-handling-in-tool-call-results.plan.md b/connectors/agentic-ai/docs/adr/003-document-handling-in-tool-call-results.plan.md new file mode 100644 index 00000000000..80990146e12 --- /dev/null +++ b/connectors/agentic-ai/docs/adr/003-document-handling-in-tool-call-results.plan.md @@ -0,0 +1,168 @@ +# Implementation Plan: Document Handling in Tool Call Results + +Reference: [ADR-003](003-document-handling-in-tool-call-results.md) + +## Phase 1: New document extraction utility + +### 1.1 Create `ToolCallResultDocumentExtractor` + +**Package**: `io.camunda.connector.agenticai.aiagent.agent` + +**Public API**: + +```java +public class ToolCallResultDocumentExtractor { + + /** Groups of documents extracted from tool call results, preserving order. */ + record ToolCallDocuments(String toolCallId, String toolCallName, List documents) {} + + /** Extracts all Document instances from a list of tool call results, grouped by tool call. */ + List extractDocuments(List toolCallResults); + + /** Extracts all Document instances from an arbitrary object tree. */ + List extractDocuments(Object contentTree); +} +``` + +**Recursive walker**: Handles `Document` (collect), `Map` (recurse values), +`List` (recurse elements), everything else (skip). Supports root-level `Document` content. + +### 1.2 Create `ToolCallResultDocumentExtractorTest` + +- Root-level `Document` +- `Document` in map value +- `Document` in list element +- Deeply nested `Document` (map -> list -> map -> document) +- Mixed content (documents + scalars) +- No documents found -> empty list +- Null content -> empty list +- Grouping: multiple tool calls, some with documents, some without +- Tool calls without documents are excluded from results + +--- + +## Phase 2: Integrate extraction into `AgentMessagesHandlerImpl` + +### 2.1 Inject `ToolCallResultDocumentExtractor` as dependency + +Update constructor and the configuration/wiring that creates `AgentMessagesHandlerImpl`. + +### 2.2 Create document user message for tool call results + +In `addUserMessages()`, after `createToolCallResultMessage()` returns non-null: + +```java +if (toolCallResultMessage != null) { + messages.add(toolCallResultMessage); + var documentMessage = createDocumentMessageForToolResults(toolCallResultMessage.results()); + if (documentMessage != null) messages.add(documentMessage); + messages.addAll(eventMessages); +} +``` + +`createDocumentMessageForToolResults()`: +- Call `extractor.extractDocuments(results)` +- If no documents found, return null +- Build a single `UserMessage` with alternating `TextContent` separators + (`"Tool call '' () documents:"`) and `DocumentContent` blocks + +### 2.3 Extract documents from event messages + +In `createEventMessage()`, after building the main content block: + +- Call `extractor.extractDocuments(eventContent)` +- Append `DocumentContent.documentContent(doc)` for each extracted document + +### 2.4 Update `AgentMessagesHandlerTest` + +New test cases: +- Tool call results containing documents -> document `UserMessage` created after `ToolCallResultMessage`, before events +- Multiple tool calls with documents -> single `UserMessage` with text separators per tool call +- Tool call results without documents -> no extra message +- Event message with documents -> `DocumentContent` blocks appended to event `UserMessage` +- Mixed: tool results with documents + event messages -> correct ordering + +Update existing test setup to inject the new `ToolCallResultDocumentExtractor` dependency. + +--- + +## Phase 3: Simplify L4J tool result serialization + +### 3.1 Modify `ToolCallConverterImpl` + +- Remove `ContentConverter` dependency from constructor; keep only `ObjectMapper` +- Replace `contentConverter.convertToString(result)` with inline logic: + ```java + private String contentAsString(String toolName, Object result) { + if (result == null) return null; + if (result instanceof String s) return s; + return objectMapper.writeValueAsString(result); + } + ``` +- The injected `ObjectMapper` is the `@ConnectorsObjectMapper` which has `DocumentSerializer` registered, + so `Document` instances serialize as document references. + +### 3.2 Update `ToolCallConverterTest` + +- Update constructor: remove `ContentConverterImpl` parameter +- `supportsResultsContainingCamundaDocuments`: assert document reference format instead of + `DocumentToContentResponseModel` format (base64/text content blocks) + +### 3.3 Simplify `ContentConverterImpl` + +- Remove `contentObjectMapper` field (the ObjectMapper copy with `DocumentToContentModule`) +- `convertToString()` uses the injected `objectMapper` directly +- Constructor simplifies (still receives `DocumentToContentConverter` for `convertToContent()`) + +### 3.4 Update `ContentConverterTest` + +- `supportsObjectContentContainingCamundaDocuments`: assert document reference format + +### 3.5 Update `AgenticAiLangchain4JFrameworkConfiguration` + +- `langchain4JToolCallConverter` bean: remove `ContentConverter` parameter, pass only `ObjectMapper` + +--- + +## Phase 4: Delete stale infrastructure + +### 4.1 Delete classes + +- `DocumentToContentSerializer.java` +- `DocumentToContentModule.java` +- `DocumentToContentResponseModel.java` + +### 4.2 Delete tests + +- `DocumentToContentSerializerTest.java` + +### 4.3 Remove unused imports + +Clean up any remaining references to deleted classes across the codebase. + +--- + +## Phase 5: Update E2E tests + +### 5.1 Update `BaseL4JAiAgentJobWorkerTest` and `BaseL4JAiAgentConnectorTest` + +- Change `DownloadFileToolResult` record: replace `DocumentToContentResponseModel document` field + with the document reference format (or a generic `Object` that matches the serialized reference) +- Remove `DocumentToContentResponseModel` import + +### 5.2 Update `L4JAiAgentJobWorkerToolCallingTests` and `L4JAiAgentConnectorToolCallingTests` + +Update `supportsDocumentResponsesFromToolCalls`: +- Expected `ToolExecutionResultMessage` text: document reference format instead of `DocumentToContentResponseModel` +- Expected conversation includes a new `UserMessage` after the `ToolExecutionResultMessage` containing: + - `TextContent` separator with tool call name and ID + - Document content block (`TextContent` for text types, `ImageContent` for images, `PdfFileContent` for PDF) +- Adjust `assertLastChatRequest` expected conversation list and chat request count if needed + +--- + +## Verification + +1. `mvn clean install -pl connectors/agentic-ai` -- unit tests pass +2. Run e2e tests manually (per CLAUDE.md: never run e2e tests automatically) +3. Verify conversation history serialization round-trips correctly (existing tests should cover this) diff --git a/connectors/agentic-ai/pom.xml b/connectors/agentic-ai/pom.xml index 5ec4bdbc365..95b0bd65c93 100644 --- a/connectors/agentic-ai/pom.xml +++ b/connectors/agentic-ai/pom.xml @@ -292,6 +292,11 @@ + + io.camunda.connector + jackson-datatype-document + test + com.google.guava guava diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java index 35c82b6d7f7..f4977ad92a1 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java @@ -33,6 +33,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -47,6 +48,11 @@ public class AgentMessagesHandlerImpl implements AgentMessagesHandler { private static final Logger LOGGER = LoggerFactory.getLogger(AgentMessagesHandlerImpl.class); + /** + * Metadata key identifying a user message as containing documents extracted from tool results. + */ + static final String METADATA_TOOL_CALL_DOCUMENTS = "toolCallDocuments"; + private static final String EVENT_CONTENT_EMPTY = "An event was triggered but no content was returned."; private static final String EVENT_CONTENT_EMPTY_INTERRUPT_TOOL_CALLS_EMPTY_MESSAGE = @@ -57,11 +63,15 @@ public class AgentMessagesHandlerImpl implements AgentMessagesHandler { private final GatewayToolHandlerRegistry gatewayToolHandlers; private final SystemPromptComposer systemPromptComposer; + private final ToolCallResultDocumentExtractor documentExtractor; public AgentMessagesHandlerImpl( - GatewayToolHandlerRegistry gatewayToolHandlers, SystemPromptComposer systemPromptComposer) { + GatewayToolHandlerRegistry gatewayToolHandlers, + SystemPromptComposer systemPromptComposer, + ToolCallResultDocumentExtractor documentExtractor) { this.gatewayToolHandlers = gatewayToolHandlers; this.systemPromptComposer = systemPromptComposer; + this.documentExtractor = documentExtractor; } @Override @@ -125,6 +135,10 @@ public List addUserMessages( // if message is null, we wait on further tool call results to be added if (toolCallResultMessage != null) { messages.add(toolCallResultMessage); + var documentMessage = createDocumentMessageForToolResults(toolCallResultMessage.results()); + if (documentMessage != null) { + messages.add(documentMessage); + } messages.addAll(eventMessages); } } else { @@ -205,6 +219,27 @@ private ToolCallResultMessage createToolCallResultMessage( .build(); } + private UserMessage createDocumentMessageForToolResults(List results) { + final var toolCallDocuments = documentExtractor.extractDocuments(results); + if (toolCallDocuments.isEmpty()) { + return null; + } + + final var content = new ArrayList(); + for (var entry : toolCallDocuments) { + content.add( + textContent( + "Tool call '%s' (%s) documents:" + .formatted(entry.toolCallName(), entry.toolCallId()))); + entry.documents().stream().map(DocumentContent::documentContent).forEach(content::add); + } + + final var metadata = new HashMap(defaultMessageMetadata()); + metadata.put(METADATA_TOOL_CALL_DOCUMENTS, true); + + return UserMessage.builder().content(content).metadata(metadata).build(); + } + private Message createEventMessage( ToolCallResult eventResult, boolean interruptToolCallsOnEventResults) { Object eventContent = eventResult.content(); @@ -224,6 +259,11 @@ private Message createEventMessage( }); } + // extract documents from event content and add as document content blocks + documentExtractor + .extractDocuments(eventContent) + .forEach(doc -> userMessageContent.add(DocumentContent.documentContent(doc))); + return UserMessage.builder() .content(userMessageContent) .metadata(defaultMessageMetadata()) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java new file mode 100644 index 00000000000..3e35c9e8a5b --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java @@ -0,0 +1,77 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.agent; + +import io.camunda.connector.agenticai.model.tool.ToolCallResult; +import io.camunda.connector.api.document.Document; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; + +/** + * Extracts {@link Document} instances from tool call result content trees. Documents can appear at + * any level: as the root content, within maps, or within lists. + */ +public class ToolCallResultDocumentExtractor { + + /** Documents extracted from a single tool call result, grouped with the tool call identity. */ + public record ToolCallDocuments( + String toolCallId, String toolCallName, List documents) {} + + /** + * Extracts all {@link Document} instances from the given tool call results, grouped by tool call. + * Tool calls without documents are excluded from the result. Order is preserved. + */ + public List extractDocuments(List toolCallResults) { + final var result = new ArrayList(); + + for (ToolCallResult toolCallResult : toolCallResults) { + final var documents = extractDocuments(toolCallResult.content()); + if (!documents.isEmpty()) { + result.add( + new ToolCallDocuments( + StringUtils.defaultString(toolCallResult.id()), + StringUtils.defaultIfBlank(toolCallResult.name(), "unknown"), + documents)); + } + } + + return result; + } + + /** + * Recursively extracts all {@link Document} instances from an arbitrary object tree. Handles + * {@link Document}, {@link Map}, {@link List}/{@link Collection}, and skips all other types. + */ + public List extractDocuments(Object content) { + if (content == null) { + return List.of(); + } + + final var documents = new ArrayList(); + collectDocuments(content, documents); + return documents; + } + + private void collectDocuments(Object node, List documents) { + if (node == null) { + return; + } + + switch (node) { + case Document document -> documents.add(document); + case Map map -> map.values().forEach(value -> collectDocuments(value, documents)); + case Collection collection -> + collection.forEach(element -> collectDocuments(element, documents)); + default -> { + // scalars and other types - nothing to extract + } + } + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ContentConverterImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ContentConverterImpl.java index 12854642afd..38a0cc3dbe1 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ContentConverterImpl.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ContentConverterImpl.java @@ -9,7 +9,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.document.DocumentToContentConverter; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.document.DocumentToContentModule; import io.camunda.connector.agenticai.model.message.content.Content; import io.camunda.connector.agenticai.model.message.content.DocumentContent; import io.camunda.connector.agenticai.model.message.content.ObjectContent; @@ -17,13 +16,12 @@ public class ContentConverterImpl implements ContentConverter { private final DocumentToContentConverter documentToContentConverter; - private final ObjectMapper contentObjectMapper; + private final ObjectMapper objectMapper; public ContentConverterImpl( ObjectMapper objectMapper, DocumentToContentConverter documentToContentConverter) { this.documentToContentConverter = documentToContentConverter; - this.contentObjectMapper = - objectMapper.copy().registerModule(new DocumentToContentModule(documentToContentConverter)); + this.objectMapper = objectMapper; } @Override @@ -49,6 +47,6 @@ public String convertToString(Object content) throws JsonProcessingException { return stringContent; } - return contentObjectMapper.writeValueAsString(content); + return objectMapper.writeValueAsString(content); } } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java index 4dc0d605b32..27924b197f4 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java @@ -52,8 +52,8 @@ public ContentConverter langchain4JContentConverter( @Bean @ConditionalOnMissingBean public ToolCallConverter langchain4JToolCallConverter( - @ConnectorsObjectMapper ObjectMapper objectMapper, ContentConverter contentConverter) { - return new ToolCallConverterImpl(objectMapper, contentConverter); + @ConnectorsObjectMapper ObjectMapper objectMapper) { + return new ToolCallConverterImpl(objectMapper); } @Bean diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/document/DocumentToContentModule.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/document/DocumentToContentModule.java deleted file mode 100644 index 32f66f714f1..00000000000 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/document/DocumentToContentModule.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH - * under one or more contributor license agreements. Licensed under a proprietary license. - * See the License.txt file for more information. You may not use this file - * except in compliance with the proprietary license. - */ -package io.camunda.connector.agenticai.aiagent.framework.langchain4j.document; - -import com.fasterxml.jackson.databind.module.SimpleModule; -import io.camunda.connector.api.document.Document; - -public class DocumentToContentModule extends SimpleModule { - - private final DocumentToContentConverter documentConverter; - - public DocumentToContentModule(DocumentToContentConverter documentConverter) { - super(); - this.documentConverter = documentConverter; - } - - @Override - public void setupModule(SetupContext context) { - addSerializer(Document.class, new DocumentToContentSerializer(documentConverter)); - super.setupModule(context); - } -} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/document/DocumentToContentResponseModel.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/document/DocumentToContentResponseModel.java deleted file mode 100644 index e561ebf80d4..00000000000 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/document/DocumentToContentResponseModel.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH - * under one or more contributor license agreements. Licensed under a proprietary license. - * See the License.txt file for more information. You may not use this file - * except in compliance with the proprietary license. - */ -package io.camunda.connector.agenticai.aiagent.framework.langchain4j.document; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.PropertyNamingStrategies; -import com.fasterxml.jackson.databind.annotation.JsonNaming; - -/** - * Adheres to the Claude schema for content blocks in user messages. We might need to revisit the - * structure depending on the supported models. - */ -@JsonInclude(JsonInclude.Include.NON_EMPTY) -@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) -public record DocumentToContentResponseModel(String type, String mediaType, String data) {} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/document/DocumentToContentSerializer.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/document/DocumentToContentSerializer.java deleted file mode 100644 index b8c709d7144..00000000000 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/document/DocumentToContentSerializer.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH - * under one or more contributor license agreements. Licensed under a proprietary license. - * See the License.txt file for more information. You may not use this file - * except in compliance with the proprietary license. - */ -package io.camunda.connector.agenticai.aiagent.framework.langchain4j.document; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import dev.langchain4j.data.message.Content; -import dev.langchain4j.data.message.ImageContent; -import dev.langchain4j.data.message.PdfFileContent; -import dev.langchain4j.data.message.TextContent; -import io.camunda.connector.api.document.Document; -import io.camunda.connector.api.document.DocumentMetadata; -import io.camunda.connector.api.document.DocumentReference.CamundaDocumentReference; -import java.io.IOException; - -/** - * Serializes a {@link Document} to a JSON structure containing the document content. The structure - * is similar to what Claude is using for document support. Depending on the content type, the - * document content is included as plain text or as base64-encoded data. - * - * 
- * {
- *   "type": "text",
- *   "media_type": "text/plain",
- *   "data": "..."
- * }
- *
- * - *

or - * - * 

- * {
- *   "type": "base64",
- *   "media_type": "application/pdf",
- *   "data": "...base64..."
- * }
- *
- * - *

Note: To make the behavior consistent, this serializer first converts the document to a {@link - * Content} block, so the supported formats are limited to the same set as the documents which can - * be provided as the user prompt. - */ -public class DocumentToContentSerializer extends JsonSerializer { - - private static final String TYPE_TEXT = "text"; - private static final String TYPE_BASE64 = "base64"; - - private final DocumentToContentConverter contentConverter; - - public DocumentToContentSerializer(DocumentToContentConverter contentConverter) { - this.contentConverter = contentConverter; - } - - @Override - public void serialize( - Document document, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) - throws IOException { - - final var reference = document.reference(); - if (!(reference instanceof CamundaDocumentReference camundaReference)) { - throw new IllegalArgumentException("Unsupported document reference type: " + reference); - } - - final var contentBlock = contentConverter.convert(document); - final var response = - convertContentBlock(camundaReference, camundaReference.getMetadata(), contentBlock); - - jsonGenerator.writeObject(response); - } - - private DocumentToContentResponseModel convertContentBlock( - CamundaDocumentReference reference, DocumentMetadata metadata, Content contentBlock) { - - return switch (contentBlock) { - case TextContent tc -> - new DocumentToContentResponseModel(TYPE_TEXT, metadata.getContentType(), tc.text()); - - case PdfFileContent tf -> - new DocumentToContentResponseModel( - TYPE_BASE64, metadata.getContentType(), tf.pdfFile().base64Data()); - - case ImageContent tf -> - new DocumentToContentResponseModel( - TYPE_BASE64, metadata.getContentType(), tf.image().base64Data()); - - default -> - throw new IllegalArgumentException( - "Unsupported content block type '%s' for document with reference '%s'" - .formatted(contentBlock.getClass().getSimpleName(), reference.toString())); - }; - } -} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/tool/ToolCallConverterImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/tool/ToolCallConverterImpl.java index 1761ddc0ec4..566239953d0 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/tool/ToolCallConverterImpl.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/tool/ToolCallConverterImpl.java @@ -14,7 +14,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.langchain4j.agent.tool.ToolExecutionRequest; import dev.langchain4j.data.message.ToolExecutionResultMessage; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ContentConverter; import io.camunda.connector.agenticai.model.tool.ToolCall; import io.camunda.connector.agenticai.model.tool.ToolCallResult; import io.camunda.connector.api.error.ConnectorException; @@ -27,11 +26,9 @@ public class ToolCallConverterImpl implements ToolCallConverter { private final ObjectMapper objectMapper; - private final ContentConverter contentConverter; - public ToolCallConverterImpl(ObjectMapper objectMapper, ContentConverter contentConverter) { + public ToolCallConverterImpl(ObjectMapper objectMapper) { this.objectMapper = objectMapper; - this.contentConverter = contentConverter; } @Override @@ -78,8 +75,8 @@ private ToolCall asToolCall(String id, String name, String inputJson) { /** * Converts the result of a tool call to a {@link ToolExecutionResultMessage}. * - *

If the result is not a string, it will be serialized to a JSON string, using the {@link - * ContentConverter} to serialize document contents. + *

If the result is not a string, it will be serialized to a JSON string using the connectors + * ObjectMapper. Document instances in the content tree are serialized as document references. */ @Override public ToolExecutionResultMessage asToolExecutionResultMessage(ToolCallResult toolCallResult) { @@ -96,7 +93,13 @@ public ToolExecutionResultMessage asToolExecutionResultMessage(ToolCallResult to private String contentAsString(String toolName, Object result) { try { - return contentConverter.convertToString(result); + if (result == null) { + return null; + } + if (result instanceof String s) { + return s; + } + return objectMapper.writeValueAsString(result); } catch (JsonProcessingException e) { throw new ConnectorException( "Failed to convert result of tool call '%s' to string: %s" diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java index 06f5781dd30..1d15cc8c203 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java @@ -38,6 +38,7 @@ import io.camunda.connector.agenticai.aiagent.agent.AgentToolsResolverImpl; import io.camunda.connector.agenticai.aiagent.agent.JobWorkerAgentRequestHandler; import io.camunda.connector.agenticai.aiagent.agent.OutboundConnectorAgentRequestHandler; +import io.camunda.connector.agenticai.aiagent.agent.ToolCallResultDocumentExtractor; import io.camunda.connector.agenticai.aiagent.framework.AiFrameworkAdapter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelHttpProxySupport; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.configuration.AgenticAiLangchain4JFrameworkConfiguration; @@ -230,11 +231,20 @@ public SystemPromptComposer aiAgentSystemPromptComposer( return new SystemPromptComposerImpl(contributors); } + @Bean + @ConditionalOnMissingBean + public ToolCallResultDocumentExtractor toolCallResultDocumentExtractor() { + return new ToolCallResultDocumentExtractor(); + } + @Bean @ConditionalOnMissingBean public AgentMessagesHandler aiAgentMessagesHandler( - GatewayToolHandlerRegistry gatewayToolHandlers, SystemPromptComposer systemPromptComposer) { - return new AgentMessagesHandlerImpl(gatewayToolHandlers, systemPromptComposer); + GatewayToolHandlerRegistry gatewayToolHandlers, + SystemPromptComposer systemPromptComposer, + ToolCallResultDocumentExtractor documentExtractor) { + return new AgentMessagesHandlerImpl( + gatewayToolHandlers, systemPromptComposer, documentExtractor); } @Bean diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java index 8869379b933..c09c8d3c730 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java @@ -86,7 +86,9 @@ class AgentMessagesHandlerTest { @BeforeEach void setUp() { systemPromptComposer = new SystemPromptComposerImpl(List.of()); - messagesHandler = new AgentMessagesHandlerImpl(gatewayToolHandlers, systemPromptComposer); + messagesHandler = + new AgentMessagesHandlerImpl( + gatewayToolHandlers, systemPromptComposer, new ToolCallResultDocumentExtractor()); runtimeMemory = spy(new DefaultRuntimeMemory()); } @@ -752,6 +754,192 @@ void interruptsToolCallsOnEventResultsWhenEventContentIsEmpty(Object eventConten EVENT_INTERRUPT_TOOL_CALLS_EMPTY_MESSAGE))))); } + @Test + void createsDocumentUserMessageWhenToolResultsContainDocuments() { + final var doc1 = mock(Document.class); + final var doc2 = mock(Document.class); + + final var toolCallResultsWithDocs = + List.of( + ToolCallResult.builder() + .id("abcdef") + .name("getWeather") + .content(Map.of("result", "Sunny", "attachment", doc1)) + .build(), + ToolCallResult.builder() + .id("fedcba") + .name("getDateTime") + .content(Map.of("report", doc2)) + .build()); + + final var assistantMessage = + assistantMessage("Assistant message with tool calls", TOOL_CALLS); + runtimeMemory.addMessage(assistantMessage); + + when(gatewayToolHandlers.transformToolCallResults(AGENT_CONTEXT, toolCallResultsWithDocs)) + .thenReturn(toolCallResultsWithDocs); + + final var addedMessages = + messagesHandler.addUserMessages( + executionContext, + AGENT_CONTEXT, + runtimeMemory, + userPromptWithDocuments, + toolCallResultsWithDocs); + + assertThat(addedMessages) + .hasSize(2) + .satisfiesExactly( + message -> assertThat(message).isInstanceOf(ToolCallResultMessage.class), + message -> + assertThat(message) + .isInstanceOfSatisfying( + UserMessage.class, + userMessage -> { + assertThat(userMessage.metadata()) + .containsEntry("toolCallDocuments", true) + .containsKey("timestamp"); + assertThat(userMessage.content()) + .hasSize(4) + .satisfiesExactly( + c -> + assertThat(c) + .isEqualTo( + textContent( + "Tool call 'getWeather' (abcdef) documents:")), + c -> + assertThat(c) + .isEqualTo(DocumentContent.documentContent(doc1)), + c -> + assertThat(c) + .isEqualTo( + textContent( + "Tool call 'getDateTime' (fedcba) documents:")), + c -> + assertThat(c) + .isEqualTo(DocumentContent.documentContent(doc2))); + })); + } + + @Test + void doesNotCreateDocumentUserMessageWhenNoDocumentsInToolResults() { + final var assistantMessage = + assistantMessage("Assistant message with tool calls", TOOL_CALLS); + runtimeMemory.addMessage(assistantMessage); + + when(gatewayToolHandlers.transformToolCallResults(AGENT_CONTEXT, TOOL_CALL_RESULTS)) + .thenReturn(TOOL_CALL_RESULTS.stream().toList()); + + final var addedMessages = + messagesHandler.addUserMessages( + executionContext, + AGENT_CONTEXT, + runtimeMemory, + userPromptWithDocuments, + TOOL_CALL_RESULTS); + + // no document user message - only the tool call result message + assertThat(addedMessages).hasSize(1).first().isInstanceOf(ToolCallResultMessage.class); + } + + @Test + void ordersDocumentUserMessageBetweenToolResultsAndEvents() { + when(executionContext.events()) + .thenReturn(new EventHandlingConfiguration(WAIT_FOR_TOOL_CALL_RESULTS)); + + final var doc = mock(Document.class); + final var toolCallResultsWithDocsAndEvents = + List.of( + ToolCallResult.builder() + .id("abcdef") + .name("getWeather") + .content(Map.of("file", doc)) + .build(), + ToolCallResult.builder().id("fedcba").name("getDateTime").content("15:00").build(), + EVENT_TOOL_CALL_RESULTS.get(0)); + + final var assistantMessage = + assistantMessage("Assistant message with tool calls", TOOL_CALLS); + runtimeMemory.addMessage(assistantMessage); + + when(gatewayToolHandlers.transformToolCallResults(eq(AGENT_CONTEXT), anyList())) + .thenAnswer(invocationOnMock -> invocationOnMock.getArgument(1)); + + final var addedMessages = + messagesHandler.addUserMessages( + executionContext, + AGENT_CONTEXT, + runtimeMemory, + userPromptWithDocuments, + toolCallResultsWithDocsAndEvents); + + // order: ToolCallResultMessage -> document UserMessage -> event UserMessage + assertThat(addedMessages) + .hasSize(3) + .satisfiesExactly( + message -> assertThat(message).isInstanceOf(ToolCallResultMessage.class), + message -> + assertThat(message) + .isInstanceOfSatisfying( + UserMessage.class, + um -> + assertThat(um.content()) + .hasSize(2) + .satisfiesExactly( + c -> + assertThat(c) + .isEqualTo( + textContent( + "Tool call 'getWeather' (abcdef) documents:")), + c -> + assertThat(c) + .isEqualTo(DocumentContent.documentContent(doc)))), + message -> + assertThat(message) + .isInstanceOfSatisfying( + UserMessage.class, + um -> + assertThat(um.content()) + .first() + .isEqualTo(textContent("Event data")))); + } + + @Test + void appendsDocumentsToEventMessage() { + final var doc = mock(Document.class); + final var eventWithDoc = + ToolCallResult.builder().content(Map.of("text", "event", "file", doc)).build(); + + final var assistantMessage = + assistantMessage("Assistant message with tool calls", TOOL_CALLS); + runtimeMemory.addMessage(assistantMessage); + + when(gatewayToolHandlers.transformToolCallResults(eq(AGENT_CONTEXT), anyList())) + .thenReturn(TOOL_CALL_RESULTS.stream().toList()); + + final var addedMessages = + messagesHandler.addUserMessages( + executionContext, + AGENT_CONTEXT, + runtimeMemory, + userPromptWithDocuments, + List.of(TOOL_CALL_RESULTS.get(0), TOOL_CALL_RESULTS.get(1), eventWithDoc)); + + // find the event message (last one) + final var eventMessage = addedMessages.getLast(); + assertThat(eventMessage) + .isInstanceOfSatisfying( + UserMessage.class, + um -> + assertThat(um.content()) + .hasSize(2) + .satisfiesExactly( + c -> + assertThat(c) + .isEqualTo(objectContent(Map.of("text", "event", "file", doc))), + c -> assertThat(c).isEqualTo(DocumentContent.documentContent(doc)))); + } + static List toolCallResultsWithEventsAndEventBehavior() { final List arguments = new ArrayList<>(); toolCallResultsWithEvents() diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractorTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractorTest.java new file mode 100644 index 00000000000..b86ccadfe19 --- /dev/null +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractorTest.java @@ -0,0 +1,223 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.agent; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.camunda.connector.agenticai.model.tool.ToolCallResult; +import io.camunda.connector.api.document.Document; +import io.camunda.connector.api.document.DocumentCreationRequest; +import io.camunda.connector.api.document.DocumentFactory; +import io.camunda.connector.runtime.core.document.DocumentFactoryImpl; +import io.camunda.connector.runtime.core.document.store.InMemoryDocumentStore; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +class ToolCallResultDocumentExtractorTest { + + private final InMemoryDocumentStore documentStore = InMemoryDocumentStore.INSTANCE; + private final DocumentFactory documentFactory = new DocumentFactoryImpl(documentStore); + private final ToolCallResultDocumentExtractor extractor = new ToolCallResultDocumentExtractor(); + + @BeforeEach + void setUp() { + documentStore.clear(); + } + + @Nested + class ExtractFromContentTree { + + @Test + void extractsRootLevelDocument() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + final var result = extractor.extractDocuments((Object) doc); + assertThat(result).containsExactly(doc); + } + + @Test + void extractsDocumentFromMapValue() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + final var result = extractor.extractDocuments((Object) Map.of("file", doc, "key", "value")); + assertThat(result).containsExactly(doc); + } + + @Test + void extractsDocumentFromList() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + final var result = extractor.extractDocuments((Object) List.of("text", doc, 42)); + assertThat(result).containsExactly(doc); + } + + @Test + void extractsDeeplyNestedDocuments() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + final var nested = Map.of("level1", Map.of("level2", List.of(Map.of("file", doc)))); + final var result = extractor.extractDocuments((Object) nested); + assertThat(result).containsExactly(doc); + } + + @Test + void extractsMultipleDocuments() { + final var doc1 = createDocument("hello", "text/plain", "test.txt"); + final var doc2 = createDocument("", "application/pdf", "report.pdf"); + final var content = new LinkedHashMap(); + content.put("text", doc1); + content.put("report", doc2); + content.put("other", "value"); + + final var result = extractor.extractDocuments((Object) content); + assertThat(result).containsExactly(doc1, doc2); + } + + @Test + void returnsEmptyForContentWithoutDocuments() { + final var result = + extractor.extractDocuments((Object) Map.of("key", "value", "list", List.of(1, 2, 3))); + assertThat(result).isEmpty(); + } + + @Test + void returnsEmptyForNullContent() { + final var result = extractor.extractDocuments((Object) null); + assertThat(result).isEmpty(); + } + + @Test + void returnsEmptyForScalarContent() { + assertThat(extractor.extractDocuments((Object) "text")).isEmpty(); + assertThat(extractor.extractDocuments((Object) 42)).isEmpty(); + assertThat(extractor.extractDocuments((Object) true)).isEmpty(); + } + + @Test + void handlesNullValuesInMap() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + final var content = new LinkedHashMap(); + content.put("file", doc); + content.put("missing", null); + + final var result = extractor.extractDocuments((Object) content); + assertThat(result).containsExactly(doc); + } + + @Test + void handlesNullElementsInList() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + final var content = new ArrayList<>(); + content.add(doc); + content.add(null); + content.add("text"); + + final var result = extractor.extractDocuments((Object) content); + assertThat(result).containsExactly(doc); + } + } + + @Nested + class ExtractFromToolCallResults { + + @Test + void groupsDocumentsByToolCall() { + final var doc1 = createDocument("hello", "text/plain", "test.txt"); + final var doc2 = createDocument("", "application/pdf", "report.pdf"); + + final var results = + List.of( + ToolCallResult.builder() + .id("call_1") + .name("tool_a") + .content(Map.of("file", doc1)) + .build(), + ToolCallResult.builder() + .id("call_2") + .name("tool_b") + .content(Map.of("report", doc2)) + .build()); + + final var extracted = extractor.extractDocuments(results); + + assertThat(extracted).hasSize(2); + assertThat(extracted.get(0)) + .satisfies( + e -> { + assertThat(e.toolCallId()).isEqualTo("call_1"); + assertThat(e.toolCallName()).isEqualTo("tool_a"); + assertThat(e.documents()).containsExactly(doc1); + }); + assertThat(extracted.get(1)) + .satisfies( + e -> { + assertThat(e.toolCallId()).isEqualTo("call_2"); + assertThat(e.toolCallName()).isEqualTo("tool_b"); + assertThat(e.documents()).containsExactly(doc2); + }); + } + + @Test + void excludesToolCallsWithoutDocuments() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + + final var results = + List.of( + ToolCallResult.builder() + .id("call_1") + .name("tool_a") + .content(Map.of("file", doc)) + .build(), + ToolCallResult.builder() + .id("call_2") + .name("tool_b") + .content("plain text result") + .build()); + + final var extracted = extractor.extractDocuments(results); + + assertThat(extracted).hasSize(1); + assertThat(extracted.getFirst().toolCallId()).isEqualTo("call_1"); + } + + @Test + void returnsEmptyWhenNoToolCallsContainDocuments() { + final var results = + List.of( + ToolCallResult.builder().id("call_1").name("tool_a").content("text result").build()); + + assertThat(extractor.extractDocuments(results)).isEmpty(); + } + + @Test + void handlesNullNameAndId() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + + final var results = List.of(ToolCallResult.builder().content(Map.of("file", doc)).build()); + + final var extracted = extractor.extractDocuments(results); + + assertThat(extracted).hasSize(1); + assertThat(extracted.getFirst()) + .satisfies( + e -> { + assertThat(e.toolCallId()).isEmpty(); + assertThat(e.toolCallName()).isEqualTo("unknown"); + }); + } + } + + private Document createDocument(String content, String contentType, String filename) { + return documentFactory.create( + DocumentCreationRequest.from(content.getBytes(StandardCharsets.UTF_8)) + .contentType(contentType) + .fileName(filename) + .build()); + } +} diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ContentConverterTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ContentConverterTest.java index d61889fd5d9..722af400699 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ContentConverterTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ContentConverterTest.java @@ -18,6 +18,7 @@ import io.camunda.connector.api.document.Document; import io.camunda.connector.api.document.DocumentCreationRequest; import io.camunda.connector.api.document.DocumentFactory; +import io.camunda.connector.document.jackson.JacksonModuleDocumentSerializer; import io.camunda.connector.runtime.core.document.DocumentFactoryImpl; import io.camunda.connector.runtime.core.document.store.InMemoryDocumentStore; import java.nio.charset.StandardCharsets; @@ -35,8 +36,10 @@ class ContentConverterTest { private final InMemoryDocumentStore documentStore = InMemoryDocumentStore.INSTANCE; private final DocumentFactory documentFactory = new DocumentFactoryImpl(documentStore); + private final ObjectMapper objectMapper = + new ObjectMapper().registerModule(new JacksonModuleDocumentSerializer()); private final ContentConverter contentConverter = - new ContentConverterImpl(new ObjectMapper(), new DocumentToContentConverterImpl()); + new ContentConverterImpl(objectMapper, new DocumentToContentConverterImpl()); @BeforeEach void setUp() { @@ -120,7 +123,7 @@ void supportsObjectContent() throws JsonProcessingException, JSONException { } @Test - void supportsObjectContentContainingCamundaDocuments() + void serializesDocumentsAsReferencesInObjectContent() throws JsonProcessingException, JSONException { final var content = new LinkedHashMap(); content.put("hello", "world"); @@ -128,24 +131,30 @@ void supportsObjectContentContainingCamundaDocuments() content.put("document2", createDocument("", "application/pdf", "test.pdf")); final var stringResult = contentConverter.convertToString(content); + + // documents are serialized as document references (lenient: ignores dynamic IDs) JSONAssert.assertEquals( """ { "hello": "world", "document1": { - "type": "text", - "media_type": "text/plain", - "data": "Hello, world!" + "camunda.document.type": "camunda", + "metadata": { + "contentType": "text/plain", + "fileName": "test.txt" + } }, "document2": { - "type": "base64", - "media_type": "application/pdf", - "data": "PFBERiBDT05URU5UPg==" + "camunda.document.type": "camunda", + "metadata": { + "contentType": "application/pdf", + "fileName": "test.pdf" + } } } """, stringResult, - true); + false); } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/document/DocumentToContentSerializerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/document/DocumentToContentSerializerTest.java deleted file mode 100644 index 68e3826faa8..00000000000 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/document/DocumentToContentSerializerTest.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH - * under one or more contributor license agreements. Licensed under a proprietary license. - * See the License.txt file for more information. You may not use this file - * except in compliance with the proprietary license. - */ -package io.camunda.connector.agenticai.aiagent.framework.langchain4j.document; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.Mockito.when; - -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import dev.langchain4j.data.message.VideoContent; -import io.camunda.connector.api.document.Document; -import io.camunda.connector.api.document.DocumentCreationRequest; -import io.camunda.connector.api.document.DocumentFactory; -import io.camunda.connector.runtime.core.document.DocumentFactoryImpl; -import io.camunda.connector.runtime.core.document.store.InMemoryDocumentStore; -import java.nio.charset.StandardCharsets; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; -import org.mockito.Spy; -import org.mockito.junit.jupiter.MockitoExtension; -import org.skyscreamer.jsonassert.JSONAssert; - -@ExtendWith(MockitoExtension.class) -class DocumentToContentSerializerTest { - - @Spy - private final DocumentToContentConverter contentConverter = new DocumentToContentConverterImpl(); - - private final InMemoryDocumentStore documentStore = InMemoryDocumentStore.INSTANCE; - private final DocumentFactory documentFactory = new DocumentFactoryImpl(documentStore); - - private ObjectMapper objectMapper; - - @BeforeEach - void setUp() { - objectMapper = new ObjectMapper().registerModule(new DocumentToContentModule(contentConverter)); - - documentStore.clear(); - } - - @ParameterizedTest - @CsvSource({ - "test.txt,text/plain", - "test.csv,text/csv", - "test.json,application/json", - "test.xml,application/xml", - "test.yaml,application/yaml" - }) - void serializesTextTypesAsText(String filename, String contentType) throws Exception { - final var document = createDocument("Hello, world!", contentType, filename); - final var serializedDocument = objectMapper.writeValueAsString(document); - - JSONAssert.assertEquals( - """ - { - "type" : "text", - "media_type" : "%s", - "data" : "Hello, world!" - } - """ - .formatted(contentType), - serializedDocument, - true); - } - - @ParameterizedTest - @CsvSource({ - "test.gif,image/gif", - "test.jpg,image/jpeg", - "test.pdf,application/pdf", - "test.png,image/png", - "test.webp,image/webp" - }) - void serializesFileContentAsBase64(String filename, String contentType) throws Exception { - final var document = createDocument("Hello, world!", contentType, filename); - final var serializedDocument = objectMapper.writeValueAsString(document); - - JSONAssert.assertEquals( - """ - { - "type" : "base64", - "media_type" : "%s", - "data" : "SGVsbG8sIHdvcmxkIQ==" - } - """ - .formatted(contentType), - serializedDocument, - true); - } - - @Test - void supportsSerializingDocumentsInNestedStructures() throws Exception { - final var input = new LinkedHashMap(); - input.put("hello", "world"); - input.put("document1", createDocument("Hello, world!", "text/plain", "test.txt")); - input.put( - "documents", - List.of( - createDocument("", "application/pdf", "test.pdf"), - createDocument("", "image/png", "image.png"))); - input.put( - "map", - Map.of( - "some_json", createDocument("{\"foo\": \"bar\"}", "application/json", "test.json"), - "some_csv", createDocument("foo,bar", "text/csv", "test.csv"))); - - final var serialized = objectMapper.writeValueAsString(input); - - JSONAssert.assertEquals( - """ - { - "hello": "world", - "document1": { - "type": "text", - "media_type": "text/plain", - "data": "Hello, world!" - }, - "documents": [ - { - "type": "base64", - "media_type": "application/pdf", - "data": "PFBERiBDT05URU5UPg==" - }, - { - "type": "base64", - "media_type": "image/png", - "data": "PElNQUdFIENPTlRFTlQ+" - } - ], - "map": { - "some_csv": { - "type": "text", - "media_type": "text/csv", - "data": "foo,bar" - }, - "some_json": { - "type": "text", - "media_type": "application/json", - "data": "{\\"foo\\": \\"bar\\"}" - } - } - } - """, - serialized, - true); - } - - @Test - void throwsExceptionOnUnsupportedContentType() { - final var document = createDocument("Hello, world!", "text/plain", "test.txt"); - when(contentConverter.convert(document)) - .thenReturn(VideoContent.from("

{@code
+   * {"storeId":"in-memory","documentId":"25ece9fa-aeea-423d-98ed-67c1f08b137b",...}
+   * }
+ * + * This method extracts the first segment of the documentId UUID ("25ece9fa"), which is used as a + * compact correlation key in the document XML tags. */ static String extractDocumentShortId(String toolResultText) { var matcher = DOCUMENT_ID_PATTERN.matcher(toolResultText); diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/BaseL4JAiAgentJobWorkerTest.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/BaseL4JAiAgentJobWorkerTest.java index 0b4c58e0a8a..49a72d676a2 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/BaseL4JAiAgentJobWorkerTest.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/BaseL4JAiAgentJobWorkerTest.java @@ -45,6 +45,7 @@ import io.camunda.connector.e2e.agenticai.aiagent.BaseAiAgentJobWorkerTest; import io.camunda.connector.e2e.agenticai.assertj.JobWorkerAgentResponseAssert; import io.camunda.connector.e2e.agenticai.assertj.ToolExecutionRequestEqualsPredicate; +import io.camunda.connector.e2e.agenticai.assertj.ToolExecutionResultMessageEqualsPredicate; import io.camunda.connector.test.utils.annotation.SlowTest; import java.time.Duration; import java.util.ArrayList; @@ -316,6 +317,9 @@ protected void assertLastChatRequest( RecursiveComparisonConfiguration.builder() .withEqualsForType( new ToolExecutionRequestEqualsPredicate(), ToolExecutionRequest.class) + .withEqualsForType( + new ToolExecutionResultMessageEqualsPredicate(), + ToolExecutionResultMessage.class) .build()) .containsExactlyElementsOf( expectedConversation.subList(0, expectedConversation.size() - 1)); diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerMcpIntegrationTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerMcpIntegrationTests.java index aeb9e634b6c..52baabb1167 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerMcpIntegrationTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerMcpIntegrationTests.java @@ -395,20 +395,13 @@ void extractsDocumentsFromMcpImageToolCallResult() throws IOException { assertThat(chatRequestCaptor.getAllValues()).hasSize(2); final var lastMessages = chatRequestCaptor.getValue().messages(); - - // Expected message order: - // 0: SystemMessage - // 1: UserMessage (initial prompt) - // 2: AiMessage (tool call) - // 3: ToolExecutionResultMessage (MCP result with document reference) - // 4: UserMessage (document content extracted from MCP tool result) assertThat(lastMessages).hasSize(5); assertThat(lastMessages.get(0)).isInstanceOf(SystemMessage.class); - assertThat(lastMessages.get(1)).isInstanceOf(UserMessage.class); - assertThat(lastMessages.get(2)).isInstanceOf(AiMessage.class); + assertThat(lastMessages.get(1)).isInstanceOf(UserMessage.class); // initial prompt + assertThat(lastMessages.get(2)).isInstanceOf(AiMessage.class); // tool call - // Tool result: document serialized as document reference + // tool result: document serialized as document reference assertThat(lastMessages.get(3)) .isInstanceOfSatisfying( ToolExecutionResultMessage.class, @@ -418,11 +411,11 @@ void extractsDocumentsFromMcpImageToolCallResult() throws IOException { assertThat(msg.text()).contains("camunda.document.type"); }); - // Extract the document short ID from the tool result reference + // extract the document short ID (first UUID segment) from the serialized document reference var toolResultText = ((ToolExecutionResultMessage) lastMessages.get(3)).text(); var documentShortId = extractDocumentShortId(toolResultText); - // Document user message: extracted document content + // document user message: extracted document content assertThat(lastMessages.get(4)) .isInstanceOfSatisfying( UserMessage.class, @@ -441,15 +434,14 @@ void extractsDocumentsFromMcpImageToolCallResult() throws IOException { tc -> assertThat(tc.text()) .isEqualTo( - "")); + "" + .formatted(documentShortId))); assertThat(contents.get(2)) .isInstanceOfSatisfying( ImageContent.class, img -> { assertThat(img.image().mimeType()).isEqualTo("image/png"); - assertThat(img.image().base64Data()).isNotBlank(); + assertThat(img.image().base64Data()).isEqualTo(imageBase64); }); }); diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java index b0413f485e3..6486c965702 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java @@ -84,7 +84,7 @@ void supportsDocumentResponsesFromToolCalls( throws Exception { final var initialUserPrompt = "Go and download a document!"; - // The AI message with tool call and the subsequent responses for the conversation + // the AI message with tool call and the subsequent responses for the conversation final var aiToolCallMessage = new AiMessage( "The user asked me to download a document. I will call the Download_A_File tool to do so.", @@ -138,28 +138,19 @@ void supportsDocumentResponsesFromToolCalls( Map.of("userPrompt", initialUserPrompt)) .waitForProcessCompletion(); - // Assert the conversation structure with document extraction await() .alias("Chat request captured") .untilAsserted(() -> assertThat(chatRequestCaptor.getValue()).isNotNull()); + assertThat(chatRequestCaptor.getAllValues()).hasSize(3); final var lastMessages = chatRequestCaptor.getValue().messages(); - - // Expected message order (8 messages, last AI response not included in request): - // 0: SystemMessage - // 1: UserMessage (initial prompt) - // 2: AiMessage (tool call) - // 3: ToolExecutionResultMessage (document serialized as reference) - // 4: UserMessage (document content extracted from tool result) - // 5: AiMessage (response after tool) - // 6: UserMessage (follow-up question) assertThat(lastMessages).hasSize(7); assertThat(lastMessages.get(0)).isInstanceOf(SystemMessage.class); - assertThat(lastMessages.get(1)).isInstanceOf(UserMessage.class); - assertThat(lastMessages.get(2)).isInstanceOf(AiMessage.class); + assertThat(lastMessages.get(1)).isInstanceOf(UserMessage.class); // initial prompt + assertThat(lastMessages.get(2)).isInstanceOf(AiMessage.class); // tool call - // Tool result: document serialized as document reference + // tool result: document serialized as document reference assertThat(lastMessages.get(3)) .isInstanceOfSatisfying( ToolExecutionResultMessage.class, @@ -170,11 +161,12 @@ void supportsDocumentResponsesFromToolCalls( assertThat(msg.text()).contains(mimeType); }); - // Extract the document short ID from the tool result reference + // extract the document short ID (first UUID segment) from the serialized document reference + // e.g. from {"documentId":"25ece9fa-...", ...} -> "25ece9fa" var toolResultText = ((ToolExecutionResultMessage) lastMessages.get(3)).text(); var documentShortId = extractDocumentShortId(toolResultText); - // Document user message: extracted document content + // document user message: extracted document content assertThat(lastMessages.get(4)) .isInstanceOfSatisfying( UserMessage.class, @@ -193,13 +185,12 @@ void supportsDocumentResponsesFromToolCalls( tc -> assertThat(tc.text()) .isEqualTo( - "")); + "" + .formatted(documentShortId))); assertDocumentContentBlock(contents.get(2), type, mimeType); }); - assertThat(lastMessages.get(5)).isInstanceOf(AiMessage.class); + assertThat(lastMessages.get(5)).isInstanceOf(AiMessage.class); // response after tool assertThat(lastMessages.get(6)) .isInstanceOfSatisfying( diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/BaseL4JAiAgentConnectorTest.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/BaseL4JAiAgentConnectorTest.java index a6dd08aa8f8..2f314140ab0 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/BaseL4JAiAgentConnectorTest.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/BaseL4JAiAgentConnectorTest.java @@ -44,6 +44,7 @@ import io.camunda.connector.e2e.agenticai.aiagent.BaseAiAgentConnectorTest; import io.camunda.connector.e2e.agenticai.assertj.AgentResponseAssert; import io.camunda.connector.e2e.agenticai.assertj.ToolExecutionRequestEqualsPredicate; +import io.camunda.connector.e2e.agenticai.assertj.ToolExecutionResultMessageEqualsPredicate; import io.camunda.connector.test.utils.annotation.SlowTest; import java.util.ArrayList; import java.util.Arrays; @@ -313,6 +314,9 @@ protected void assertLastChatRequest( RecursiveComparisonConfiguration.builder() .withEqualsForType( new ToolExecutionRequestEqualsPredicate(), ToolExecutionRequest.class) + .withEqualsForType( + new ToolExecutionResultMessageEqualsPredicate(), + ToolExecutionResultMessage.class) .build()) .containsExactlyElementsOf( expectedConversation.subList(0, expectedConversation.size() - 1)); diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorMcpIntegrationTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorMcpIntegrationTests.java index 84f89eafbe8..03025efb70d 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorMcpIntegrationTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorMcpIntegrationTests.java @@ -397,20 +397,13 @@ void extractsDocumentsFromMcpImageToolCallResult() throws IOException { assertThat(chatRequestCaptor.getAllValues()).hasSize(2); final var lastMessages = chatRequestCaptor.getValue().messages(); - - // Expected message order: - // 0: SystemMessage - // 1: UserMessage (initial prompt) - // 2: AiMessage (tool call) - // 3: ToolExecutionResultMessage (MCP result with document reference) - // 4: UserMessage (document content extracted from MCP tool result) assertThat(lastMessages).hasSize(5); assertThat(lastMessages.get(0)).isInstanceOf(SystemMessage.class); - assertThat(lastMessages.get(1)).isInstanceOf(UserMessage.class); - assertThat(lastMessages.get(2)).isInstanceOf(AiMessage.class); + assertThat(lastMessages.get(1)).isInstanceOf(UserMessage.class); // initial prompt + assertThat(lastMessages.get(2)).isInstanceOf(AiMessage.class); // tool call - // Tool result: document serialized as document reference + // tool result: document serialized as document reference assertThat(lastMessages.get(3)) .isInstanceOfSatisfying( ToolExecutionResultMessage.class, @@ -420,11 +413,11 @@ void extractsDocumentsFromMcpImageToolCallResult() throws IOException { assertThat(msg.text()).contains("camunda.document.type"); }); - // Extract the document short ID from the tool result reference + // extract the document short ID (first UUID segment) from the serialized document reference var toolResultText = ((ToolExecutionResultMessage) lastMessages.get(3)).text(); var documentShortId = extractDocumentShortId(toolResultText); - // Document user message: extracted document content + // document user message: extracted document content assertThat(lastMessages.get(4)) .isInstanceOfSatisfying( UserMessage.class, @@ -443,15 +436,14 @@ void extractsDocumentsFromMcpImageToolCallResult() throws IOException { tc -> assertThat(tc.text()) .isEqualTo( - "")); + "" + .formatted(documentShortId))); assertThat(contents.get(2)) .isInstanceOfSatisfying( ImageContent.class, img -> { assertThat(img.image().mimeType()).isEqualTo("image/png"); - assertThat(img.image().base64Data()).isNotBlank(); + assertThat(img.image().base64Data()).isEqualTo(imageBase64); }); }); diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java index f0fffa2b4e2..2274a59c7bd 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java @@ -131,28 +131,19 @@ void supportsDocumentResponsesFromToolCalls( Map.of("userPrompt", initialUserPrompt)) .waitForProcessCompletion(); - // Assert the conversation structure with document extraction await() .alias("Chat request captured") .untilAsserted(() -> assertThat(chatRequestCaptor.getValue()).isNotNull()); + assertThat(chatRequestCaptor.getAllValues()).hasSize(3); final var lastMessages = chatRequestCaptor.getValue().messages(); - - // Expected message order (8 messages, last AI response not included in request): - // 0: SystemMessage - // 1: UserMessage (initial prompt) - // 2: AiMessage (tool call) - // 3: ToolExecutionResultMessage (document serialized as reference) - // 4: UserMessage (document content extracted from tool result) - // 5: AiMessage (response after tool) - // 6: UserMessage (follow-up question) assertThat(lastMessages).hasSize(7); assertThat(lastMessages.get(0)).isInstanceOf(SystemMessage.class); - assertThat(lastMessages.get(1)).isInstanceOf(UserMessage.class); - assertThat(lastMessages.get(2)).isInstanceOf(AiMessage.class); + assertThat(lastMessages.get(1)).isInstanceOf(UserMessage.class); // initial prompt + assertThat(lastMessages.get(2)).isInstanceOf(AiMessage.class); // tool call - // Tool result: document serialized as document reference + // tool result: document serialized as document reference assertThat(lastMessages.get(3)) .isInstanceOfSatisfying( ToolExecutionResultMessage.class, @@ -163,11 +154,12 @@ void supportsDocumentResponsesFromToolCalls( assertThat(msg.text()).contains(mimeType); }); - // Extract the document short ID from the tool result reference + // extract the document short ID (first UUID segment) from the serialized document reference + // e.g. from {"documentId":"25ece9fa-...", ...} -> "25ece9fa" var toolResultText = ((ToolExecutionResultMessage) lastMessages.get(3)).text(); var documentShortId = extractDocumentShortId(toolResultText); - // Document user message: extracted document content + // document user message: extracted document content assertThat(lastMessages.get(4)) .isInstanceOfSatisfying( UserMessage.class, @@ -186,13 +178,12 @@ void supportsDocumentResponsesFromToolCalls( tc -> assertThat(tc.text()) .isEqualTo( - "")); + "" + .formatted(documentShortId))); assertDocumentContentBlock(contents.get(2), type, mimeType); }); - assertThat(lastMessages.get(5)).isInstanceOf(AiMessage.class); + assertThat(lastMessages.get(5)).isInstanceOf(AiMessage.class); // response after tool assertThat(lastMessages.get(6)) .isInstanceOfSatisfying( diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/assertj/ToolExecutionResultMessageEqualsPredicate.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/assertj/ToolExecutionResultMessageEqualsPredicate.java new file mode 100644 index 00000000000..b4e8ad0ce53 --- /dev/null +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/assertj/ToolExecutionResultMessageEqualsPredicate.java @@ -0,0 +1,60 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. Camunda licenses this file to you under the Apache License, + * Version 2.0; you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.camunda.connector.e2e.agenticai.assertj; + +import dev.langchain4j.data.message.ToolExecutionResultMessage; +import java.util.Objects; +import java.util.function.BiPredicate; +import org.json.JSONException; +import org.skyscreamer.jsonassert.JSONCompare; +import org.skyscreamer.jsonassert.JSONCompareMode; + +/** + * Compares {@link ToolExecutionResultMessage} instances using JSON-equivalent comparison for the + * text field, falling back to plain string equality for non-JSON content. + */ +public class ToolExecutionResultMessageEqualsPredicate + implements BiPredicate { + + @Override + public boolean test(ToolExecutionResultMessage a, ToolExecutionResultMessage b) { + if (!Objects.equals(a.id(), b.id())) { + return false; + } + + if (!Objects.equals(a.toolName(), b.toolName())) { + return false; + } + + return jsonEquals(a.text(), b.text()); + } + + private static boolean jsonEquals(String a, String b) { + if (Objects.equals(a, b)) { + return true; + } + if (a == null || b == null) { + return false; + } + try { + return !JSONCompare.compareJSON(a, b, JSONCompareMode.STRICT).failed(); + } catch (JSONException e) { + // not valid JSON (and not equal strings, since Objects.equals check above handles that) + return false; + } + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandler.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandler.java index 46b820e05df..4c46f8299e9 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandler.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandler.java @@ -220,7 +220,7 @@ private List getA2aClientIds(AgentContext agentContext) { private ToolCallResult toolCallResultFromA2aSendMessage(ToolCallResult toolCallResult) { final var identifier = new A2aToolCallIdentifier(toolCallResult.name()); - // Use raw content from the original tool call result (preserving document references + // use raw content from the original tool call result (preserving document references // as deserialized by the engine) rather than the typed A2aSendMessageResult, which would // lose document reference fidelity during re-serialization. The ToolCallResultDocumentExtractor // can only walk Map/Collection/Document — typed records are invisible to it. diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandler.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandler.java index 958de228d2c..c5eb2f3a421 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandler.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandler.java @@ -259,7 +259,7 @@ private ToolCallResult toolCallResultFromMcpToolCall(ToolCallResult toolCallResu && callToolResult.content().getFirst() instanceof McpTextContent textContent) { toolCallResultBuilder.content(textContent.text()); } else { - // Use the raw content from the original tool call result (preserving document references + // use the raw content from the original tool call result (preserving document references // as deserialized by the engine) rather than the typed McpContent list, which may lose // document reference fidelity during re-serialization. toolCallResultBuilder.content(getRawMcpContent(toolCallResult)); diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandlerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandlerTest.java index 9d6ece8805c..8e02fda91d6 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandlerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandlerTest.java @@ -465,7 +465,7 @@ void preservesRawContentWithDocuments_forDocumentExtraction() { var agentContext = AgentContext.empty().withProperty(PROPERTY_A2A_CLIENTS, List.of("a2a1")); var document = mock(Document.class); - // Simulate raw content as it arrives from the engine: a Map/List tree with Document instances + // simulate raw content as it arrives from the engine: a Map/List tree with Document instances var rawContent = Map.of( "kind", @@ -479,7 +479,7 @@ void preservesRawContentWithDocuments_forDocumentExtraction() { assertThat(result).hasSize(1); assertThat(result.getFirst().name()).isEqualTo("A2A_a2a1"); - // Verify the document extractor can find documents in the preserved raw content + // verify the document extractor can find documents in the preserved raw content var extractor = new ToolCallResultDocumentExtractor(); var extractedDocuments = extractor.extractDocuments(result); assertThat(extractedDocuments).hasSize(1); diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java index ef2cdf15a68..b2fb2301ed4 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java @@ -933,9 +933,8 @@ void createsDocumentUserMessageWhenToolResultsContainDocuments() { assertThat(c) .isEqualTo( textContent( - "")), + "" + .formatted(shortId1))), c -> assertThat(c) .isEqualTo(DocumentContent.documentContent(doc1)), @@ -943,9 +942,8 @@ void createsDocumentUserMessageWhenToolResultsContainDocuments() { assertThat(c) .isEqualTo( textContent( - "")), + "" + .formatted(shortId2))), c -> assertThat(c) .isEqualTo(DocumentContent.documentContent(doc2))); @@ -1027,9 +1025,8 @@ void ordersDocumentUserMessageBetweenToolResultsAndEvents() { assertThat(c) .isEqualTo( textContent( - "")), + "" + .formatted(shortId))), c -> assertThat(c) .isEqualTo(DocumentContent.documentContent(doc)))), @@ -1081,9 +1078,8 @@ void appendsDocumentsToEventMessage() { assertThat(c) .isEqualTo( textContent( - "")), + "" + .formatted(shortId))), c -> assertThat(c).isEqualTo(DocumentContent.documentContent(doc)))); } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandlerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandlerTest.java index 56edc3b70f5..b480696ce9f 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandlerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandlerTest.java @@ -401,16 +401,26 @@ void retainsListOfContentBlocksIfResultIsNotASingleTextBlock() { McpTextContent.textContent("Second content")), false); // simulate engine deserialization: content arrives as a raw Map, not a typed POJO + @SuppressWarnings("unchecked") var contentAsMap = objectMapper.convertValue(mcpCallToolResult, Map.class); var toolCallResults = List.of(createToolCallResultWithContent("call1", "mcp1", contentAsMap)); var result = handler.transformToolCallResults(agentContext, toolCallResults); assertThat(result).hasSize(1); - // getRawMcpContent extracts the "content" key from the map + // getRawMcpContent extracts the "content" key from the map, preserving the raw list assertThat(result.getFirst().content()) .asInstanceOf(InstanceOfAssertFactories.LIST) - .hasSize(2); + .hasSize(2) + .satisfiesExactly( + first -> + assertThat(first) + .asInstanceOf(InstanceOfAssertFactories.MAP) + .containsEntry("text", "First content"), + second -> + assertThat(second) + .asInstanceOf(InstanceOfAssertFactories.MAP) + .containsEntry("text", "Second content")); } @Test From e28558e1f27c1372c6fc4d35380165ecd168d230 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Wed, 22 Apr 2026 21:08:08 +0200 Subject: [PATCH 17/81] docs(agentic-ai): update ADR-004 to reflect implementation - update document user message format to XML tags with correlation attributes (tool, call-id, document-short-id, filename) - document the message window memory behavior for document messages - document event document labeling consistency - add future optimization note for UserMessage rebuild strategy - update walker to include Object[] support - fix provider list (remove specific provider references) - delete implementation plan file --- ...-document-handling-in-tool-call-results.md | 56 ++++-- ...ment-handling-in-tool-call-results.plan.md | 168 ------------------ 2 files changed, 38 insertions(+), 186 deletions(-) delete mode 100644 connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.plan.md diff --git a/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md b/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md index 4b5a885f53e..f8078aa0262 100644 --- a/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md +++ b/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md @@ -25,8 +25,8 @@ the model as native multi-modal content blocks. ## Decision Drivers * **Model compatibility**: Documents must be provided in a format that LLMs can actually process. -* **Provider independence**: The solution must work across all supported providers (Anthropic, OpenAI via Bedrock), - not just those with native multi-content tool result support. +* **Provider independence**: The solution must work across all supported providers, not just those with native + multi-content tool result support. * **Auditability**: Document handling must be visible in the persisted conversation history. * **Abstraction layer**: Changes should be made in the generic agent layer (internal message model), not deep in the LangChain4J framework adapter, so they are framework-agnostic and auditable. @@ -41,8 +41,7 @@ Extract documents from tool call results and add them as separate `Content` bloc **Rejected** because: -- LangChain4J provider adapters have inconsistent support for multi-content tool results across providers. The Anthropic - adapter may handle `ImageContent` in tool results, but Bedrock support is unclear. +- LangChain4J provider adapters have inconsistent support for multi-content tool results across providers. - Changes would be invisible in the conversation history (only visible at the L4J wire format level). - Tightly couples the solution to LangChain4J capabilities. @@ -67,11 +66,13 @@ correlate the reference with the actual content in the user message. **Generic layer** (`AgentMessagesHandlerImpl`): -1. After building the `ToolCallResultMessage`, scan each `ToolCallResult.content()` tree for `Document` instances. -2. Build a single `UserMessage` containing `TextContent` separators per tool call (for association) and `DocumentContent` - blocks for each extracted document. -3. Apply the same extraction to event messages: append `DocumentContent` blocks to the event `UserMessage`. -4. Message ordering: `ToolCallResultMessage` -> document `UserMessage` -> event `UserMessage`(s). +1. After building the `ToolCallResultMessage`, scan each `ToolCallResult.content()` tree for `Document` instances using + `ToolCallResultDocumentExtractor` (handles `Map`, `Collection`, `Object[]`, and `Document`). +2. Build a single `UserMessage` (metadata: `toolCallDocuments=true`) containing a preamble, per-document XML tags with + correlation attributes, and `DocumentContent` blocks. +3. Apply the same extraction to event messages: prepend `` XML tags before each `DocumentContent` block in the + event `UserMessage`. +4. Message ordering: `ToolCallResultMessage` → document `UserMessage` → event `UserMessage`(s). **LangChain4J layer** (`ToolCallConverterImpl`): @@ -85,16 +86,34 @@ correlate the reference with the actual content in the user message. references when persisted). The follow-up `UserMessage` with `DocumentContent` blocks is a regular message in the conversation. Both are visible and auditable. -**User message format** (example with two tool calls): +### Document user message format + +The document `UserMessage` contains interleaved `TextContent` tags and `DocumentContent` blocks. Each document is +preceded by a self-closing XML tag with correlation attributes: ``` -TextContent: "Tool call 'generate_report' (call_1) documents:" -DocumentContent: report.pdf -DocumentContent: chart.png -TextContent: "Tool call 'fetch_data' (call_2) documents:" -DocumentContent: data.csv +TextContent: "Documents extracted from tool call results:" +TextContent: +DocumentContent: [report.pdf content] +TextContent: +DocumentContent: [chart.png content] +TextContent: +DocumentContent: [data.csv content] ``` +The `document-short-id` is the first segment of the document's UUID identifier (e.g. `25ece9fa` from +`25ece9fa-aeea-423d-98ed-67c1f08b137b`). It provides a compact correlation key for the model to match the reference in +the tool result JSON with the actual content. All attribute values are XML-escaped. + +For event documents, the same `` tag format is used, but without `tool` and `call-id` attributes since events +are not associated with a specific tool call. + +### Message window memory + +The synthetic document `UserMessage` (identified by `UserMessage.METADATA_TOOL_CALL_DOCUMENTS`) does not count toward +the `maxMessages` context window limit. When evicting messages, the document `UserMessage` is removed together with its +associated `ToolCallResultMessage` — it is never orphaned. + ### Gateway tool handlers must preserve raw content Gateway tool handlers (MCP, A2A) transform `ToolCallResult` objects — renaming tool calls with fully qualified @@ -104,7 +123,7 @@ instances in the content tree have already been deserialized by the connectors ` Gateway handlers **must not** convert this raw content to typed domain objects (e.g., `McpClientCallToolResult`, `A2aSendMessageResult`) and put the typed object back as `ToolCallResult.content()`. The `ToolCallResultDocumentExtractor` -walks the content tree using `instanceof` checks for `Document`, `Map`, and `Collection`. Typed records and POJOs are +walks the content tree using `instanceof` checks for `Document`, `Map`, `Collection`, and `Object[]`. Typed records and POJOs are invisible to it — documents nested inside them would not be extracted. Instead, handlers should: @@ -117,7 +136,8 @@ Instead, handlers should: A follow-up optimization can promote specific document types from the user message back into the `ToolExecutionResultMessage` for providers that support native multi-content tool results (e.g., images on Anthropic). This would be a post-processing step in the L4J framework adapter, transparent to the generic layer and conversation -history. +history. The document `UserMessage` can be rebuilt from the `ToolCallResult` content tree (by re-running extraction) +with only the non-promoted documents remaining. ## Consequences @@ -135,5 +155,5 @@ history. * Adds one extra user message to the conversation when tool results contain documents, consuming additional tokens. * The model must correlate document references in the tool result text with document content in the follow-up user - message. The text separators with tool call name and ID mitigate this. + message. The XML tags with tool name, call ID, and document short ID mitigate this. * Slight increase in conversation history size due to the additional message. diff --git a/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.plan.md b/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.plan.md deleted file mode 100644 index 0274a1428f6..00000000000 --- a/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.plan.md +++ /dev/null @@ -1,168 +0,0 @@ -# Implementation Plan: Document Handling in Tool Call Results - -Reference: [ADR-004](004-document-handling-in-tool-call-results.md) - -## Phase 1: New document extraction utility - -### 1.1 Create `ToolCallResultDocumentExtractor` - -**Package**: `io.camunda.connector.agenticai.aiagent.agent` - -**Public API**: - -```java -public class ToolCallResultDocumentExtractor { - - /** Groups of documents extracted from tool call results, preserving order. */ - record ToolCallDocuments(String toolCallId, String toolCallName, List documents) {} - - /** Extracts all Document instances from a list of tool call results, grouped by tool call. */ - List extractDocuments(List toolCallResults); - - /** Extracts all Document instances from an arbitrary object tree. */ - List extractDocuments(Object contentTree); -} -``` - -**Recursive walker**: Handles `Document` (collect), `Map` (recurse values), -`List` (recurse elements), everything else (skip). Supports root-level `Document` content. - -### 1.2 Create `ToolCallResultDocumentExtractorTest` - -- Root-level `Document` -- `Document` in map value -- `Document` in list element -- Deeply nested `Document` (map -> list -> map -> document) -- Mixed content (documents + scalars) -- No documents found -> empty list -- Null content -> empty list -- Grouping: multiple tool calls, some with documents, some without -- Tool calls without documents are excluded from results - ---- - -## Phase 2: Integrate extraction into `AgentMessagesHandlerImpl` - -### 2.1 Inject `ToolCallResultDocumentExtractor` as dependency - -Update constructor and the configuration/wiring that creates `AgentMessagesHandlerImpl`. - -### 2.2 Create document user message for tool call results - -In `addUserMessages()`, after `createToolCallResultMessage()` returns non-null: - -```java -if (toolCallResultMessage != null) { - messages.add(toolCallResultMessage); - var documentMessage = createDocumentMessageForToolResults(toolCallResultMessage.results()); - if (documentMessage != null) messages.add(documentMessage); - messages.addAll(eventMessages); -} -``` - -`createDocumentMessageForToolResults()`: -- Call `extractor.extractDocuments(results)` -- If no documents found, return null -- Build a single `UserMessage` with alternating `TextContent` separators - (`"Tool call '' () documents:"`) and `DocumentContent` blocks - -### 2.3 Extract documents from event messages - -In `createEventMessage()`, after building the main content block: - -- Call `extractor.extractDocuments(eventContent)` -- Append `DocumentContent.documentContent(doc)` for each extracted document - -### 2.4 Update `AgentMessagesHandlerTest` - -New test cases: -- Tool call results containing documents -> document `UserMessage` created after `ToolCallResultMessage`, before events -- Multiple tool calls with documents -> single `UserMessage` with text separators per tool call -- Tool call results without documents -> no extra message -- Event message with documents -> `DocumentContent` blocks appended to event `UserMessage` -- Mixed: tool results with documents + event messages -> correct ordering - -Update existing test setup to inject the new `ToolCallResultDocumentExtractor` dependency. - ---- - -## Phase 3: Simplify L4J tool result serialization - -### 3.1 Modify `ToolCallConverterImpl` - -- Remove `ContentConverter` dependency from constructor; keep only `ObjectMapper` -- Replace `contentConverter.convertToString(result)` with inline logic: - ```java - private String contentAsString(String toolName, Object result) { - if (result == null) return null; - if (result instanceof String s) return s; - return objectMapper.writeValueAsString(result); - } - ``` -- The injected `ObjectMapper` is the `@ConnectorsObjectMapper` which has `DocumentSerializer` registered, - so `Document` instances serialize as document references. - -### 3.2 Update `ToolCallConverterTest` - -- Update constructor: remove `ContentConverterImpl` parameter -- `supportsResultsContainingCamundaDocuments`: assert document reference format instead of - `DocumentToContentResponseModel` format (base64/text content blocks) - -### 3.3 Simplify `ContentConverterImpl` - -- Remove `contentObjectMapper` field (the ObjectMapper copy with `DocumentToContentModule`) -- `convertToString()` uses the injected `objectMapper` directly -- Constructor simplifies (still receives `DocumentToContentConverter` for `convertToContent()`) - -### 3.4 Update `ContentConverterTest` - -- `supportsObjectContentContainingCamundaDocuments`: assert document reference format - -### 3.5 Update `AgenticAiLangchain4JFrameworkConfiguration` - -- `langchain4JToolCallConverter` bean: remove `ContentConverter` parameter, pass only `ObjectMapper` - ---- - -## Phase 4: Delete stale infrastructure - -### 4.1 Delete classes - -- `DocumentToContentSerializer.java` -- `DocumentToContentModule.java` -- `DocumentToContentResponseModel.java` - -### 4.2 Delete tests - -- `DocumentToContentSerializerTest.java` - -### 4.3 Remove unused imports - -Clean up any remaining references to deleted classes across the codebase. - ---- - -## Phase 5: Update E2E tests - -### 5.1 Update `BaseL4JAiAgentJobWorkerTest` and `BaseL4JAiAgentConnectorTest` - -- Change `DownloadFileToolResult` record: replace `DocumentToContentResponseModel document` field - with the document reference format (or a generic `Object` that matches the serialized reference) -- Remove `DocumentToContentResponseModel` import - -### 5.2 Update `L4JAiAgentJobWorkerToolCallingTests` and `L4JAiAgentConnectorToolCallingTests` - -Update `supportsDocumentResponsesFromToolCalls`: -- Expected `ToolExecutionResultMessage` text: document reference format instead of `DocumentToContentResponseModel` -- Expected conversation includes a new `UserMessage` after the `ToolExecutionResultMessage` containing: - - `TextContent` separator with tool call name and ID - - Document content block (`TextContent` for text types, `ImageContent` for images, `PdfFileContent` for PDF) -- Adjust `assertLastChatRequest` expected conversation list and chat request count if needed - ---- - -## Verification - -1. `mvn clean install -pl connectors/agentic-ai` -- unit tests pass -2. Run e2e tests manually (per CLAUDE.md: never run e2e tests automatically) -3. Verify conversation history serialization round-trips correctly (existing tests should cover this) From 0d28ff0518d7907685a6ea1875c7f6e9814c1b7e Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Wed, 22 Apr 2026 21:13:59 +0200 Subject: [PATCH 18/81] fix(agentic-ai): guard effectiveCount decrement for document messages Only decrement the effective message count when the evicted message is not a tool-call document message, preventing under-counting if an orphaned document message ends up at the eviction position. --- .../runtime/MessageWindowRuntimeMemory.java | 4 ++- .../MessageWindowRuntimeMemoryTest.java | 28 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/memory/runtime/MessageWindowRuntimeMemory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/memory/runtime/MessageWindowRuntimeMemory.java index bec492ee9c4..66d795bfab7 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/memory/runtime/MessageWindowRuntimeMemory.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/memory/runtime/MessageWindowRuntimeMemory.java @@ -85,7 +85,9 @@ private static List filteredMessages(List messages, int maxMes // remove the message at the current index Message evictedMessage = filtered.remove(messageToEvictIndex); - effectiveCount--; + if (!isToolCallDocumentMessage(evictedMessage)) { + effectiveCount--; + } // remove follow-up tool call results if existing as some LLM providers return an error when // receiving tool call results without the original tool call request diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/memory/runtime/MessageWindowRuntimeMemoryTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/memory/runtime/MessageWindowRuntimeMemoryTest.java index d50a64c4f63..acf02976b81 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/memory/runtime/MessageWindowRuntimeMemoryTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/memory/runtime/MessageWindowRuntimeMemoryTest.java @@ -249,6 +249,34 @@ void evictsDocumentUserMessageWithToolCallResult() { && tcr.results().stream().anyMatch(r -> "call_1".equals(r.id()))); } + @Test + void handlesOrphanedDocumentMessageDuringEviction() { + // edge case: a document message ends up at the eviction position without a preceding + // tool call result (e.g. from corrupted/migrated persisted history) + final var documentUserMessage = + UserMessage.builder() + .content(List.of(textContent("Documents extracted from tool call results:"))) + .metadata(Map.of(UserMessage.METADATA_TOOL_CALL_DOCUMENTS, true)) + .build(); + + List messages = new ArrayList<>(); + messages.add(systemMessage("System")); + // orphaned document message right after system message + messages.add(documentUserMessage); + for (int i = 1; i <= MAX_MESSAGES + 1; i++) { + messages.add(userMessage("Message " + i)); + } + + memory.addMessages(messages); + + // the document message should be evicted without affecting the effective count; + // effective count = system + 7 remaining user messages = 8 = MAX_MESSAGES + var filtered = memory.filteredMessages(); + assertThat(filtered).noneMatch(m -> m == documentUserMessage); + assertThat(filtered).hasSize(MAX_MESSAGES); + assertThat(filtered.getFirst()).isEqualTo(systemMessage("System")); + } + @Test void returnsLastMessage() { memory.addMessages(TEST_MESSAGES); From 65b2ae17fc3f67dbfdda6f5fbc74ea16ba69486d Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Wed, 22 Apr 2026 21:22:46 +0200 Subject: [PATCH 19/81] refactor(agentic-ai): extract DocumentXmlTag record from handler Move XML tag building, attribute escaping, and document short ID extraction into a dedicated DocumentXmlTag record with factory methods and toXml() serialization. Tests moved to DocumentXmlTagTest. --- ...4JAiAgentJobWorkerMcpIntegrationTests.java | 2 +- .../L4JAiAgentJobWorkerToolCallingTests.java | 2 +- ...4JAiAgentConnectorMcpIntegrationTests.java | 2 +- .../L4JAiAgentConnectorToolCallingTests.java | 2 +- ...-document-handling-in-tool-call-results.md | 6 +- .../agent/AgentMessagesHandlerImpl.java | 64 +---------- .../model/message/DocumentXmlTag.java | 88 +++++++++++++++ .../agent/AgentMessagesHandlerTest.java | 100 +----------------- .../model/message/DocumentXmlTagTest.java | 93 ++++++++++++++++ 9 files changed, 196 insertions(+), 163 deletions(-) create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/DocumentXmlTag.java create mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/model/message/DocumentXmlTagTest.java diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerMcpIntegrationTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerMcpIntegrationTests.java index 52baabb1167..f63c664be7a 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerMcpIntegrationTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerMcpIntegrationTests.java @@ -434,7 +434,7 @@ void extractsDocumentsFromMcpImageToolCallResult() throws IOException { tc -> assertThat(tc.text()) .isEqualTo( - "" + "" .formatted(documentShortId))); assertThat(contents.get(2)) .isInstanceOfSatisfying( diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java index 6486c965702..d1158597541 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java @@ -185,7 +185,7 @@ void supportsDocumentResponsesFromToolCalls( tc -> assertThat(tc.text()) .isEqualTo( - "" + "" .formatted(documentShortId))); assertDocumentContentBlock(contents.get(2), type, mimeType); }); diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorMcpIntegrationTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorMcpIntegrationTests.java index 03025efb70d..b45dceb335d 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorMcpIntegrationTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorMcpIntegrationTests.java @@ -436,7 +436,7 @@ void extractsDocumentsFromMcpImageToolCallResult() throws IOException { tc -> assertThat(tc.text()) .isEqualTo( - "" + "" .formatted(documentShortId))); assertThat(contents.get(2)) .isInstanceOfSatisfying( diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java index 2274a59c7bd..8e6326fa295 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java @@ -178,7 +178,7 @@ void supportsDocumentResponsesFromToolCalls( tc -> assertThat(tc.text()) .isEqualTo( - "" + "" .formatted(documentShortId))); assertDocumentContentBlock(contents.get(2), type, mimeType); }); diff --git a/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md b/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md index f8078aa0262..1d8cd804c59 100644 --- a/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md +++ b/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md @@ -93,11 +93,11 @@ preceded by a self-closing XML tag with correlation attributes: ``` TextContent: "Documents extracted from tool call results:" -TextContent: +TextContent: DocumentContent: [report.pdf content] -TextContent: +TextContent: DocumentContent: [chart.png content] -TextContent: +TextContent: DocumentContent: [data.csv content] ``` diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java index 19f01d1a59d..5c03b521315 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java @@ -20,6 +20,7 @@ import io.camunda.connector.agenticai.aiagent.systemprompt.SystemPromptComposer; import io.camunda.connector.agenticai.aiagent.tool.GatewayToolHandlerRegistry; import io.camunda.connector.agenticai.model.message.AssistantMessage; +import io.camunda.connector.agenticai.model.message.DocumentXmlTag; import io.camunda.connector.agenticai.model.message.Message; import io.camunda.connector.agenticai.model.message.SystemMessage; import io.camunda.connector.agenticai.model.message.ToolCallResultMessage; @@ -28,8 +29,6 @@ import io.camunda.connector.agenticai.model.message.content.DocumentContent; import io.camunda.connector.agenticai.model.tool.ToolCall; import io.camunda.connector.agenticai.model.tool.ToolCallResult; -import io.camunda.connector.api.document.Document; -import io.camunda.connector.api.document.DocumentReference.CamundaDocumentReference; import io.camunda.connector.api.error.ConnectorException; import java.time.ZonedDateTime; import java.util.ArrayList; @@ -226,7 +225,9 @@ private UserMessage createDocumentMessageForToolResults(List res content.add(textContent("Documents extracted from tool call results:")); for (var entry : toolCallDocuments) { for (var doc : entry.documents()) { - content.add(textContent(documentXmlTag(doc, entry.toolCallName(), entry.toolCallId()))); + content.add( + textContent( + DocumentXmlTag.from(doc, entry.toolCallName(), entry.toolCallId()).toXml())); content.add(DocumentContent.documentContent(doc)); } } @@ -260,7 +261,7 @@ private Message createEventMessage( var eventDocuments = documentExtractor.extractDocumentsFromContent(eventContent); if (!eventDocuments.isEmpty()) { for (var doc : eventDocuments) { - userMessageContent.add(textContent(documentXmlTag(doc))); + userMessageContent.add(textContent(DocumentXmlTag.from(doc).toXml())); userMessageContent.add(DocumentContent.documentContent(doc)); } } @@ -290,61 +291,6 @@ private boolean interruptToolCallsOnEventResults(AgentExecutionContext execution return behavior == EventHandlingConfiguration.EventHandlingBehavior.INTERRUPT_TOOL_CALLS; } - /** - * Builds an XML self-closing tag describing a document for model correlation. The tag includes - * optional attributes for tool name, call ID, the document short ID (first segment of the UUID - * document identifier), and filename. All attribute values are XML-escaped. - */ - static String documentXmlTag(Document document, String toolName, String toolCallId) { - var sb = new StringBuilder(""); - return sb.toString(); - } - - static String documentXmlTag(Document document) { - return documentXmlTag(document, null, null); - } - - /** - * Returns the first segment of the document's UUID identifier (e.g. "25ece9fa" from - * "25ece9fa-aeea-423d-98ed-67c1f08b137b"), providing a compact correlation key that is sufficient - * for in-conversation matching between the document reference in the tool result and the document - * content in the follow-up user message. - */ - private static String documentShortId(Document document) { - if (document.reference() instanceof CamundaDocumentReference camundaRef) { - var documentId = camundaRef.getDocumentId(); - if (documentId != null) { - int dashIndex = documentId.indexOf('-'); - return dashIndex > 0 ? documentId.substring(0, dashIndex) : documentId; - } - } - return null; - } - - private static void appendXmlAttribute(StringBuilder sb, String name, String value) { - if (StringUtils.isNotBlank(value)) { - sb.append(" %s=\"%s\"".formatted(name, escapeXmlAttribute(value))); - } - } - - static String escapeXmlAttribute(String value) { - if (value == null) { - return null; - } - return value - .replace("&", "&") - .replace("<", "<") - .replace(">", ">") - .replace("\"", """) - .replace("'", "'"); - } - private Map defaultMessageMetadata() { return Map.of("timestamp", ZonedDateTime.now()); } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/DocumentXmlTag.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/DocumentXmlTag.java new file mode 100644 index 00000000000..66d94445d66 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/DocumentXmlTag.java @@ -0,0 +1,88 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.model.message; + +import io.camunda.connector.api.document.Document; +import io.camunda.connector.api.document.DocumentReference.CamundaDocumentReference; +import org.apache.commons.lang3.StringUtils; +import org.springframework.lang.Nullable; + +/** + * Represents a self-closing XML tag used to label a document in the synthetic user message, e.g.: + * + * 
{@code
+ * 
+ * }
+ * + *

The tag provides correlation attributes so the model can match the document content block with + * the document reference in the tool result JSON. + */ +public record DocumentXmlTag( + @Nullable String toolName, + @Nullable String toolCallId, + @Nullable String documentShortId, + @Nullable String filename) { + + /** + * Creates a tag from a document and its tool call context. The document short ID is extracted as + * the first segment of the document's UUID identifier (e.g. "25ece9fa" from + * "25ece9fa-aeea-423d-98ed-67c1f08b137b"). + */ + public static DocumentXmlTag from(Document document, String toolName, String toolCallId) { + return new DocumentXmlTag( + toolName, toolCallId, extractDocumentShortId(document), extractFileName(document)); + } + + /** Creates a tag from a document without tool call context (e.g. for event documents). */ + public static DocumentXmlTag from(Document document) { + return from(document, null, null); + } + + /** Serializes this tag to an XML self-closing element string. */ + public String toXml() { + var sb = new StringBuilder(""); + return sb.toString(); + } + + private static void appendAttribute(StringBuilder sb, String name, String value) { + if (StringUtils.isNotBlank(value)) { + sb.append(" %s=\"%s\"".formatted(name, escapeXmlAttribute(value))); + } + } + + private static String escapeXmlAttribute(String value) { + if (value == null) { + return null; + } + return value + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace("\"", """) + .replace("'", "'"); + } + + private static String extractDocumentShortId(Document document) { + if (document.reference() instanceof CamundaDocumentReference camundaRef) { + var documentId = camundaRef.getDocumentId(); + if (documentId != null) { + int dashIndex = documentId.indexOf('-'); + return dashIndex > 0 ? documentId.substring(0, dashIndex) : documentId; + } + } + return null; + } + + private static String extractFileName(Document document) { + return document.metadata() != null ? document.metadata().getFileName() : null; + } +} diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java index b2fb2301ed4..840a94a43f5 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java @@ -118,100 +118,6 @@ private static String documentShortId(Document document) { return null; } - @Nested - class DocumentXmlTagTest { - - @Test - void generatesFullTagWithAllAttributes() { - var doc = mock(Document.class); - var ref = mock(CamundaDocumentReference.class); - var metadata = mock(io.camunda.connector.api.document.DocumentMetadata.class); - when(doc.reference()).thenReturn(ref); - when(ref.getDocumentId()).thenReturn("25ece9fa-aeea-423d-98ed-67c1f08b137b"); - when(doc.metadata()).thenReturn(metadata); - when(metadata.getFileName()).thenReturn("report.pdf"); - - assertThat(AgentMessagesHandlerImpl.documentXmlTag(doc, "search", "call_abc")) - .isEqualTo( - ""); - } - - @Test - void generatesTagWithoutToolAndCallId() { - var doc = mock(Document.class); - var ref = mock(CamundaDocumentReference.class); - var metadata = mock(io.camunda.connector.api.document.DocumentMetadata.class); - when(doc.reference()).thenReturn(ref); - when(ref.getDocumentId()).thenReturn("f7b3a1d0-1234-5678-9abc-def012345678"); - when(doc.metadata()).thenReturn(metadata); - when(metadata.getFileName()).thenReturn(null); - - assertThat(AgentMessagesHandlerImpl.documentXmlTag(doc)) - .isEqualTo(""); - } - - @Test - void generatesMinimalTagForMockedDocument() { - var doc = mock(Document.class); - assertThat(AgentMessagesHandlerImpl.documentXmlTag(doc)).isEqualTo(""); - } - - @Test - void handlesDocumentIdWithoutDash() { - var doc = mock(Document.class); - var ref = mock(CamundaDocumentReference.class); - when(doc.reference()).thenReturn(ref); - when(ref.getDocumentId()).thenReturn("simpledocid"); - - assertThat(AgentMessagesHandlerImpl.documentXmlTag(doc)) - .isEqualTo(""); - } - - @Test - void escapesSpecialCharactersInFilename() { - var doc = mock(Document.class); - var metadata = mock(io.camunda.connector.api.document.DocumentMetadata.class); - when(doc.metadata()).thenReturn(metadata); - when(metadata.getFileName()).thenReturn("file\"with&chars'.pdf"); - - assertThat(AgentMessagesHandlerImpl.documentXmlTag(doc)) - .isEqualTo(""); - } - - @Test - void escapesSpecialCharactersInToolName() { - var doc = mock(Document.class); - var ref = mock(CamundaDocumentReference.class); - when(doc.reference()).thenReturn(ref); - when(ref.getDocumentId()).thenReturn("abc12345-0000-0000-0000-000000000000"); - - assertThat(AgentMessagesHandlerImpl.documentXmlTag(doc, "tool", "call_1")) - .isEqualTo( - ""); - } - } - - @Nested - class EscapeXmlAttributeTest { - - @Test - void escapesAllSpecialCharacters() { - assertThat(AgentMessagesHandlerImpl.escapeXmlAttribute("a&bd\"e'f")) - .isEqualTo("a&b<c>d"e'f"); - } - - @Test - void returnsNullForNull() { - assertThat(AgentMessagesHandlerImpl.escapeXmlAttribute(null)).isNull(); - } - - @Test - void returnsUnchangedForSafeString() { - assertThat(AgentMessagesHandlerImpl.escapeXmlAttribute("safe-value_123")) - .isEqualTo("safe-value_123"); - } - } - @Nested class SystemMessagesTest { @@ -933,7 +839,7 @@ void createsDocumentUserMessageWhenToolResultsContainDocuments() { assertThat(c) .isEqualTo( textContent( - "" + "" .formatted(shortId1))), c -> assertThat(c) @@ -942,7 +848,7 @@ void createsDocumentUserMessageWhenToolResultsContainDocuments() { assertThat(c) .isEqualTo( textContent( - "" + "" .formatted(shortId2))), c -> assertThat(c) @@ -1025,7 +931,7 @@ void ordersDocumentUserMessageBetweenToolResultsAndEvents() { assertThat(c) .isEqualTo( textContent( - "" + "" .formatted(shortId))), c -> assertThat(c) diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/model/message/DocumentXmlTagTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/model/message/DocumentXmlTagTest.java new file mode 100644 index 00000000000..33913e05eb9 --- /dev/null +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/model/message/DocumentXmlTagTest.java @@ -0,0 +1,93 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.model.message; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.camunda.connector.api.document.Document; +import io.camunda.connector.api.document.DocumentMetadata; +import io.camunda.connector.api.document.DocumentReference.CamundaDocumentReference; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +class DocumentXmlTagTest { + + @Nested + class ToXml { + + @Test + void generatesFullTagWithAllAttributes() { + var doc = mock(Document.class); + var ref = mock(CamundaDocumentReference.class); + var metadata = mock(DocumentMetadata.class); + when(doc.reference()).thenReturn(ref); + when(ref.getDocumentId()).thenReturn("25ece9fa-aeea-423d-98ed-67c1f08b137b"); + when(doc.metadata()).thenReturn(metadata); + when(metadata.getFileName()).thenReturn("report.pdf"); + + assertThat(DocumentXmlTag.from(doc, "search", "call_abc").toXml()) + .isEqualTo( + ""); + } + + @Test + void generatesTagWithoutToolAndCallId() { + var doc = mock(Document.class); + var ref = mock(CamundaDocumentReference.class); + var metadata = mock(DocumentMetadata.class); + when(doc.reference()).thenReturn(ref); + when(ref.getDocumentId()).thenReturn("f7b3a1d0-1234-5678-9abc-def012345678"); + when(doc.metadata()).thenReturn(metadata); + when(metadata.getFileName()).thenReturn(null); + + assertThat(DocumentXmlTag.from(doc).toXml()) + .isEqualTo(""); + } + + @Test + void generatesMinimalTagForMockedDocument() { + var doc = mock(Document.class); + assertThat(DocumentXmlTag.from(doc).toXml()).isEqualTo(""); + } + + @Test + void handlesDocumentIdWithoutDash() { + var doc = mock(Document.class); + var ref = mock(CamundaDocumentReference.class); + when(doc.reference()).thenReturn(ref); + when(ref.getDocumentId()).thenReturn("simpledocid"); + + assertThat(DocumentXmlTag.from(doc).toXml()) + .isEqualTo(""); + } + + @Test + void escapesSpecialCharactersInFilename() { + var doc = mock(Document.class); + var metadata = mock(DocumentMetadata.class); + when(doc.metadata()).thenReturn(metadata); + when(metadata.getFileName()).thenReturn("file\"with&chars'.pdf"); + + assertThat(DocumentXmlTag.from(doc).toXml()) + .isEqualTo(""); + } + + @Test + void escapesSpecialCharactersInToolName() { + var doc = mock(Document.class); + var ref = mock(CamundaDocumentReference.class); + when(doc.reference()).thenReturn(ref); + when(ref.getDocumentId()).thenReturn("abc12345-0000-0000-0000-000000000000"); + + assertThat(DocumentXmlTag.from(doc, "tool", "call_1").toXml()) + .isEqualTo( + ""); + } + } +} From 91024cc4055ca921fdc8e9e94f9f8ab981661efe Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Wed, 22 Apr 2026 23:58:21 +0200 Subject: [PATCH 20/81] test(e2e): add cross-provider CPT viability test for document tool call results Add a manual CPT test that validates real LLM providers can receive and reason about PDF documents extracted from tool call results via the synthetic UserMessage with XML correlation tags. The test covers three scenarios with increasing complexity: - Single document returned from a tool call - Multiple documents returned as a list - Documents embedded in a nested Map structure A BPMN process downloads PDFs from WireMock, then uses FEEL script tasks inside an ad-hoc subprocess to assemble tool results of varying shapes. The AI Agent connector processes these with a real LLM, and CPT judge assertions verify the model correctly extracted facts from the PDFs. Provider configs (toggled via env vars): OpenAI, Anthropic, AWS Bedrock, and OpenAI-compatible (Docker Model Runner). The test is @Disabled by default and not part of CI. --- .../connectors-e2e-test-agentic-ai/pom.xml | 6 + .../cpt/DocumentToolCallResultsCPTTest.java | 400 ++++++++++++++++++ .../src/test/resources/__files/cpt-doc1.pdf | Bin 0 -> 1009 bytes .../src/test/resources/__files/cpt-doc2.pdf | Bin 0 -> 1013 bytes .../src/test/resources/__files/cpt-doc3.pdf | Bin 0 -> 1025 bytes .../cpt-document-tool-call-results.bpmn | 105 +++++ 6 files changed, 511 insertions(+) create mode 100644 connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsCPTTest.java create mode 100644 connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/cpt-doc1.pdf create mode 100644 connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/cpt-doc2.pdf create mode 100644 connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/cpt-doc3.pdf create mode 100644 connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/cpt-document-tool-call-results.bpmn diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/pom.xml b/connectors-e2e-test/connectors-e2e-test-agentic-ai/pom.xml index d3bfd56563d..ae7b2973d32 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/pom.xml +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/pom.xml @@ -88,6 +88,12 @@ camunda-process-test-spring ${version.camunda} + + io.camunda + camunda-process-test-langchain4j + ${version.camunda} + test + diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsCPTTest.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsCPTTest.java new file mode 100644 index 00000000000..b15bdb40635 --- /dev/null +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsCPTTest.java @@ -0,0 +1,400 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. Camunda licenses this file to you under the Apache License, + * Version 2.0; you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.camunda.connector.e2e.agenticai.aiagent.cpt; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static io.camunda.process.test.api.CamundaAssert.assertThat; + +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import io.camunda.client.CamundaClient; +import io.camunda.connector.e2e.BpmnFile; +import io.camunda.connector.e2e.ElementTemplate; +import io.camunda.connector.e2e.agenticai.CamundaDocumentTestConfiguration; +import io.camunda.connector.e2e.app.TestConnectorRuntimeApplication; +import io.camunda.connector.runtime.core.document.store.InMemoryDocumentStore; +import io.camunda.process.test.api.CamundaSpringProcessTest; +import io.camunda.zeebe.model.bpmn.BpmnModelInstance; +import java.io.File; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.core.io.ResourceLoader; + +/** + * Cross-provider viability test for document handling in tool call results. + * + *

Validates that real LLM providers can receive and reason about PDF documents extracted from + * tool call results via the synthetic UserMessage with XML correlation tags. + * + *

This test is NOT part of the CI suite. Run it manually to assess provider compatibility. + * Configure API keys via environment variables: + * + *

    + *
  • {@code OPENAI_API_KEY} - OpenAI API key + *
  • {@code ANTHROPIC_API_KEY} - Anthropic API key + *
  • {@code AWS_BEDROCK_ACCESS_KEY} / {@code AWS_BEDROCK_SECRET_KEY} - AWS Bedrock credentials + * (also used for the judge LLM) + *
  • {@code DOCKER_MODEL_RUNNER_URL} - OpenAI-compatible endpoint (default: + * http://localhost:12434/engines/llama.cpp/v1) + *
+ */ +@SpringBootTest( + classes = {TestConnectorRuntimeApplication.class}, + properties = { + "spring.main.allow-bean-definition-overriding=true", + "camunda.connector.webhook.enabled=false", + "camunda.connector.polling.enabled=false", + "camunda.connector.agenticai.tools.process-definition.cache.enabled=false", + // Judge LLM configuration (uses Bedrock Haiku for cost efficiency) + "camunda.process-test.judge.chat-model.provider=amazon-bedrock", + "camunda.process-test.judge.chat-model.model=eu.anthropic.claude-haiku-4-5-20251001-v1:0", + "camunda.process-test.judge.chat-model.region=eu-central-1", + "camunda.process-test.judge.chat-model.credentials.access-key=${AWS_BEDROCK_ACCESS_KEY:NOT_SET}", + "camunda.process-test.judge.chat-model.credentials.secret-key=${AWS_BEDROCK_SECRET_KEY:NOT_SET}", + "camunda.process-test.judge.threshold=0.6" + }, + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@CamundaSpringProcessTest +@WireMockTest +@Import(CamundaDocumentTestConfiguration.class) +@Disabled("Manual viability test - requires real LLM API keys via environment variables") +class DocumentToolCallResultsCPTTest { + + private static final Logger LOG = LoggerFactory.getLogger(DocumentToolCallResultsCPTTest.class); + + private static final String ELEMENT_TEMPLATE_PATH = + "../../connectors/agentic-ai/element-templates/agenticai-aiagent-job-worker.json"; + + private static final String BPMN_RESOURCE = "classpath:cpt-document-tool-call-results.bpmn"; + + private static final String SYSTEM_PROMPT = + "You are a document analyst. Use the available tools to retrieve and analyze documents. " + + "When reporting findings, always quote specific facts, numbers, dates, and names " + + "found in the documents. Be concise."; + + private static final Duration PROCESS_TIMEOUT = Duration.ofMinutes(3); + + @Autowired private CamundaClient camundaClient; + @Autowired private ResourceLoader resourceLoader; + @TempDir private File tempDir; + + @BeforeEach + void clearDocumentStore() { + InMemoryDocumentStore.INSTANCE.clear(); + } + + @BeforeEach + void setupPdfStubs() { + for (int i = 1; i <= 3; i++) { + String filename = "cpt-doc" + i + ".pdf"; + stubFor( + get(urlPathEqualTo("/" + filename)) + .willReturn( + aResponse() + .withBodyFile(filename) + .withHeader("Content-Type", "application/pdf"))); + } + } + + // --------------------------------------------------------------------------- + // Scenario 1: Single document from tool call result + // --------------------------------------------------------------------------- + + @ParameterizedTest(name = "{0}") + @MethodSource("providers") + @Disabled + void singleDocumentFromToolCallResult(ProviderConfig provider, WireMockRuntimeInfo wireMock) { + var processInstance = + startProcess( + provider, + "Use the Analyze_Single_Document tool to retrieve a document, then tell me " + + "what project it mentions and when it launched.", + List.of(wireMock.getHttpBaseUrl() + "/cpt-doc1.pdf"), + wireMock); + + assertThat(processInstance) + .withAssertionTimeout(PROCESS_TIMEOUT) + .isCompleted() + .hasVariableSatisfies( + "agent", Object.class, agent -> logAgentResponse(provider, "singleDocument", agent)) + .hasVariableSatisfiesJudge( + "agent", + """ + The agent called Analyze_Single_Document, received a PDF document, and produced + a response that mentions Project Zypherion and the launch date March 15, 2026."""); + } + + // --------------------------------------------------------------------------- + // Scenario 2: Multiple documents from tool call result + // --------------------------------------------------------------------------- + + @ParameterizedTest(name = "{0}") + @MethodSource("providers") + @Disabled + void multipleDocumentsFromToolCallResult(ProviderConfig provider, WireMockRuntimeInfo wireMock) { + var processInstance = + startProcess( + provider, + "Use the Search_Documents tool to find documents and summarize what each one says.", + List.of( + wireMock.getHttpBaseUrl() + "/cpt-doc1.pdf", + wireMock.getHttpBaseUrl() + "/cpt-doc2.pdf"), + wireMock); + + assertThat(processInstance) + .withAssertionTimeout(PROCESS_TIMEOUT) + .isCompleted() + .hasVariableSatisfies( + "agent", Object.class, agent -> logAgentResponse(provider, "multipleDocuments", agent)) + .hasVariableSatisfiesJudge( + "agent", + """ + The agent called Search_Documents, received two PDF documents, and produced + a response that mentions both: Project Zypherion launching on March 15, 2026, + and a headcount of 847 employees across 12 offices."""); + } + + // --------------------------------------------------------------------------- + // Scenario 3: Documents in nested structure from tool call result + // --------------------------------------------------------------------------- + + @ParameterizedTest(name = "{0}") + @MethodSource("providers") + void nestedStructureDocumentsFromToolCallResult( + ProviderConfig provider, WireMockRuntimeInfo wireMock) { + var processInstance = + startProcess( + provider, + "Use the Fetch_Report tool to get the full report and describe the content " + + "of every document in it, including attachments and the cover page.", + List.of( + wireMock.getHttpBaseUrl() + "/cpt-doc1.pdf", + wireMock.getHttpBaseUrl() + "/cpt-doc2.pdf", + wireMock.getHttpBaseUrl() + "/cpt-doc3.pdf"), + wireMock); + + assertThat(processInstance) + .withAssertionTimeout(PROCESS_TIMEOUT) + .isCompleted() + .hasVariableSatisfies( + "agent", Object.class, agent -> logAgentResponse(provider, "nestedStructure", agent)) + .hasVariableSatisfiesJudge( + "agent", + """ + The agent called Fetch_Report, received documents embedded in a nested structure, + and produced a response that references all three documents: + 1. Project Zypherion launching on March 15, 2026 + 2. A headcount of 847 employees across 12 offices + 3. The report was prepared by Dr. Kael Thrennix, Chief Analytics Officer"""); + } + + // --------------------------------------------------------------------------- + // Provider configurations + // --------------------------------------------------------------------------- + + static Stream providers() { + return Stream.of( + // OpenAI + openai("gpt-4.1"), + openai("gpt-5.4"), + // Anthropic + anthropic("claude-sonnet-4-6"), + anthropic("claude-haiku-4-5-20251001"), + // AWS Bedrock (Anthropic models via cross-region inference) + bedrock("eu.anthropic.claude-sonnet-4-20250514-v1:0"), + bedrock("global.anthropic.claude-sonnet-4-6"), + bedrock("eu.anthropic.claude-haiku-4-5-20251001-v1:0"), + // Docker Model Runner (OpenAI-compatible) + dockerModelRunner("ai/gemma4:latest")) + .filter(ProviderConfig::isEnabled); + } + + // -- OpenAI -- + + static ProviderConfig openai(String model) { + return new ProviderConfig( + "openai/" + model, + "OPENAI_API_KEY", + Map.of( + "provider.type", + "openai", + "provider.openai.authentication.apiKey", + envOrPlaceholder("OPENAI_API_KEY"), + "provider.openai.model.model", + model)); + } + + // -- Anthropic -- + + static ProviderConfig anthropic(String model) { + return new ProviderConfig( + "anthropic/" + model, + "ANTHROPIC_API_KEY", + Map.of( + "provider.type", + "anthropic", + "provider.anthropic.authentication.apiKey", + envOrPlaceholder("ANTHROPIC_API_KEY"), + "provider.anthropic.model.model", + model)); + } + + // -- AWS Bedrock -- + + static ProviderConfig bedrock(String model) { + return new ProviderConfig( + "bedrock/" + model, + "AWS_BEDROCK_ACCESS_KEY", + Map.of( + "provider.type", + "bedrock", + "provider.bedrock.authentication.type", + "credentials", + "provider.bedrock.authentication.accessKey", + envOrPlaceholder("AWS_BEDROCK_ACCESS_KEY"), + "provider.bedrock.authentication.secretKey", + envOrPlaceholder("AWS_BEDROCK_SECRET_KEY"), + "provider.bedrock.region", + "eu-central-1", + "provider.bedrock.model.model", + model)); + } + + // -- Docker Model Runner (OpenAI-compatible) -- + + static ProviderConfig dockerModelRunner(String model) { + var url = + System.getenv() + .getOrDefault("DOCKER_MODEL_RUNNER_URL", "http://localhost:12434/engines/llama.cpp/v1"); + return new ProviderConfig( + "docker-model-runner/" + model, + "DOCKER_MODEL_RUNNER_URL", + Map.of( + "provider.type", "openaiCompatible", + "provider.openaiCompatible.endpoint", url, + "provider.openaiCompatible.model.model", model)); + } + + // --------------------------------------------------------------------------- + // Helpers + // --------------------------------------------------------------------------- + + private io.camunda.client.api.response.ProcessInstanceEvent startProcess( + ProviderConfig provider, + String userPrompt, + List downloadUrls, + WireMockRuntimeInfo wireMock) { + var model = buildModel(provider); + + // deploy and wait for process definition to be available + var zeebeTest = + io.camunda.connector.e2e.ZeebeTest.with(camundaClient) + .awaitCompleteTopology() + .deploy(model); + + return camundaClient + .newCreateInstanceCommand() + .bpmnProcessId("CPT_Document_Tool_Call_Results") + .latestVersion() + .variables( + Map.of( + "userPrompt", userPrompt, + "downloadUrls", downloadUrls)) + .send() + .join(); + } + + private BpmnModelInstance buildModel(ProviderConfig provider) { + var template = ElementTemplate.from(ELEMENT_TEMPLATE_PATH); + + // base properties + template + .property("agentContext", "=agent.context") + .property("data.systemPrompt.prompt", "=\"" + SYSTEM_PROMPT + "\"") + .property( + "data.userPrompt.prompt", + "=if (is defined(followUpUserPrompt)) then followUpUserPrompt else userPrompt") + .property("data.userPrompt.documents", "=[]") + .property("data.memory.storage.type", "in-process") + .property("data.memory.contextWindowSize", "=50") + .property("data.response.includeAssistantMessage", "=true") + .property("data.response.includeAgentContext", "=true"); + + // provider-specific properties + provider.properties().forEach(template::property); + + try { + var templateFile = template.writeTo(new File(tempDir, "template.json")); + var bpmnFile = resourceLoader.getResource(BPMN_RESOURCE).getFile(); + return new BpmnFile(bpmnFile) + .apply(templateFile, "AI_Agent", new File(tempDir, "applied.bpmn")); + } catch (Exception e) { + throw new RuntimeException("Failed to build BPMN model for " + provider.label(), e); + } + } + + private void logAgentResponse(ProviderConfig provider, String scenario, Object agent) { + Object responseText = agent; + if (agent instanceof Map map && map.containsKey("responseText")) { + responseText = map.get("responseText"); + } + LOG.info( + "\n========== [{} / {}] ==========\n{}\n==========", + provider.label(), + scenario, + responseText); + } + + private static String envOrPlaceholder(String envVar) { + return System.getenv().getOrDefault(envVar, "NOT_SET"); + } + + // --------------------------------------------------------------------------- + // Provider config record + // --------------------------------------------------------------------------- + + record ProviderConfig(String label, String requiredEnvVar, Map properties) { + + boolean isEnabled() { + // Docker Model Runner doesn't need an API key env var, just the URL + if (requiredEnvVar.equals("DOCKER_MODEL_RUNNER_URL")) { + return true; + } + return System.getenv(requiredEnvVar) != null; + } + + @Override + public String toString() { + return label; + } + } +} diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/cpt-doc1.pdf b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/cpt-doc1.pdf new file mode 100644 index 0000000000000000000000000000000000000000..559b0a9653852fc40fb37a8ec981dd79d8f55e6e GIT binary patch literal 1009 zcmah|O=}ZD7_Lgej(U+?st5xmP(?GdJG<+ygg~<&G_<90w@_^8VVg|i%62B~PBiu8 z#fx5wUIab$FX-KiKcN3W6hW*PwST~wG>J=la4xg+%)Zb2ywCeix#qw4DnKqDFab4yR+3H*TSQ)k+W@W6P7>Go5Vo*HOvkL~Hna_`VmtD7W3Nvk zYQ{a9gIWzx)=_kNfp)!5*-~31oQ*jl3KFpl*&t0NfeKxX_xM0Sw9cr@)4?t~fpe(j zg_9(1EaH^+3Tc$Yf9jc_OytlwiD|}}JWd!wn)5-{ra82Vlqz|FPlZnSf+Q6*lUmbQ zP>;5GhXS-pSx-EK8YU_#7a5Ir!SEdke|^;tiMhG>Jp=a-pO6jw;gI0w#YWV>a&B(^ z`qBK>;+wavqX#p7R5H%L`13w|y;3^3bx99j>c_{W?CQ$#D-uSfXX)op2j4Ewy3vi< zg$oO0?fL%geP?E_a_x)qcARnSvRRN@k8>JuSuk3r>0>IAb_|fur0j(CAlgjW63deb z08lj85u?_r2FjkF^ieSs{7_y^{ZQo9Dhf2H@`kd0)XaE0q5`&%oGOSm=uqZw>`GJX zR literal 0 HcmV?d00001 diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/cpt-doc2.pdf b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/cpt-doc2.pdf new file mode 100644 index 0000000000000000000000000000000000000000..823e3d107bcdd06b4885c7d42ea23d1c5923af3b GIT binary patch literal 1013 zcmah|L1+^}6upRtsZ>4KgIM`OgZ9wQ?q+wh(O7A+SzD|$O$yb<3fp87w`^z1?rcdt zD0oq%f_D`J4}ur5Cok$js67bYyom}bh)}5qDIUa`G)b5C;9O?+&Hne_zW*jsu$=*Q zFo_c1e{8%$Dk#8a=TIhtaE?WkgNpDuuNp!*DUOOW2%jfa&&{$1lof#(X=6~cz|z%0 zD=khZmlg=%f?Fpc8X7``RwbPlqa9x$bhyGjMq8YK9gmLyjeK7a$kH))iA5anI3+pe zNAq+G4nfK*wwvT!?)t1AYa=CYtEZe$9)iAAQ*Zw3@rC4 z*2gaW>tTKE;O%2CU#whvJ@t9wLh00z{F~?9dw#7>FJ#{x|FCxVW_V}gdhf)_y1Uxv zF8{gsyXWsZ{EXf|9dT~m`?l}N!~WBczI1Iym(MI7?d_5Nwz{^QHe!;qZb%#^ej1Mu zf01yn;v$@iN`6Oswtde(?2+6p}7zcSZM5IS)je#^lKHE^vWKG3XP1|r(CvO{iN;mRmT2JNl ev}vT&VPok3Uo`y_TQET46H z)LtFMzI@$WK`JP~re;wpg|N-)l!J=!h*u4vq>ERDGQuZG)pIkf0VPE!M$+ijEU@Bg zuay*UiwpAvaNezv5Dg3Ef9WYFl!p-Cs>!>7cx*cqNXY6zg@j;6EoJkFpDbb0@c;I?I37Us&+&fQ1WL#j3gFGf;yoc#D0|`OiXQ7m`HOI*~ sik8$JUDs{PHq5N0D`_i}G&D;!2mb#>lW)k~fHyHER0G9ggE^7ytkO literal 0 HcmV?d00001 diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/cpt-document-tool-call-results.bpmn b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/cpt-document-tool-call-results.bpmn new file mode 100644 index 00000000000..311f3cd8877 --- /dev/null +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/cpt-document-tool-call-results.bpmn @@ -0,0 +1,105 @@ + + + + + Flow_Start_To_Download + + + + + + + + + + + + + + + + + + + + + + Flow_Start_To_Download + Flow_Download_To_Agent + + + + + + + + + Document analysis agent for CPT viability testing + + + + Flow_Download_To_Agent + Flow_Agent_To_End + + Retrieves a single document for analysis. Returns one PDF document from the pre-downloaded files. + + + + + + Searches for and retrieves multiple documents. Returns a list of two PDF documents from the pre-downloaded files. + + + + + + Fetches a full report containing multiple documents in a nested structure. Returns a map with a summary string, an attachments list with two documents, and a metadata map containing a cover page document. + + + + + + + Flow_Agent_To_End + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From f4bd5209f8e5d16369071eac43d74216929325b8 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 23 Apr 2026 00:02:00 +0200 Subject: [PATCH 21/81] fix(agentic-ai): remove duplicate jackson-datatype-document dependency --- connectors/agentic-ai/pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/connectors/agentic-ai/pom.xml b/connectors/agentic-ai/pom.xml index 95b0bd65c93..4de3899f825 100644 --- a/connectors/agentic-ai/pom.xml +++ b/connectors/agentic-ai/pom.xml @@ -307,11 +307,6 @@ connector-object-mapper test - - io.camunda.connector - jackson-datatype-document - test - io.camunda.connector jackson-datatype-feel From e835f71922059069f77c6d564feec548b38eca7b Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 23 Apr 2026 07:42:10 +0200 Subject: [PATCH 22/81] test(e2e): add Ollama providers and convenience toggles for CPT test Add Ollama provider configs (qwen3.5, llama3.1:8b) with OLLAMA_URL env var. Add .disabled() toggle on ProviderConfig and a modelFilters allowlist for quickly focusing test runs on specific models without commenting code. --- .../cpt/DocumentToolCallResultsCPTTest.java | 47 +++++++++++++++++-- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsCPTTest.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsCPTTest.java index b15bdb40635..80b192dd655 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsCPTTest.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsCPTTest.java @@ -34,8 +34,10 @@ import io.camunda.zeebe.model.bpmn.BpmnModelInstance; import java.io.File; import java.time.Duration; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; @@ -65,6 +67,7 @@ * (also used for the judge LLM) *
  • {@code DOCKER_MODEL_RUNNER_URL} - OpenAI-compatible endpoint (default: * http://localhost:12434/engines/llama.cpp/v1) + *
  • {@code OLLAMA_URL} - Ollama OpenAI-compatible endpoint (default: http://localhost:11434/v1) * */ @SpringBootTest( @@ -222,6 +225,9 @@ void nestedStructureDocumentsFromToolCallResult( // --------------------------------------------------------------------------- static Stream providers() { + List> modelFilters = new ArrayList<>(); + modelFilters.add(p -> p.label().contains("gpt-4.1")); + return Stream.of( // OpenAI openai("gpt-4.1"), @@ -234,7 +240,15 @@ static Stream providers() { bedrock("global.anthropic.claude-sonnet-4-6"), bedrock("eu.anthropic.claude-haiku-4-5-20251001-v1:0"), // Docker Model Runner (OpenAI-compatible) - dockerModelRunner("ai/gemma4:latest")) + dockerModelRunner("ai/gemma4:latest").disabled(), + dockerModelRunner("ai/qwen3.6:latest").disabled(), + // Ollama (OpenAI-compatible) + ollama("qwen3.5:latest").disabled(), + ollama("llama3.1:8b").disabled()) + .filter( + providerConfig -> + modelFilters.isEmpty() + || modelFilters.stream().anyMatch(f -> f.test(providerConfig))) .filter(ProviderConfig::isEnabled); } @@ -304,6 +318,19 @@ static ProviderConfig dockerModelRunner(String model) { "provider.openaiCompatible.model.model", model)); } + // -- Ollama (OpenAI-compatible) -- + + static ProviderConfig ollama(String model) { + var url = System.getenv().getOrDefault("OLLAMA_URL", "http://localhost:11434/v1"); + return new ProviderConfig( + "ollama/" + model, + "OLLAMA_URL", + Map.of( + "provider.type", "openaiCompatible", + "provider.openaiCompatible.endpoint", url, + "provider.openaiCompatible.model.model", model)); + } + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- @@ -382,11 +409,23 @@ private static String envOrPlaceholder(String envVar) { // Provider config record // --------------------------------------------------------------------------- - record ProviderConfig(String label, String requiredEnvVar, Map properties) { + record ProviderConfig( + String label, String requiredEnvVar, Map properties, boolean enabled) { + + ProviderConfig(String label, String requiredEnvVar, Map properties) { + this(label, requiredEnvVar, properties, true); + } + + ProviderConfig disabled() { + return new ProviderConfig(label, requiredEnvVar, properties, false); + } boolean isEnabled() { - // Docker Model Runner doesn't need an API key env var, just the URL - if (requiredEnvVar.equals("DOCKER_MODEL_RUNNER_URL")) { + if (!enabled) { + return false; + } + // Local providers don't need an API key env var, just the URL + if (requiredEnvVar.equals("DOCKER_MODEL_RUNNER_URL") || requiredEnvVar.equals("OLLAMA_URL")) { return true; } return System.getenv(requiredEnvVar) != null; From 2a04d0992e39d18d7a9bd39da2b17642b471bfe7 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 23 Apr 2026 07:45:06 +0200 Subject: [PATCH 23/81] refactor(e2e): rename DocumentToolCallResultsCPTTest to DocumentToolCallResultsIT --- ...CallResultsCPTTest.java => DocumentToolCallResultsIT.java} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/{DocumentToolCallResultsCPTTest.java => DocumentToolCallResultsIT.java} (99%) diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsCPTTest.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsIT.java similarity index 99% rename from connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsCPTTest.java rename to connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsIT.java index 80b192dd655..1189cfea522 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsCPTTest.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsIT.java @@ -90,9 +90,9 @@ @WireMockTest @Import(CamundaDocumentTestConfiguration.class) @Disabled("Manual viability test - requires real LLM API keys via environment variables") -class DocumentToolCallResultsCPTTest { +class DocumentToolCallResultsIT { - private static final Logger LOG = LoggerFactory.getLogger(DocumentToolCallResultsCPTTest.class); + private static final Logger LOG = LoggerFactory.getLogger(DocumentToolCallResultsIT.class); private static final String ELEMENT_TEMPLATE_PATH = "../../connectors/agentic-ai/element-templates/agenticai-aiagent-job-worker.json"; From 9582a512db218fd12a3d57ded1409a5f245370fd Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 23 Apr 2026 07:52:06 +0200 Subject: [PATCH 24/81] fix(e2e): remove unused parameter and variable in DocumentToolCallResultsIT --- .../cpt/DocumentToolCallResultsIT.java | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsIT.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsIT.java index 1189cfea522..80693f9f7a5 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsIT.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsIT.java @@ -141,8 +141,7 @@ void singleDocumentFromToolCallResult(ProviderConfig provider, WireMockRuntimeIn provider, "Use the Analyze_Single_Document tool to retrieve a document, then tell me " + "what project it mentions and when it launched.", - List.of(wireMock.getHttpBaseUrl() + "/cpt-doc1.pdf"), - wireMock); + List.of(wireMock.getHttpBaseUrl() + "/cpt-doc1.pdf")); assertThat(processInstance) .withAssertionTimeout(PROCESS_TIMEOUT) @@ -170,8 +169,7 @@ void multipleDocumentsFromToolCallResult(ProviderConfig provider, WireMockRuntim "Use the Search_Documents tool to find documents and summarize what each one says.", List.of( wireMock.getHttpBaseUrl() + "/cpt-doc1.pdf", - wireMock.getHttpBaseUrl() + "/cpt-doc2.pdf"), - wireMock); + wireMock.getHttpBaseUrl() + "/cpt-doc2.pdf")); assertThat(processInstance) .withAssertionTimeout(PROCESS_TIMEOUT) @@ -202,8 +200,7 @@ void nestedStructureDocumentsFromToolCallResult( List.of( wireMock.getHttpBaseUrl() + "/cpt-doc1.pdf", wireMock.getHttpBaseUrl() + "/cpt-doc2.pdf", - wireMock.getHttpBaseUrl() + "/cpt-doc3.pdf"), - wireMock); + wireMock.getHttpBaseUrl() + "/cpt-doc3.pdf")); assertThat(processInstance) .withAssertionTimeout(PROCESS_TIMEOUT) @@ -336,17 +333,11 @@ static ProviderConfig ollama(String model) { // --------------------------------------------------------------------------- private io.camunda.client.api.response.ProcessInstanceEvent startProcess( - ProviderConfig provider, - String userPrompt, - List downloadUrls, - WireMockRuntimeInfo wireMock) { + ProviderConfig provider, String userPrompt, List downloadUrls) { var model = buildModel(provider); // deploy and wait for process definition to be available - var zeebeTest = - io.camunda.connector.e2e.ZeebeTest.with(camundaClient) - .awaitCompleteTopology() - .deploy(model); + io.camunda.connector.e2e.ZeebeTest.with(camundaClient).awaitCompleteTopology().deploy(model); return camundaClient .newCreateInstanceCommand() From 94ef8c9893aa72e26bf5b480006f9a97adde5d3e Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 23 Apr 2026 07:57:09 +0200 Subject: [PATCH 25/81] refactor(e2e): move IT out of cpt package and rename test resources Move DocumentToolCallResultsIT to io.camunda.connector.e2e.agenticai.aiagent package, rename PDF fixtures to descriptive names (project-launch.pdf, headcount-report.pdf, author-info.pdf) under document-tool-call-results/ directory, and drop the cpt- prefix from the BPMN file. --- .../{cpt => }/DocumentToolCallResultsIT.java | 34 +++++++++--------- .../author-info.pdf} | Bin .../headcount-report.pdf} | Bin .../project-launch.pdf} | Bin ...s.bpmn => document-tool-call-results.bpmn} | 0 5 files changed, 18 insertions(+), 16 deletions(-) rename connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/{cpt => }/DocumentToolCallResultsIT.java (93%) rename connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/{cpt-doc3.pdf => document-tool-call-results/author-info.pdf} (100%) rename connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/{cpt-doc2.pdf => document-tool-call-results/headcount-report.pdf} (100%) rename connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/{cpt-doc1.pdf => document-tool-call-results/project-launch.pdf} (100%) rename connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/{cpt-document-tool-call-results.bpmn => document-tool-call-results.bpmn} (100%) diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsIT.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/DocumentToolCallResultsIT.java similarity index 93% rename from connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsIT.java rename to connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/DocumentToolCallResultsIT.java index 80693f9f7a5..c5e44d4be2c 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/cpt/DocumentToolCallResultsIT.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/DocumentToolCallResultsIT.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.camunda.connector.e2e.agenticai.aiagent.cpt; +package io.camunda.connector.e2e.agenticai.aiagent; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.get; @@ -97,7 +97,12 @@ class DocumentToolCallResultsIT { private static final String ELEMENT_TEMPLATE_PATH = "../../connectors/agentic-ai/element-templates/agenticai-aiagent-job-worker.json"; - private static final String BPMN_RESOURCE = "classpath:cpt-document-tool-call-results.bpmn"; + private static final String BPMN_RESOURCE = "classpath:document-tool-call-results.bpmn"; + + private static final String DOC_DIR = "document-tool-call-results/"; + private static final String DOC_PROJECT_LAUNCH = DOC_DIR + "project-launch.pdf"; + private static final String DOC_HEADCOUNT_REPORT = DOC_DIR + "headcount-report.pdf"; + private static final String DOC_AUTHOR_INFO = DOC_DIR + "author-info.pdf"; private static final String SYSTEM_PROMPT = "You are a document analyst. Use the available tools to retrieve and analyze documents. " @@ -117,14 +122,11 @@ void clearDocumentStore() { @BeforeEach void setupPdfStubs() { - for (int i = 1; i <= 3; i++) { - String filename = "cpt-doc" + i + ".pdf"; + for (var doc : List.of(DOC_PROJECT_LAUNCH, DOC_HEADCOUNT_REPORT, DOC_AUTHOR_INFO)) { stubFor( - get(urlPathEqualTo("/" + filename)) + get(urlPathEqualTo("/" + doc)) .willReturn( - aResponse() - .withBodyFile(filename) - .withHeader("Content-Type", "application/pdf"))); + aResponse().withBodyFile(doc).withHeader("Content-Type", "application/pdf"))); } } @@ -141,7 +143,7 @@ void singleDocumentFromToolCallResult(ProviderConfig provider, WireMockRuntimeIn provider, "Use the Analyze_Single_Document tool to retrieve a document, then tell me " + "what project it mentions and when it launched.", - List.of(wireMock.getHttpBaseUrl() + "/cpt-doc1.pdf")); + List.of(wireMock.getHttpBaseUrl() + "/" + DOC_PROJECT_LAUNCH)); assertThat(processInstance) .withAssertionTimeout(PROCESS_TIMEOUT) @@ -168,8 +170,8 @@ void multipleDocumentsFromToolCallResult(ProviderConfig provider, WireMockRuntim provider, "Use the Search_Documents tool to find documents and summarize what each one says.", List.of( - wireMock.getHttpBaseUrl() + "/cpt-doc1.pdf", - wireMock.getHttpBaseUrl() + "/cpt-doc2.pdf")); + wireMock.getHttpBaseUrl() + "/" + DOC_PROJECT_LAUNCH, + wireMock.getHttpBaseUrl() + "/" + DOC_HEADCOUNT_REPORT)); assertThat(processInstance) .withAssertionTimeout(PROCESS_TIMEOUT) @@ -198,9 +200,9 @@ void nestedStructureDocumentsFromToolCallResult( "Use the Fetch_Report tool to get the full report and describe the content " + "of every document in it, including attachments and the cover page.", List.of( - wireMock.getHttpBaseUrl() + "/cpt-doc1.pdf", - wireMock.getHttpBaseUrl() + "/cpt-doc2.pdf", - wireMock.getHttpBaseUrl() + "/cpt-doc3.pdf")); + wireMock.getHttpBaseUrl() + "/" + DOC_PROJECT_LAUNCH, + wireMock.getHttpBaseUrl() + "/" + DOC_HEADCOUNT_REPORT, + wireMock.getHttpBaseUrl() + "/" + DOC_AUTHOR_INFO)); assertThat(processInstance) .withAssertionTimeout(PROCESS_TIMEOUT) @@ -223,7 +225,7 @@ void nestedStructureDocumentsFromToolCallResult( static Stream providers() { List> modelFilters = new ArrayList<>(); - modelFilters.add(p -> p.label().contains("gpt-4.1")); + // modelFilters.add(p -> p.label().contains("gpt-4.1")); return Stream.of( // OpenAI @@ -240,7 +242,7 @@ static Stream providers() { dockerModelRunner("ai/gemma4:latest").disabled(), dockerModelRunner("ai/qwen3.6:latest").disabled(), // Ollama (OpenAI-compatible) - ollama("qwen3.5:latest").disabled(), + ollama("qwen3.6:latest").disabled(), ollama("llama3.1:8b").disabled()) .filter( providerConfig -> diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/cpt-doc3.pdf b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/document-tool-call-results/author-info.pdf similarity index 100% rename from connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/cpt-doc3.pdf rename to connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/document-tool-call-results/author-info.pdf diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/cpt-doc2.pdf b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/document-tool-call-results/headcount-report.pdf similarity index 100% rename from connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/cpt-doc2.pdf rename to connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/document-tool-call-results/headcount-report.pdf diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/cpt-doc1.pdf b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/document-tool-call-results/project-launch.pdf similarity index 100% rename from connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/cpt-doc1.pdf rename to connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/__files/document-tool-call-results/project-launch.pdf diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/cpt-document-tool-call-results.bpmn b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/document-tool-call-results.bpmn similarity index 100% rename from connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/cpt-document-tool-call-results.bpmn rename to connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/resources/document-tool-call-results.bpmn From 11a505f35aff8f1cda76e681302196fcd519fda6 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 23 Apr 2026 09:26:14 +0200 Subject: [PATCH 26/81] docs(agentic-ai): update ADR-004 status to Implemented --- .../docs/adr/004-document-handling-in-tool-call-results.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md b/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md index 1d8cd804c59..ca6cd0ba85c 100644 --- a/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md +++ b/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md @@ -5,7 +5,7 @@ ## Status -**Proposed**. +**Implemented** (PR #6999). ## Context and Problem Statement From fb933399c29bd4eb98b2a49f530bde45e1d1d4b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Apr 2026 12:58:08 +0000 Subject: [PATCH 27/81] refactor(agentic-ai): per-handler tool call result document extraction Move document extraction off the raw-Map content tree and into the GatewayToolHandler SPI. Each handler now exposes extractDocuments and walks its own typed content (sealed-type switch); the generic content tree walker stays as the default fallback for plain BPMN tools and handlers that return raw maps. Removes the constraint that gateway handlers must keep raw Map content solely so the instanceof-based walker can find Documents inside them. * New ContentTreeDocumentWalker: public utility extracted from the old ToolCallResultDocumentExtractor walker. Public so third-party handlers whose typed content embeds raw subtrees can reuse it. * GatewayToolHandler.extractDocuments(ToolCallResult): default delegates to ContentTreeDocumentWalker, override to walk a typed structure. * GatewayToolHandlerRegistry.extractDocuments routes per-result to the responsible handler, falling back to the walker. * ToolCallResultDocumentExtractor becomes a thin coordinator that iterates ToolCallResults and calls the registry; constructor now takes the registry. * MCP handler restores typed McpClientCallToolResult content and walks McpDocumentContent and McpEmbeddedResourceContent.BlobDocumentResource. Drops the getRawMcpContent workaround. * A2A handler restores typed A2aSendMessageResult content and walks A2aMessage.contents, A2aTask.artifacts, and A2aTask.history (recursive). Drops the raw-content preservation comment. * ADR-004 updated: replaces the "must preserve raw content" subsection with a "Per-handler document extraction" subsection. * Tests split into ContentTreeDocumentWalkerTest (walker behaviour) and ToolCallResultDocumentExtractorTest (registry routing). Per-variant unit coverage added on McpClientGatewayToolHandlerTest and A2aGatewayToolHandlerTest. https://claude.ai/code/session_01SM8HzedSAVWqnDaEKrmCpR --- ...-document-handling-in-tool-call-results.md | 44 ++- .../agentic/tool/A2aGatewayToolHandler.java | 60 ++- .../agent/AgentMessagesHandlerImpl.java | 4 +- .../agent/ContentTreeDocumentWalker.java | 60 +++ .../ToolCallResultDocumentExtractor.java | 54 +-- .../aiagent/tool/GatewayToolHandler.java | 19 + .../tool/GatewayToolHandlerRegistry.java | 9 + .../tool/GatewayToolHandlerRegistryImpl.java | 12 + .../AgenticAiConnectorsAutoConfiguration.java | 5 +- .../McpClientGatewayToolHandler.java | 41 ++- .../tool/A2aGatewayToolHandlerTest.java | 124 ++++++- .../agent/AgentMessagesHandlerTest.java | 16 +- .../agent/ContentTreeDocumentWalkerTest.java | 142 ++++++++ .../ToolCallResultDocumentExtractorTest.java | 341 ++++++++---------- .../McpClientGatewayToolHandlerTest.java | 154 +++++++- 15 files changed, 788 insertions(+), 297 deletions(-) create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalker.java create mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalkerTest.java diff --git a/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md b/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md index ca6cd0ba85c..a76723931a5 100644 --- a/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md +++ b/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md @@ -66,8 +66,10 @@ correlate the reference with the actual content in the user message. **Generic layer** (`AgentMessagesHandlerImpl`): -1. After building the `ToolCallResultMessage`, scan each `ToolCallResult.content()` tree for `Document` instances using - `ToolCallResultDocumentExtractor` (handles `Map`, `Collection`, `Object[]`, and `Document`). +1. After building the `ToolCallResultMessage`, scan each `ToolCallResult.content()` for `Document` instances using + `ToolCallResultDocumentExtractor`. The extractor delegates per result to the responsible + `GatewayToolHandler.extractDocuments(...)` (with `ContentTreeDocumentWalker` as the default fallback for results not + managed by any gateway handler) — see "Per-handler document extraction" below. 2. Build a single `UserMessage` (metadata: `toolCallDocuments=true`) containing a preamble, per-document XML tags with correlation attributes, and `DocumentContent` blocks. 3. Apply the same extraction to event messages: prepend `` XML tags before each `DocumentContent` block in the @@ -114,22 +116,32 @@ The synthetic document `UserMessage` (identified by `UserMessage.METADATA_TOOL_C the `maxMessages` context window limit. When evicting messages, the document `UserMessage` is removed together with its associated `ToolCallResultMessage` — it is never orphaned. -### Gateway tool handlers must preserve raw content +### Per-handler document extraction Gateway tool handlers (MCP, A2A) transform `ToolCallResult` objects — renaming tool calls with fully qualified -identifiers and processing the result content. When tool call results arrive from the process engine, `Document` -instances in the content tree have already been deserialized by the connectors `ObjectMapper` (which recognizes -`camunda.document.type` references). The content is a raw tree of `Map`, `List`, `String`, and `Document` objects. - -Gateway handlers **must not** convert this raw content to typed domain objects (e.g., `McpClientCallToolResult`, -`A2aSendMessageResult`) and put the typed object back as `ToolCallResult.content()`. The `ToolCallResultDocumentExtractor` -walks the content tree using `instanceof` checks for `Document`, `Map`, `Collection`, and `Object[]`. Typed records and POJOs are -invisible to it — documents nested inside them would not be extracted. - -Instead, handlers should: -1. Convert to typed objects only when needed to extract metadata (e.g., MCP tool name for the fully qualified identifier). -2. Pass the **raw content** through to the output `ToolCallResult`, preserving the original `Map`/`List`/`Document` tree. -3. For simple text-only results, extract the text string directly (optimization to avoid unnecessary JSON wrapping). +identifiers and producing typed domain objects (`McpClientCallToolResult`, `A2aSendMessageResult`) as the transformed +`content()`. Each handler exposes a domain-specific `extractDocuments(ToolCallResult)` method on the +`GatewayToolHandler` SPI that walks its own typed structure (sealed-type `switch`) to collect `Document` instances: + +* MCP: walks `McpClientCallToolResult.content()` and matches `McpDocumentContent` and + `McpEmbeddedResourceContent.BlobDocumentResource`. +* A2A: walks `A2aSendMessageResult` (sealed `A2aMessage | A2aTask`), descending into artifacts and recursive task + history to collect `DocumentContent` instances. + +`ToolCallResultDocumentExtractor` routes each result to the handler that manages it (via +`GatewayToolHandlerRegistry.handlerForToolDefinition(toolName)`). When no handler claims the result — typical for plain +BPMN tool calls whose `content()` is a raw `Map`/`List`/`Document` tree from the engine — the extractor falls back to +`ContentTreeDocumentWalker`, which performs the original `instanceof`-based recursion over `Map`, `Collection`, +`Object[]`, and `Document`. + +The walker is also public: handler implementations whose typed content embeds raw user-generated subtrees (e.g. opaque +`Map` payloads from a downstream system) can call `ContentTreeDocumentWalker.INSTANCE` for those parts. + +The default `GatewayToolHandler.extractDocuments` implementation delegates to the walker, so third-party handlers that +return raw content do not need to override anything. + +For simple text-only MCP results, the handler still extracts the text string directly (optimization to avoid +unnecessary JSON wrapping); in that case `extractDocuments` returns an empty list since the content is a `String`. ### Future optimization (out of scope) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandler.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandler.java index 4c46f8299e9..fd50e48867a 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandler.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandler.java @@ -12,20 +12,29 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.connector.agenticai.a2a.client.common.model.result.A2aArtifact; +import io.camunda.connector.agenticai.a2a.client.common.model.result.A2aMessage; +import io.camunda.connector.agenticai.a2a.client.common.model.result.A2aSendMessageResult; +import io.camunda.connector.agenticai.a2a.client.common.model.result.A2aTask; import io.camunda.connector.agenticai.a2a.client.outbound.model.A2aStandaloneOperationConfiguration; import io.camunda.connector.agenticai.a2a.client.outbound.model.A2aStandaloneOperationConfiguration.FetchAgentCardOperationConfiguration; import io.camunda.connector.agenticai.aiagent.model.AgentContext; import io.camunda.connector.agenticai.aiagent.tool.GatewayToolDefinitionUpdates; import io.camunda.connector.agenticai.aiagent.tool.GatewayToolDiscoveryInitiationResult; import io.camunda.connector.agenticai.aiagent.tool.GatewayToolHandler; +import io.camunda.connector.agenticai.model.message.content.Content; +import io.camunda.connector.agenticai.model.message.content.DocumentContent; import io.camunda.connector.agenticai.model.tool.GatewayToolDefinition; import io.camunda.connector.agenticai.model.tool.ToolCall; import io.camunda.connector.agenticai.model.tool.ToolCallResult; import io.camunda.connector.agenticai.model.tool.ToolDefinition; import io.camunda.connector.agenticai.util.CollectionUtils; +import io.camunda.connector.api.document.Document; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -219,18 +228,59 @@ private List getA2aClientIds(AgentContext agentContext) { private ToolCallResult toolCallResultFromA2aSendMessage(ToolCallResult toolCallResult) { final var identifier = new A2aToolCallIdentifier(toolCallResult.name()); + final var typedContent = + objectMapper.convertValue(toolCallResult.content(), A2aSendMessageResult.class); - // use raw content from the original tool call result (preserving document references - // as deserialized by the engine) rather than the typed A2aSendMessageResult, which would - // lose document reference fidelity during re-serialization. The ToolCallResultDocumentExtractor - // can only walk Map/Collection/Document — typed records are invisible to it. return ToolCallResult.builder() .id(toolCallResult.id()) .name(identifier.fullyQualifiedName()) - .content(toolCallResult.content()) + .content(typedContent) .build(); } + @Override + public List extractDocuments(ToolCallResult toolCallResult) { + if (!(toolCallResult.content() instanceof A2aSendMessageResult result)) { + return List.of(); + } + + final var documents = new ArrayList(); + collectDocumentsFromResult(result, documents); + return documents; + } + + private void collectDocumentsFromResult(A2aSendMessageResult result, List documents) { + switch (result) { + case A2aMessage message -> collectDocumentsFromContents(message.contents(), documents); + case A2aTask task -> { + Optional.ofNullable(task.artifacts()) + .ifPresent( + artifacts -> + artifacts.forEach( + artifact -> collectDocumentsFromArtifact(artifact, documents))); + Optional.ofNullable(task.history()) + .ifPresent( + history -> + history.forEach(message -> collectDocumentsFromResult(message, documents))); + } + } + } + + private void collectDocumentsFromArtifact(A2aArtifact artifact, List documents) { + collectDocumentsFromContents(artifact.contents(), documents); + } + + private void collectDocumentsFromContents(List contents, List documents) { + if (contents == null) { + return; + } + for (Content content : contents) { + if (content instanceof DocumentContent documentContent) { + documents.add(documentContent.document()); + } + } + } + /** * Creates a description for the tool definition by serializing the agent card and appending a * fixed explanation of the tool call result structure. diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java index 5c03b521315..884340835b7 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java @@ -258,7 +258,9 @@ private Message createEventMessage( } // extract documents from event content and add as document content blocks - var eventDocuments = documentExtractor.extractDocumentsFromContent(eventContent); + // events originate from BPMN event sub-processes (not a gateway handler), so walk the raw tree + var eventDocuments = + ContentTreeDocumentWalker.INSTANCE.extractDocumentsFromContent(eventContent); if (!eventDocuments.isEmpty()) { for (var doc : eventDocuments) { userMessageContent.add(textContent(DocumentXmlTag.from(doc).toXml())); diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalker.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalker.java new file mode 100644 index 00000000000..5a3c6756492 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalker.java @@ -0,0 +1,60 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.agent; + +import io.camunda.connector.api.document.Document; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * Recursively walks an arbitrary content tree and collects {@link Document} instances. Handles + * {@link Document}, {@link Map}, {@link Collection}, and {@code Object[]}; all other types are + * skipped. + * + *

    This is the default extraction strategy used by {@link + * io.camunda.connector.agenticai.aiagent.tool.GatewayToolHandler#extractDocuments} and the fallback + * for tool call results not managed by any gateway handler. It is intentionally public so that + * gateway handler implementations whose typed content wraps raw user-generated subtrees (e.g. + * arbitrary maps from a downstream system) can delegate to it for those subtrees. + */ +public final class ContentTreeDocumentWalker { + + public static final ContentTreeDocumentWalker INSTANCE = new ContentTreeDocumentWalker(); + + public List extractDocumentsFromContent(Object content) { + if (content == null) { + return List.of(); + } + + final var documents = new ArrayList(); + collectDocuments(content, documents); + return documents; + } + + private void collectDocuments(Object node, List documents) { + if (node == null) { + return; + } + + switch (node) { + case Document document -> documents.add(document); + case Map map -> map.values().forEach(value -> collectDocuments(value, documents)); + case Collection collection -> + collection.forEach(element -> collectDocuments(element, documents)); + case Object[] array -> { + for (Object element : array) { + collectDocuments(element, documents); + } + } + default -> { + // scalars and other types - nothing to extract + } + } + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java index b3c7bbbacaa..4fb093c9fe0 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java @@ -6,17 +6,20 @@ */ package io.camunda.connector.agenticai.aiagent.agent; +import io.camunda.connector.agenticai.aiagent.tool.GatewayToolHandlerRegistry; import io.camunda.connector.agenticai.model.tool.ToolCallResult; import io.camunda.connector.api.document.Document; import java.util.ArrayList; -import java.util.Collection; import java.util.List; -import java.util.Map; import org.apache.commons.lang3.StringUtils; /** - * Extracts {@link Document} instances from tool call result content trees. Documents can appear at - * any level: as the root content, within maps, or within lists. + * Extracts {@link Document} instances from a list of tool call results, grouped by tool call. + * + *

    The actual extraction strategy is delegated to the {@link GatewayToolHandlerRegistry}: each + * tool call result is routed to its responsible {@code GatewayToolHandler} (which may walk a typed + * domain object), with the generic content-tree walker as the default fallback for tool calls not + * managed by any gateway handler. */ public class ToolCallResultDocumentExtractor { @@ -24,6 +27,12 @@ public class ToolCallResultDocumentExtractor { public record ToolCallDocuments( String toolCallId, String toolCallName, List documents) {} + private final GatewayToolHandlerRegistry gatewayToolHandlers; + + public ToolCallResultDocumentExtractor(GatewayToolHandlerRegistry gatewayToolHandlers) { + this.gatewayToolHandlers = gatewayToolHandlers; + } + /** * Extracts all {@link Document} instances from the given tool call results, grouped by tool call. * Tool calls without documents are excluded from the result. Order is preserved. @@ -32,7 +41,7 @@ public List extractDocuments(List toolCallRes final var result = new ArrayList(); for (ToolCallResult toolCallResult : toolCallResults) { - final var documents = extractDocumentsFromContent(toolCallResult.content()); + final var documents = gatewayToolHandlers.extractDocuments(toolCallResult); if (!documents.isEmpty()) { result.add( new ToolCallDocuments( @@ -44,39 +53,4 @@ public List extractDocuments(List toolCallRes return result; } - - /** - * Recursively extracts all {@link Document} instances from an arbitrary object tree. Handles - * {@link Document}, {@link Map}, {@link List}/{@link Collection}, and skips all other types. - */ - public List extractDocumentsFromContent(Object content) { - if (content == null) { - return List.of(); - } - - final var documents = new ArrayList(); - collectDocuments(content, documents); - return documents; - } - - private void collectDocuments(Object node, List documents) { - if (node == null) { - return; - } - - switch (node) { - case Document document -> documents.add(document); - case Map map -> map.values().forEach(value -> collectDocuments(value, documents)); - case Collection collection -> - collection.forEach(element -> collectDocuments(element, documents)); - case Object[] array -> { - for (Object element : array) { - collectDocuments(element, documents); - } - } - default -> { - // scalars and other types - nothing to extract - } - } - } } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandler.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandler.java index 27a75df9820..bf6d04cf7cc 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandler.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandler.java @@ -7,10 +7,12 @@ package io.camunda.connector.agenticai.aiagent.tool; import io.camunda.connector.agenticai.adhoctoolsschema.schema.GatewayToolDefinitionResolver; +import io.camunda.connector.agenticai.aiagent.agent.ContentTreeDocumentWalker; import io.camunda.connector.agenticai.aiagent.model.AgentContext; import io.camunda.connector.agenticai.model.tool.GatewayToolDefinition; import io.camunda.connector.agenticai.model.tool.ToolCallResult; import io.camunda.connector.agenticai.model.tool.ToolDefinition; +import io.camunda.connector.api.document.Document; import java.util.List; /** @@ -54,4 +56,21 @@ boolean allToolDiscoveryResultsPresent( /** Handles tool discovery results matching the handlesToolDiscoveryResult() predicate. */ List handleToolDiscoveryResults( AgentContext agentContext, List toolCallResults); + + /** + * Extracts {@link Document} instances from a tool call result managed by this handler. Called + * after {@link #transformToolCallResults} so the {@code toolCallResult.content()} carries this + * handler's transformed shape (typically a typed domain object). + * + *

    The default implementation delegates to {@link ContentTreeDocumentWalker}, which walks + * {@link java.util.Map}, {@link java.util.Collection}, {@code Object[]} and {@link Document} + * nodes. This is sufficient for handlers whose transformed content remains a raw tree. + * + *

    Handlers that return typed records or POJOs as content must override this method and walk + * their own structure (typically via a sealed-type switch). For mixed shapes — typed wrappers + * around nested raw subtrees — call {@link ContentTreeDocumentWalker#INSTANCE} on the raw parts. + */ + default List extractDocuments(ToolCallResult toolCallResult) { + return ContentTreeDocumentWalker.INSTANCE.extractDocumentsFromContent(toolCallResult.content()); + } } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandlerRegistry.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandlerRegistry.java index b0ad452f694..8dec4a57c41 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandlerRegistry.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandlerRegistry.java @@ -9,6 +9,7 @@ import io.camunda.connector.agenticai.aiagent.model.AgentContext; import io.camunda.connector.agenticai.model.tool.GatewayToolDefinition; import io.camunda.connector.agenticai.model.tool.ToolCallResult; +import io.camunda.connector.api.document.Document; import java.util.List; import java.util.Map; import java.util.Optional; @@ -42,4 +43,12 @@ boolean allToolDiscoveryResultsPresent( GatewayToolDiscoveryResult handleToolDiscoveryResults( AgentContext agentContext, List toolCallResults); + + /** + * Extracts {@link Document} instances from a tool call result by routing to the responsible + * gateway handler, falling back to the default content-tree walker when no handler manages the + * tool. Expected to be called on already-transformed tool call results (i.e. after {@link + * #transformToolCallResults}). + */ + List extractDocuments(ToolCallResult toolCallResult); } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandlerRegistryImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandlerRegistryImpl.java index 89dc82e7354..52f52c8e86b 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandlerRegistryImpl.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandlerRegistryImpl.java @@ -6,11 +6,13 @@ */ package io.camunda.connector.agenticai.aiagent.tool; +import io.camunda.connector.agenticai.aiagent.agent.ContentTreeDocumentWalker; import io.camunda.connector.agenticai.aiagent.model.AgentContext; import io.camunda.connector.agenticai.model.tool.GatewayToolDefinition; import io.camunda.connector.agenticai.model.tool.ToolCall; import io.camunda.connector.agenticai.model.tool.ToolCallResult; import io.camunda.connector.agenticai.model.tool.ToolDefinition; +import io.camunda.connector.api.document.Document; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -163,4 +165,14 @@ public List transformToolCallResults( return transformedToolCallResults; } + + @Override + public List extractDocuments(ToolCallResult toolCallResult) { + return handlerForToolDefinition(toolCallResult.name()) + .map(handler -> handler.extractDocuments(toolCallResult)) + .orElseGet( + () -> + ContentTreeDocumentWalker.INSTANCE.extractDocumentsFromContent( + toolCallResult.content())); + } } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java index 1d15cc8c203..9d2efd3f823 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java @@ -233,8 +233,9 @@ public SystemPromptComposer aiAgentSystemPromptComposer( @Bean @ConditionalOnMissingBean - public ToolCallResultDocumentExtractor toolCallResultDocumentExtractor() { - return new ToolCallResultDocumentExtractor(); + public ToolCallResultDocumentExtractor toolCallResultDocumentExtractor( + GatewayToolHandlerRegistry gatewayToolHandlers) { + return new ToolCallResultDocumentExtractor(gatewayToolHandlers); } @Bean diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandler.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandler.java index c5eb2f3a421..7f9c259e50f 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandler.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandler.java @@ -17,6 +17,9 @@ import io.camunda.connector.agenticai.mcp.client.model.McpClientOperation; import io.camunda.connector.agenticai.mcp.client.model.McpClientOperationDefinitions; import io.camunda.connector.agenticai.mcp.client.model.McpToolDefinition; +import io.camunda.connector.agenticai.mcp.client.model.content.McpDocumentContent; +import io.camunda.connector.agenticai.mcp.client.model.content.McpEmbeddedResourceContent; +import io.camunda.connector.agenticai.mcp.client.model.content.McpEmbeddedResourceContent.BlobDocumentResource; import io.camunda.connector.agenticai.mcp.client.model.content.McpTextContent; import io.camunda.connector.agenticai.mcp.client.model.result.McpClientCallToolResult; import io.camunda.connector.agenticai.mcp.client.model.result.McpClientListToolsResult; @@ -26,7 +29,9 @@ import io.camunda.connector.agenticai.model.tool.ToolDefinition; import io.camunda.connector.agenticai.util.CollectionUtils; import io.camunda.connector.agenticai.util.ObjectMapperConstants; +import io.camunda.connector.api.document.Document; import io.camunda.connector.api.error.ConnectorException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -259,27 +264,35 @@ private ToolCallResult toolCallResultFromMcpToolCall(ToolCallResult toolCallResu && callToolResult.content().getFirst() instanceof McpTextContent textContent) { toolCallResultBuilder.content(textContent.text()); } else { - // use the raw content from the original tool call result (preserving document references - // as deserialized by the engine) rather than the typed McpContent list, which may lose - // document reference fidelity during re-serialization. - toolCallResultBuilder.content(getRawMcpContent(toolCallResult)); + toolCallResultBuilder.content(callToolResult); } return toolCallResultBuilder.build(); } - private Object getRawMcpContent(ToolCallResult toolCallResult) { - if (toolCallResult.content() instanceof Map map) { - var content = map.get("content"); - if (content == null && !map.isEmpty()) { - LOGGER.warn( - "MCP tool call result map has no 'content' key but contains keys: {}. " - + "Documents may be lost if the response structure has changed.", - map.keySet()); + @Override + public List extractDocuments(ToolCallResult toolCallResult) { + if (!(toolCallResult.content() instanceof McpClientCallToolResult callToolResult)) { + // string-content optimization or unmanaged shape — nothing to walk + return List.of(); + } + + final var documents = new ArrayList(); + for (var content : callToolResult.content()) { + switch (content) { + case McpDocumentContent documentContent -> documents.add(documentContent.document()); + case McpEmbeddedResourceContent embeddedResourceContent -> { + if (embeddedResourceContent.resource() + instanceof BlobDocumentResource blobDocumentResource) { + documents.add(blobDocumentResource.document()); + } + } + default -> { + // text/object/blob/resourceLink — no documents + } } - return content; } - return toolCallResult.content(); + return documents; } private Map mcpClientOperationAsMap(McpClientOperation mcpClientOperation) { diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandlerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandlerTest.java index 8e02fda91d6..003baadef41 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandlerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandlerTest.java @@ -23,8 +23,8 @@ import io.camunda.connector.agenticai.a2a.client.common.model.result.A2aMessage; import io.camunda.connector.agenticai.a2a.client.common.model.result.A2aTask; import io.camunda.connector.agenticai.a2a.client.common.model.result.A2aTaskStatus; -import io.camunda.connector.agenticai.aiagent.agent.ToolCallResultDocumentExtractor; import io.camunda.connector.agenticai.aiagent.model.AgentContext; +import io.camunda.connector.agenticai.model.message.content.DocumentContent; import io.camunda.connector.agenticai.model.message.content.TextContent; import io.camunda.connector.agenticai.model.tool.GatewayToolDefinition; import io.camunda.connector.agenticai.model.tool.ToolCall; @@ -461,29 +461,133 @@ void handlesEmptyA2aClientsList() { } @Test - void preservesRawContentWithDocuments_forDocumentExtraction() { + void convertsRawMapContentIntoTypedA2aSendMessageResult() { var agentContext = AgentContext.empty().withProperty(PROPERTY_A2A_CLIENTS, List.of("a2a1")); - var document = mock(Document.class); - // simulate raw content as it arrives from the engine: a Map/List tree with Document instances + // simulate raw content as it arrives from the engine: a Map tree var rawContent = Map.of( "kind", "message", + "role", + "agent", + "messageId", + "msg-1", + "contextId", + "ctx-1", "contents", - List.of(Map.of("type", "document", "document", document))); + List.of(Map.of("type", "text", "text", "Agent reply"))); var toolCallResults = List.of(createToolCallResultWithContent("call1", "a2a1", rawContent)); var result = handler.transformToolCallResults(agentContext, toolCallResults); assertThat(result).hasSize(1); assertThat(result.getFirst().name()).isEqualTo("A2A_a2a1"); + assertThat(result.getFirst().content()) + .isInstanceOfSatisfying( + A2aMessage.class, + message -> { + assertThat(message.role()).isEqualTo(A2aMessage.Role.AGENT); + assertThat(message.contents()) + .containsExactly(TextContent.textContent("Agent reply")); + }); + } + } + + @Nested + class ExtractDocuments { + + @Test + void extractsDocumentFromA2aMessageContents() { + var document = mock(Document.class); + var message = + A2aMessage.builder() + .role(A2aMessage.Role.AGENT) + .messageId("msg-1") + .contextId("ctx-1") + .contents( + List.of( + TextContent.textContent("description"), + DocumentContent.documentContent(document))) + .build(); + var toolCallResult = createToolCallResultWithContent("call1", "A2A_a2a1", message); + + var documents = handler.extractDocuments(toolCallResult); + + assertThat(documents).containsExactly(document); + } + + @Test + void extractsDocumentsFromA2aTaskArtifactsAndHistory() { + var artifactDoc = mock(Document.class); + var historyDoc = mock(Document.class); + + var artifact = + A2aArtifact.builder() + .artifactId("art-1") + .contents(List.of(DocumentContent.documentContent(artifactDoc))) + .build(); + var historyMessage = + A2aMessage.builder() + .role(A2aMessage.Role.AGENT) + .messageId("msg-1") + .contextId("ctx-1") + .contents(List.of(DocumentContent.documentContent(historyDoc))) + .build(); + var task = + A2aTask.builder() + .id("task-1") + .contextId("ctx-1") + .status(A2aTaskStatus.builder().state(A2aTaskStatus.TaskState.COMPLETED).build()) + .artifacts(List.of(artifact)) + .history(List.of(historyMessage)) + .build(); + var toolCallResult = createToolCallResultWithContent("call1", "A2A_a2a1", task); + + var documents = handler.extractDocuments(toolCallResult); + + // artifacts before history + assertThat(documents).containsExactly(artifactDoc, historyDoc); + } + + @Test + void returnsEmptyListWhenContentIsNotA2aSendMessageResult() { + var toolCallResult = createToolCallResultWithContent("call1", "A2A_a2a1", "plain text"); + + var documents = handler.extractDocuments(toolCallResult); + + assertThat(documents).isEmpty(); + } + + @Test + void returnsEmptyListWhenA2aMessageHasNoDocuments() { + var message = + A2aMessage.builder() + .role(A2aMessage.Role.AGENT) + .messageId("msg-1") + .contextId("ctx-1") + .contents(List.of(TextContent.textContent("only text"))) + .build(); + var toolCallResult = createToolCallResultWithContent("call1", "A2A_a2a1", message); + + var documents = handler.extractDocuments(toolCallResult); + + assertThat(documents).isEmpty(); + } + + @Test + void returnsEmptyListWhenA2aTaskHasNoArtifactsOrHistory() { + var task = + A2aTask.builder() + .id("task-1") + .contextId("ctx-1") + .status(A2aTaskStatus.builder().state(A2aTaskStatus.TaskState.COMPLETED).build()) + .build(); + var toolCallResult = createToolCallResultWithContent("call1", "A2A_a2a1", task); + + var documents = handler.extractDocuments(toolCallResult); - // verify the document extractor can find documents in the preserved raw content - var extractor = new ToolCallResultDocumentExtractor(); - var extractedDocuments = extractor.extractDocuments(result); - assertThat(extractedDocuments).hasSize(1); - assertThat(extractedDocuments.getFirst().documents()).containsExactly(document); + assertThat(documents).isEmpty(); } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java index 840a94a43f5..83041cae67f 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java @@ -20,8 +20,10 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.within; import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -95,9 +97,21 @@ class AgentMessagesHandlerTest { void setUp() { documentStore.clear(); systemPromptComposer = new SystemPromptComposerImpl(List.of()); + // default: route document extraction through the generic walker (no handler-specific behaviour + // under test here). Individual tests can override this stub for handler-specific extraction. + lenient() + .when(gatewayToolHandlers.extractDocuments(any(ToolCallResult.class))) + .thenAnswer( + invocation -> { + ToolCallResult result = invocation.getArgument(0); + return ContentTreeDocumentWalker.INSTANCE.extractDocumentsFromContent( + result.content()); + }); messagesHandler = new AgentMessagesHandlerImpl( - gatewayToolHandlers, systemPromptComposer, new ToolCallResultDocumentExtractor()); + gatewayToolHandlers, + systemPromptComposer, + new ToolCallResultDocumentExtractor(gatewayToolHandlers)); runtimeMemory = spy(new DefaultRuntimeMemory()); } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalkerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalkerTest.java new file mode 100644 index 00000000000..6e9a6c1d696 --- /dev/null +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalkerTest.java @@ -0,0 +1,142 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.agent; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.camunda.connector.api.document.Document; +import io.camunda.connector.api.document.DocumentCreationRequest; +import io.camunda.connector.api.document.DocumentFactory; +import io.camunda.connector.runtime.core.document.DocumentFactoryImpl; +import io.camunda.connector.runtime.core.document.store.InMemoryDocumentStore; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class ContentTreeDocumentWalkerTest { + + private final InMemoryDocumentStore documentStore = InMemoryDocumentStore.INSTANCE; + private final DocumentFactory documentFactory = new DocumentFactoryImpl(documentStore); + private final ContentTreeDocumentWalker walker = ContentTreeDocumentWalker.INSTANCE; + + @BeforeEach + void setUp() { + documentStore.clear(); + } + + @Test + void extractsRootLevelDocument() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + final var result = walker.extractDocumentsFromContent(doc); + assertThat(result).containsExactly(doc); + } + + @Test + void extractsDocumentFromMapValue() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + final var result = walker.extractDocumentsFromContent(Map.of("file", doc, "key", "value")); + assertThat(result).containsExactly(doc); + } + + @Test + void extractsDocumentFromList() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + final var result = walker.extractDocumentsFromContent(List.of("text", doc, 42)); + assertThat(result).containsExactly(doc); + } + + @Test + void extractsDeeplyNestedDocuments() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + final var nested = Map.of("level1", Map.of("level2", List.of(Map.of("file", doc)))); + final var result = walker.extractDocumentsFromContent(nested); + assertThat(result).containsExactly(doc); + } + + @Test + void extractsMultipleDocuments() { + final var doc1 = createDocument("hello", "text/plain", "test.txt"); + final var doc2 = createDocument("", "application/pdf", "report.pdf"); + final var content = new LinkedHashMap(); + content.put("text", doc1); + content.put("report", doc2); + content.put("other", "value"); + + final var result = walker.extractDocumentsFromContent(content); + assertThat(result).containsExactly(doc1, doc2); + } + + @Test + void returnsEmptyForContentWithoutDocuments() { + final var result = + walker.extractDocumentsFromContent(Map.of("key", "value", "list", List.of(1, 2, 3))); + assertThat(result).isEmpty(); + } + + @Test + void returnsEmptyForNullContent() { + final var result = walker.extractDocumentsFromContent(null); + assertThat(result).isEmpty(); + } + + @Test + void returnsEmptyForScalarContent() { + assertThat(walker.extractDocumentsFromContent("text")).isEmpty(); + assertThat(walker.extractDocumentsFromContent(42)).isEmpty(); + assertThat(walker.extractDocumentsFromContent(true)).isEmpty(); + } + + @Test + void extractsDocumentFromArray() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + final var result = walker.extractDocumentsFromContent(new Object[] {"text", doc, 42}); + assertThat(result).containsExactly(doc); + } + + @Test + void extractsDocumentFromNestedArray() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + final var nested = Map.of("items", new Object[] {doc}); + final var result = walker.extractDocumentsFromContent(nested); + assertThat(result).containsExactly(doc); + } + + @Test + void handlesNullValuesInMap() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + final var content = new LinkedHashMap(); + content.put("file", doc); + content.put("missing", null); + + final var result = walker.extractDocumentsFromContent(content); + assertThat(result).containsExactly(doc); + } + + @Test + void handlesNullElementsInList() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + final var content = new ArrayList<>(); + content.add(doc); + content.add(null); + content.add("text"); + + final var result = walker.extractDocumentsFromContent(content); + assertThat(result).containsExactly(doc); + } + + private Document createDocument(String content, String contentType, String filename) { + return documentFactory.create( + DocumentCreationRequest.from(content.getBytes(StandardCharsets.UTF_8)) + .contentType(contentType) + .fileName(filename) + .build()); + } +} diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractorTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractorTest.java index 05b24458393..edc7c498495 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractorTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractorTest.java @@ -7,7 +7,14 @@ package io.camunda.connector.agenticai.aiagent.agent; import static org.assertj.core.api.Assertions.assertThat; - +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import io.camunda.connector.agenticai.aiagent.tool.GatewayToolHandler; +import io.camunda.connector.agenticai.aiagent.tool.GatewayToolHandlerRegistry; +import io.camunda.connector.agenticai.aiagent.tool.GatewayToolHandlerRegistryImpl; import io.camunda.connector.agenticai.model.tool.ToolCallResult; import io.camunda.connector.api.document.Document; import io.camunda.connector.api.document.DocumentCreationRequest; @@ -15,217 +22,159 @@ import io.camunda.connector.runtime.core.document.DocumentFactoryImpl; import io.camunda.connector.runtime.core.document.store.InMemoryDocumentStore; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +@ExtendWith(MockitoExtension.class) class ToolCallResultDocumentExtractorTest { private final InMemoryDocumentStore documentStore = InMemoryDocumentStore.INSTANCE; private final DocumentFactory documentFactory = new DocumentFactoryImpl(documentStore); - private final ToolCallResultDocumentExtractor extractor = new ToolCallResultDocumentExtractor(); + + @Mock private GatewayToolHandlerRegistry registry; + + private ToolCallResultDocumentExtractor extractor; @BeforeEach void setUp() { documentStore.clear(); + extractor = new ToolCallResultDocumentExtractor(registry); + } + + @Test + void groupsDocumentsByToolCall_routedThroughRegistry() { + final var doc1 = createDocument("hello", "text/plain", "test.txt"); + final var doc2 = createDocument("", "application/pdf", "report.pdf"); + + final var result1 = + ToolCallResult.builder().id("call_1").name("tool_a").content(Map.of("file", doc1)).build(); + final var result2 = + ToolCallResult.builder() + .id("call_2") + .name("tool_b") + .content(Map.of("report", doc2)) + .build(); + + when(registry.extractDocuments(result1)).thenReturn(List.of(doc1)); + when(registry.extractDocuments(result2)).thenReturn(List.of(doc2)); + + final var extracted = extractor.extractDocuments(List.of(result1, result2)); + + assertThat(extracted).hasSize(2); + assertThat(extracted.get(0)) + .satisfies( + e -> { + assertThat(e.toolCallId()).isEqualTo("call_1"); + assertThat(e.toolCallName()).isEqualTo("tool_a"); + assertThat(e.documents()).containsExactly(doc1); + }); + assertThat(extracted.get(1)) + .satisfies( + e -> { + assertThat(e.toolCallId()).isEqualTo("call_2"); + assertThat(e.toolCallName()).isEqualTo("tool_b"); + assertThat(e.documents()).containsExactly(doc2); + }); + } + + @Test + void excludesToolCallsWithoutDocuments() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + + final var withDoc = + ToolCallResult.builder().id("call_1").name("tool_a").content(Map.of("file", doc)).build(); + final var withoutDoc = + ToolCallResult.builder().id("call_2").name("tool_b").content("plain text result").build(); + + when(registry.extractDocuments(withDoc)).thenReturn(List.of(doc)); + when(registry.extractDocuments(withoutDoc)).thenReturn(List.of()); + + final var extracted = extractor.extractDocuments(List.of(withDoc, withoutDoc)); + + assertThat(extracted).hasSize(1); + assertThat(extracted.getFirst().toolCallId()).isEqualTo("call_1"); + } + + @Test + void returnsEmptyWhenNoToolCallsContainDocuments() { + final var result = + ToolCallResult.builder().id("call_1").name("tool_a").content("text result").build(); + when(registry.extractDocuments(result)).thenReturn(List.of()); + + assertThat(extractor.extractDocuments(List.of(result))).isEmpty(); } - @Nested - class ExtractFromContentTree { - - @Test - void extractsRootLevelDocument() { - final var doc = createDocument("hello", "text/plain", "test.txt"); - final var result = extractor.extractDocumentsFromContent(doc); - assertThat(result).containsExactly(doc); - } - - @Test - void extractsDocumentFromMapValue() { - final var doc = createDocument("hello", "text/plain", "test.txt"); - final var result = extractor.extractDocumentsFromContent(Map.of("file", doc, "key", "value")); - assertThat(result).containsExactly(doc); - } - - @Test - void extractsDocumentFromList() { - final var doc = createDocument("hello", "text/plain", "test.txt"); - final var result = extractor.extractDocumentsFromContent(List.of("text", doc, 42)); - assertThat(result).containsExactly(doc); - } - - @Test - void extractsDeeplyNestedDocuments() { - final var doc = createDocument("hello", "text/plain", "test.txt"); - final var nested = Map.of("level1", Map.of("level2", List.of(Map.of("file", doc)))); - final var result = extractor.extractDocumentsFromContent(nested); - assertThat(result).containsExactly(doc); - } - - @Test - void extractsMultipleDocuments() { - final var doc1 = createDocument("hello", "text/plain", "test.txt"); - final var doc2 = createDocument("", "application/pdf", "report.pdf"); - final var content = new LinkedHashMap(); - content.put("text", doc1); - content.put("report", doc2); - content.put("other", "value"); - - final var result = extractor.extractDocumentsFromContent(content); - assertThat(result).containsExactly(doc1, doc2); - } - - @Test - void returnsEmptyForContentWithoutDocuments() { - final var result = - extractor.extractDocumentsFromContent(Map.of("key", "value", "list", List.of(1, 2, 3))); - assertThat(result).isEmpty(); - } - - @Test - void returnsEmptyForNullContent() { - final var result = extractor.extractDocumentsFromContent(null); - assertThat(result).isEmpty(); - } - - @Test - void returnsEmptyForScalarContent() { - assertThat(extractor.extractDocumentsFromContent("text")).isEmpty(); - assertThat(extractor.extractDocumentsFromContent(42)).isEmpty(); - assertThat(extractor.extractDocumentsFromContent(true)).isEmpty(); - } - - @Test - void extractsDocumentFromArray() { - final var doc = createDocument("hello", "text/plain", "test.txt"); - final var result = extractor.extractDocumentsFromContent(new Object[] {"text", doc, 42}); - assertThat(result).containsExactly(doc); - } - - @Test - void extractsDocumentFromNestedArray() { - final var doc = createDocument("hello", "text/plain", "test.txt"); - final var nested = Map.of("items", new Object[] {doc}); - final var result = extractor.extractDocumentsFromContent(nested); - assertThat(result).containsExactly(doc); - } - - @Test - void handlesNullValuesInMap() { - final var doc = createDocument("hello", "text/plain", "test.txt"); - final var content = new LinkedHashMap(); - content.put("file", doc); - content.put("missing", null); - - final var result = extractor.extractDocumentsFromContent(content); - assertThat(result).containsExactly(doc); - } - - @Test - void handlesNullElementsInList() { - final var doc = createDocument("hello", "text/plain", "test.txt"); - final var content = new ArrayList<>(); - content.add(doc); - content.add(null); - content.add("text"); - - final var result = extractor.extractDocumentsFromContent(content); - assertThat(result).containsExactly(doc); - } + @Test + void handlesNullNameAndId() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + final var result = ToolCallResult.builder().content(Map.of("file", doc)).build(); + when(registry.extractDocuments(result)).thenReturn(List.of(doc)); + + final var extracted = extractor.extractDocuments(List.of(result)); + + assertThat(extracted).hasSize(1); + assertThat(extracted.getFirst()) + .satisfies( + e -> { + assertThat(e.toolCallId()).isEmpty(); + assertThat(e.toolCallName()).isEqualTo("unknown"); + }); } - @Nested - class ExtractFromToolCallResults { - - @Test - void groupsDocumentsByToolCall() { - final var doc1 = createDocument("hello", "text/plain", "test.txt"); - final var doc2 = createDocument("", "application/pdf", "report.pdf"); - - final var results = - List.of( - ToolCallResult.builder() - .id("call_1") - .name("tool_a") - .content(Map.of("file", doc1)) - .build(), - ToolCallResult.builder() - .id("call_2") - .name("tool_b") - .content(Map.of("report", doc2)) - .build()); - - final var extracted = extractor.extractDocuments(results); - - assertThat(extracted).hasSize(2); - assertThat(extracted.get(0)) - .satisfies( - e -> { - assertThat(e.toolCallId()).isEqualTo("call_1"); - assertThat(e.toolCallName()).isEqualTo("tool_a"); - assertThat(e.documents()).containsExactly(doc1); - }); - assertThat(extracted.get(1)) - .satisfies( - e -> { - assertThat(e.toolCallId()).isEqualTo("call_2"); - assertThat(e.toolCallName()).isEqualTo("tool_b"); - assertThat(e.documents()).containsExactly(doc2); - }); - } - - @Test - void excludesToolCallsWithoutDocuments() { - final var doc = createDocument("hello", "text/plain", "test.txt"); - - final var results = - List.of( - ToolCallResult.builder() - .id("call_1") - .name("tool_a") - .content(Map.of("file", doc)) - .build(), - ToolCallResult.builder() - .id("call_2") - .name("tool_b") - .content("plain text result") - .build()); - - final var extracted = extractor.extractDocuments(results); - - assertThat(extracted).hasSize(1); - assertThat(extracted.getFirst().toolCallId()).isEqualTo("call_1"); - } - - @Test - void returnsEmptyWhenNoToolCallsContainDocuments() { - final var results = - List.of( - ToolCallResult.builder().id("call_1").name("tool_a").content("text result").build()); - - assertThat(extractor.extractDocuments(results)).isEmpty(); - } - - @Test - void handlesNullNameAndId() { - final var doc = createDocument("hello", "text/plain", "test.txt"); - - final var results = List.of(ToolCallResult.builder().content(Map.of("file", doc)).build()); - - final var extracted = extractor.extractDocuments(results); - - assertThat(extracted).hasSize(1); - assertThat(extracted.getFirst()) - .satisfies( - e -> { - assertThat(e.toolCallId()).isEmpty(); - assertThat(e.toolCallName()).isEqualTo("unknown"); - }); - } + @Test + void returnsEmptyForEmptyInputWithoutTouchingRegistry() { + final var extracted = extractor.extractDocuments(List.of()); + + assertThat(extracted).isEmpty(); + verifyNoInteractions(registry); + } + + @Test + void integrationWithRealRegistry_fallsBackToContentTreeWalkerWhenNoHandlerMatches() { + // Use a real (empty) registry: no handlers -> default walker fallback. + final var realRegistry = new GatewayToolHandlerRegistryImpl(List.of()); + final var integrationExtractor = new ToolCallResultDocumentExtractor(realRegistry); + + final var doc = createDocument("hello", "text/plain", "test.txt"); + final var result = + ToolCallResult.builder() + .id("call_1") + .name("plain_bpmn_tool") + .content(Map.of("attachment", doc)) + .build(); + + final var extracted = integrationExtractor.extractDocuments(List.of(result)); + + assertThat(extracted).hasSize(1); + assertThat(extracted.getFirst().documents()).containsExactly(doc); + } + + @Test + void integrationWithRealRegistry_routesToHandlerOverridingExtraction( + @Mock GatewayToolHandler handler) { + final var doc = createDocument("typed", "text/plain", "typed.txt"); + final var typedContent = new TypedHandlerContent(doc); + + when(handler.type()).thenReturn("typed"); + when(handler.isGatewayManaged("typed_tool")).thenReturn(true); + when(handler.extractDocuments(any(ToolCallResult.class))).thenReturn(List.of(doc)); + + final var realRegistry = new GatewayToolHandlerRegistryImpl(List.of(handler)); + final var integrationExtractor = new ToolCallResultDocumentExtractor(realRegistry); + + final var result = + ToolCallResult.builder().id("call_1").name("typed_tool").content(typedContent).build(); + + final var extracted = integrationExtractor.extractDocuments(List.of(result)); + + assertThat(extracted).hasSize(1); + assertThat(extracted.getFirst().documents()).containsExactly(doc); + verify(handler).extractDocuments(result); } private Document createDocument(String content, String contentType, String filename) { @@ -235,4 +184,6 @@ private Document createDocument(String content, String contentType, String filen .fileName(filename) .build()); } + + private record TypedHandlerContent(Document document) {} } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandlerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandlerTest.java index b480696ce9f..d142cda995b 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandlerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandlerTest.java @@ -10,17 +10,27 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.mockito.Mockito.mock; import com.fasterxml.jackson.databind.ObjectMapper; import io.camunda.connector.agenticai.aiagent.model.AgentContext; import io.camunda.connector.agenticai.mcp.client.model.McpToolDefinition; import io.camunda.connector.agenticai.mcp.client.model.McpToolDefinitionBuilder; +import io.camunda.connector.agenticai.mcp.client.model.content.McpBlobContent; +import io.camunda.connector.agenticai.mcp.client.model.content.McpDocumentContent; +import io.camunda.connector.agenticai.mcp.client.model.content.McpEmbeddedResourceContent; +import io.camunda.connector.agenticai.mcp.client.model.content.McpEmbeddedResourceContent.BlobDocumentResource; +import io.camunda.connector.agenticai.mcp.client.model.content.McpEmbeddedResourceContent.BlobResource; +import io.camunda.connector.agenticai.mcp.client.model.content.McpEmbeddedResourceContent.TextResource; +import io.camunda.connector.agenticai.mcp.client.model.content.McpObjectContent; +import io.camunda.connector.agenticai.mcp.client.model.content.McpResourceLinkContent; import io.camunda.connector.agenticai.mcp.client.model.content.McpTextContent; import io.camunda.connector.agenticai.mcp.client.model.result.McpClientCallToolResult; import io.camunda.connector.agenticai.mcp.client.model.result.McpClientListToolsResult; import io.camunda.connector.agenticai.model.tool.GatewayToolDefinition; import io.camunda.connector.agenticai.model.tool.ToolCall; import io.camunda.connector.agenticai.model.tool.ToolCallResult; +import io.camunda.connector.api.document.Document; import io.camunda.connector.api.error.ConnectorException; import java.util.List; import java.util.Map; @@ -391,7 +401,7 @@ void transformsMcpClientResults_toMcpToolCallResults() { } @Test - void retainsListOfContentBlocksIfResultIsNotASingleTextBlock() { + void retainsTypedCallToolResultIfResultIsNotASingleTextBlock() { var agentContext = AgentContext.empty().withProperty(PROPERTY_MCP_CLIENTS, List.of("mcp1")); var mcpCallToolResult = new McpClientCallToolResult( @@ -408,19 +418,16 @@ void retainsListOfContentBlocksIfResultIsNotASingleTextBlock() { var result = handler.transformToolCallResults(agentContext, toolCallResults); assertThat(result).hasSize(1); - // getRawMcpContent extracts the "content" key from the map, preserving the raw list assertThat(result.getFirst().content()) - .asInstanceOf(InstanceOfAssertFactories.LIST) - .hasSize(2) - .satisfiesExactly( - first -> - assertThat(first) - .asInstanceOf(InstanceOfAssertFactories.MAP) - .containsEntry("text", "First content"), - second -> - assertThat(second) - .asInstanceOf(InstanceOfAssertFactories.MAP) - .containsEntry("text", "Second content")); + .isInstanceOfSatisfying( + McpClientCallToolResult.class, + typedResult -> { + assertThat(typedResult.name()).isEqualTo("tool1"); + assertThat(typedResult.content()) + .containsExactly( + McpTextContent.textContent("First content"), + McpTextContent.textContent("Second content")); + }); } @Test @@ -577,4 +584,125 @@ void handlesEmptyNewGatewayToolDefinitions() { assertThat(result.removed()).containsExactly("mcp1", "mcp2"); } } + + @Nested + class ExtractDocuments { + + @Test + void extractsDocumentFromMcpDocumentContent() { + var document = mock(Document.class); + var callToolResult = + new McpClientCallToolResult( + "tool1", List.of(new McpDocumentContent(document, Map.of())), false); + var toolCallResult = + createToolCallResultWithContent("call1", "MCP_mcp1___tool1", callToolResult); + + var documents = handler.extractDocuments(toolCallResult); + + assertThat(documents).containsExactly(document); + } + + @Test + void extractsDocumentFromBlobDocumentResourceInsideEmbeddedResource() { + var document = mock(Document.class); + var callToolResult = + new McpClientCallToolResult( + "tool1", + List.of( + new McpEmbeddedResourceContent( + new BlobDocumentResource("uri://doc", "application/pdf", document), + Map.of())), + false); + var toolCallResult = + createToolCallResultWithContent("call1", "MCP_mcp1___tool1", callToolResult); + + var documents = handler.extractDocuments(toolCallResult); + + assertThat(documents).containsExactly(document); + } + + @Test + void doesNotExtractFromTextResource() { + var callToolResult = + new McpClientCallToolResult( + "tool1", + List.of( + new McpEmbeddedResourceContent( + new TextResource("uri://text", "text/plain", "hello"), Map.of())), + false); + var toolCallResult = + createToolCallResultWithContent("call1", "MCP_mcp1___tool1", callToolResult); + + assertThat(handler.extractDocuments(toolCallResult)).isEmpty(); + } + + @Test + void doesNotExtractFromBlobResource() { + var callToolResult = + new McpClientCallToolResult( + "tool1", + List.of( + new McpEmbeddedResourceContent( + new BlobResource("uri://blob", "application/octet-stream", new byte[] {1, 2}), + Map.of())), + false); + var toolCallResult = + createToolCallResultWithContent("call1", "MCP_mcp1___tool1", callToolResult); + + assertThat(handler.extractDocuments(toolCallResult)).isEmpty(); + } + + @Test + void doesNotExtractFromTextOrObjectOrBlobOrResourceLinkVariants() { + var callToolResult = + new McpClientCallToolResult( + "tool1", + List.of( + McpTextContent.textContent("just text"), + new McpObjectContent(Map.of("k", "v"), Map.of()), + new McpBlobContent(new byte[] {1}, "image/png", Map.of()), + new McpResourceLinkContent("uri://x", "link", "desc", "text/plain", Map.of())), + false); + var toolCallResult = + createToolCallResultWithContent("call1", "MCP_mcp1___tool1", callToolResult); + + assertThat(handler.extractDocuments(toolCallResult)).isEmpty(); + } + + @Test + void preservesOrderAndCollectsMultipleDocuments() { + var doc1 = mock(Document.class); + var doc2 = mock(Document.class); + var callToolResult = + new McpClientCallToolResult( + "tool1", + List.of( + new McpDocumentContent(doc1, Map.of()), + McpTextContent.textContent("between"), + new McpEmbeddedResourceContent( + new BlobDocumentResource("uri://2", "application/pdf", doc2), Map.of())), + false); + var toolCallResult = + createToolCallResultWithContent("call1", "MCP_mcp1___tool1", callToolResult); + + assertThat(handler.extractDocuments(toolCallResult)).containsExactly(doc1, doc2); + } + + @Test + void returnsEmptyListWhenContentIsAStringOptimization() { + // single-text-content optimization yields a String content; nothing to extract + var toolCallResult = + createToolCallResultWithContent("call1", "MCP_mcp1___tool1", "plain text"); + + assertThat(handler.extractDocuments(toolCallResult)).isEmpty(); + } + + @Test + void returnsEmptyListWhenContentIsNotMcpClientCallToolResult() { + var toolCallResult = + createToolCallResultWithContent("call1", "MCP_mcp1___tool1", Map.of("foo", "bar")); + + assertThat(handler.extractDocuments(toolCallResult)).isEmpty(); + } + } } From d834d013b47507a6d2ae784e74cd7f6901312904 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Apr 2026 13:18:54 +0000 Subject: [PATCH 28/81] refactor(agentic-ai): make extractor the entrypoint, walker a static utility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-up adjustments to the per-handler document extraction design: 1. ToolCallResultDocumentExtractor is the routing entrypoint. It asks GatewayToolHandlerRegistry.handlerForToolDefinition(name) to find a managing handler and delegates if one exists; otherwise it walks the raw content tree itself. The fallback no longer lives inside the registry — gateway handlers contribute extraction for the results they manage; the generic walker handles everything else. Drops GatewayToolHandlerRegistry.extractDocuments. 2. ContentTreeDocumentWalker is a fully static utility (private ctor, static methods). The previous singleton INSTANCE field was the worst of both worlds — the walker is stateless, has no dependencies, and the SPI default in GatewayToolHandler needs static-style access since interface defaults can't be DI'd. Tests use the real walker directly. https://claude.ai/code/session_01SM8HzedSAVWqnDaEKrmCpR --- .../agent/AgentMessagesHandlerImpl.java | 3 +- .../agent/ContentTreeDocumentWalker.java | 23 ++-- .../ToolCallResultDocumentExtractor.java | 20 ++- .../aiagent/tool/GatewayToolHandler.java | 2 +- .../tool/GatewayToolHandlerRegistry.java | 9 -- .../tool/GatewayToolHandlerRegistryImpl.java | 12 -- .../agent/AgentMessagesHandlerTest.java | 15 +-- .../agent/ContentTreeDocumentWalkerTest.java | 33 ++--- .../ToolCallResultDocumentExtractorTest.java | 117 +++++++++++++----- 9 files changed, 133 insertions(+), 101 deletions(-) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java index 884340835b7..78fdf288f23 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java @@ -259,8 +259,7 @@ private Message createEventMessage( // extract documents from event content and add as document content blocks // events originate from BPMN event sub-processes (not a gateway handler), so walk the raw tree - var eventDocuments = - ContentTreeDocumentWalker.INSTANCE.extractDocumentsFromContent(eventContent); + var eventDocuments = ContentTreeDocumentWalker.extractDocumentsFromContent(eventContent); if (!eventDocuments.isEmpty()) { for (var doc : eventDocuments) { userMessageContent.add(textContent(DocumentXmlTag.from(doc).toXml())); diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalker.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalker.java index 5a3c6756492..58452794289 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalker.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalker.java @@ -13,21 +13,22 @@ import java.util.Map; /** - * Recursively walks an arbitrary content tree and collects {@link Document} instances. Handles - * {@link Document}, {@link Map}, {@link Collection}, and {@code Object[]}; all other types are - * skipped. + * Stateless utility that recursively walks an arbitrary content tree and collects {@link Document} + * instances. Handles {@link Document}, {@link Map}, {@link Collection}, and {@code Object[]}; all + * other types are skipped. * - *

    This is the default extraction strategy used by {@link - * io.camunda.connector.agenticai.aiagent.tool.GatewayToolHandler#extractDocuments} and the fallback - * for tool call results not managed by any gateway handler. It is intentionally public so that - * gateway handler implementations whose typed content wraps raw user-generated subtrees (e.g. - * arbitrary maps from a downstream system) can delegate to it for those subtrees. + *

    Used as the default extraction strategy by {@link + * io.camunda.connector.agenticai.aiagent.tool.GatewayToolHandler#extractDocuments} and as the + * fallback in {@link ToolCallResultDocumentExtractor} for tool call results not managed by any + * gateway handler. Public on purpose so that gateway handler implementations whose typed content + * wraps raw user-generated subtrees (e.g. arbitrary maps from a downstream system) can delegate to + * it for those subtrees. */ public final class ContentTreeDocumentWalker { - public static final ContentTreeDocumentWalker INSTANCE = new ContentTreeDocumentWalker(); + private ContentTreeDocumentWalker() {} - public List extractDocumentsFromContent(Object content) { + public static List extractDocumentsFromContent(Object content) { if (content == null) { return List.of(); } @@ -37,7 +38,7 @@ public List extractDocumentsFromContent(Object content) { return documents; } - private void collectDocuments(Object node, List documents) { + private static void collectDocuments(Object node, List documents) { if (node == null) { return; } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java index 4fb093c9fe0..d8f45efb36b 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java @@ -16,10 +16,12 @@ /** * Extracts {@link Document} instances from a list of tool call results, grouped by tool call. * - *

    The actual extraction strategy is delegated to the {@link GatewayToolHandlerRegistry}: each - * tool call result is routed to its responsible {@code GatewayToolHandler} (which may walk a typed - * domain object), with the generic content-tree walker as the default fallback for tool calls not - * managed by any gateway handler. + *

    For each result, the extractor walks the {@code content()} tree using {@link + * ContentTreeDocumentWalker} by default. When a result belongs to a {@link + * io.camunda.connector.agenticai.aiagent.tool.GatewayToolHandler} that overrides {@link + * io.camunda.connector.agenticai.aiagent.tool.GatewayToolHandler#extractDocuments(ToolCallResult) + * extractDocuments}, the handler's typed extraction is used instead — gateway handlers contribute + * extraction for the results they manage; the generic walker handles everything else. */ public class ToolCallResultDocumentExtractor { @@ -41,7 +43,7 @@ public List extractDocuments(List toolCallRes final var result = new ArrayList(); for (ToolCallResult toolCallResult : toolCallResults) { - final var documents = gatewayToolHandlers.extractDocuments(toolCallResult); + final var documents = extractFromToolCallResult(toolCallResult); if (!documents.isEmpty()) { result.add( new ToolCallDocuments( @@ -53,4 +55,12 @@ public List extractDocuments(List toolCallRes return result; } + + private List extractFromToolCallResult(ToolCallResult toolCallResult) { + return gatewayToolHandlers + .handlerForToolDefinition(toolCallResult.name()) + .map(handler -> handler.extractDocuments(toolCallResult)) + .orElseGet( + () -> ContentTreeDocumentWalker.extractDocumentsFromContent(toolCallResult.content())); + } } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandler.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandler.java index bf6d04cf7cc..c7aba34266f 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandler.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandler.java @@ -71,6 +71,6 @@ List handleToolDiscoveryResults( * around nested raw subtrees — call {@link ContentTreeDocumentWalker#INSTANCE} on the raw parts. */ default List extractDocuments(ToolCallResult toolCallResult) { - return ContentTreeDocumentWalker.INSTANCE.extractDocumentsFromContent(toolCallResult.content()); + return ContentTreeDocumentWalker.extractDocumentsFromContent(toolCallResult.content()); } } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandlerRegistry.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandlerRegistry.java index 8dec4a57c41..b0ad452f694 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandlerRegistry.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandlerRegistry.java @@ -9,7 +9,6 @@ import io.camunda.connector.agenticai.aiagent.model.AgentContext; import io.camunda.connector.agenticai.model.tool.GatewayToolDefinition; import io.camunda.connector.agenticai.model.tool.ToolCallResult; -import io.camunda.connector.api.document.Document; import java.util.List; import java.util.Map; import java.util.Optional; @@ -43,12 +42,4 @@ boolean allToolDiscoveryResultsPresent( GatewayToolDiscoveryResult handleToolDiscoveryResults( AgentContext agentContext, List toolCallResults); - - /** - * Extracts {@link Document} instances from a tool call result by routing to the responsible - * gateway handler, falling back to the default content-tree walker when no handler manages the - * tool. Expected to be called on already-transformed tool call results (i.e. after {@link - * #transformToolCallResults}). - */ - List extractDocuments(ToolCallResult toolCallResult); } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandlerRegistryImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandlerRegistryImpl.java index 52f52c8e86b..89dc82e7354 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandlerRegistryImpl.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandlerRegistryImpl.java @@ -6,13 +6,11 @@ */ package io.camunda.connector.agenticai.aiagent.tool; -import io.camunda.connector.agenticai.aiagent.agent.ContentTreeDocumentWalker; import io.camunda.connector.agenticai.aiagent.model.AgentContext; import io.camunda.connector.agenticai.model.tool.GatewayToolDefinition; import io.camunda.connector.agenticai.model.tool.ToolCall; import io.camunda.connector.agenticai.model.tool.ToolCallResult; import io.camunda.connector.agenticai.model.tool.ToolDefinition; -import io.camunda.connector.api.document.Document; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -165,14 +163,4 @@ public List transformToolCallResults( return transformedToolCallResults; } - - @Override - public List extractDocuments(ToolCallResult toolCallResult) { - return handlerForToolDefinition(toolCallResult.name()) - .map(handler -> handler.extractDocuments(toolCallResult)) - .orElseGet( - () -> - ContentTreeDocumentWalker.INSTANCE.extractDocumentsFromContent( - toolCallResult.content())); - } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java index 83041cae67f..7b349f5ea0d 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java @@ -20,10 +20,8 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.within; import static org.junit.jupiter.params.provider.Arguments.arguments; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -97,16 +95,9 @@ class AgentMessagesHandlerTest { void setUp() { documentStore.clear(); systemPromptComposer = new SystemPromptComposerImpl(List.of()); - // default: route document extraction through the generic walker (no handler-specific behaviour - // under test here). Individual tests can override this stub for handler-specific extraction. - lenient() - .when(gatewayToolHandlers.extractDocuments(any(ToolCallResult.class))) - .thenAnswer( - invocation -> { - ToolCallResult result = invocation.getArgument(0); - return ContentTreeDocumentWalker.INSTANCE.extractDocumentsFromContent( - result.content()); - }); + // No stub for handlerForToolDefinition: the mocked registry returns Optional.empty() by + // default, so the extractor falls back to the generic content-tree walker for every result. + // Individual tests can override this when a gateway-handler-specific extraction is needed. messagesHandler = new AgentMessagesHandlerImpl( gatewayToolHandlers, diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalkerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalkerTest.java index 6e9a6c1d696..c8da47b3984 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalkerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalkerTest.java @@ -25,7 +25,6 @@ class ContentTreeDocumentWalkerTest { private final InMemoryDocumentStore documentStore = InMemoryDocumentStore.INSTANCE; private final DocumentFactory documentFactory = new DocumentFactoryImpl(documentStore); - private final ContentTreeDocumentWalker walker = ContentTreeDocumentWalker.INSTANCE; @BeforeEach void setUp() { @@ -35,21 +34,23 @@ void setUp() { @Test void extractsRootLevelDocument() { final var doc = createDocument("hello", "text/plain", "test.txt"); - final var result = walker.extractDocumentsFromContent(doc); + final var result = ContentTreeDocumentWalker.extractDocumentsFromContent(doc); assertThat(result).containsExactly(doc); } @Test void extractsDocumentFromMapValue() { final var doc = createDocument("hello", "text/plain", "test.txt"); - final var result = walker.extractDocumentsFromContent(Map.of("file", doc, "key", "value")); + final var result = + ContentTreeDocumentWalker.extractDocumentsFromContent(Map.of("file", doc, "key", "value")); assertThat(result).containsExactly(doc); } @Test void extractsDocumentFromList() { final var doc = createDocument("hello", "text/plain", "test.txt"); - final var result = walker.extractDocumentsFromContent(List.of("text", doc, 42)); + final var result = + ContentTreeDocumentWalker.extractDocumentsFromContent(List.of("text", doc, 42)); assertThat(result).containsExactly(doc); } @@ -57,7 +58,7 @@ void extractsDocumentFromList() { void extractsDeeplyNestedDocuments() { final var doc = createDocument("hello", "text/plain", "test.txt"); final var nested = Map.of("level1", Map.of("level2", List.of(Map.of("file", doc)))); - final var result = walker.extractDocumentsFromContent(nested); + final var result = ContentTreeDocumentWalker.extractDocumentsFromContent(nested); assertThat(result).containsExactly(doc); } @@ -70,34 +71,36 @@ void extractsMultipleDocuments() { content.put("report", doc2); content.put("other", "value"); - final var result = walker.extractDocumentsFromContent(content); + final var result = ContentTreeDocumentWalker.extractDocumentsFromContent(content); assertThat(result).containsExactly(doc1, doc2); } @Test void returnsEmptyForContentWithoutDocuments() { final var result = - walker.extractDocumentsFromContent(Map.of("key", "value", "list", List.of(1, 2, 3))); + ContentTreeDocumentWalker.extractDocumentsFromContent( + Map.of("key", "value", "list", List.of(1, 2, 3))); assertThat(result).isEmpty(); } @Test void returnsEmptyForNullContent() { - final var result = walker.extractDocumentsFromContent(null); + final var result = ContentTreeDocumentWalker.extractDocumentsFromContent(null); assertThat(result).isEmpty(); } @Test void returnsEmptyForScalarContent() { - assertThat(walker.extractDocumentsFromContent("text")).isEmpty(); - assertThat(walker.extractDocumentsFromContent(42)).isEmpty(); - assertThat(walker.extractDocumentsFromContent(true)).isEmpty(); + assertThat(ContentTreeDocumentWalker.extractDocumentsFromContent("text")).isEmpty(); + assertThat(ContentTreeDocumentWalker.extractDocumentsFromContent(42)).isEmpty(); + assertThat(ContentTreeDocumentWalker.extractDocumentsFromContent(true)).isEmpty(); } @Test void extractsDocumentFromArray() { final var doc = createDocument("hello", "text/plain", "test.txt"); - final var result = walker.extractDocumentsFromContent(new Object[] {"text", doc, 42}); + final var result = + ContentTreeDocumentWalker.extractDocumentsFromContent(new Object[] {"text", doc, 42}); assertThat(result).containsExactly(doc); } @@ -105,7 +108,7 @@ void extractsDocumentFromArray() { void extractsDocumentFromNestedArray() { final var doc = createDocument("hello", "text/plain", "test.txt"); final var nested = Map.of("items", new Object[] {doc}); - final var result = walker.extractDocumentsFromContent(nested); + final var result = ContentTreeDocumentWalker.extractDocumentsFromContent(nested); assertThat(result).containsExactly(doc); } @@ -116,7 +119,7 @@ void handlesNullValuesInMap() { content.put("file", doc); content.put("missing", null); - final var result = walker.extractDocumentsFromContent(content); + final var result = ContentTreeDocumentWalker.extractDocumentsFromContent(content); assertThat(result).containsExactly(doc); } @@ -128,7 +131,7 @@ void handlesNullElementsInList() { content.add(null); content.add("text"); - final var result = walker.extractDocumentsFromContent(content); + final var result = ContentTreeDocumentWalker.extractDocumentsFromContent(content); assertThat(result).containsExactly(doc); } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractorTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractorTest.java index edc7c498495..340fdc68601 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractorTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractorTest.java @@ -8,8 +8,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import io.camunda.connector.agenticai.aiagent.tool.GatewayToolHandler; @@ -24,6 +24,7 @@ import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; +import java.util.Optional; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -47,7 +48,51 @@ void setUp() { } @Test - void groupsDocumentsByToolCall_routedThroughRegistry() { + void usesContentTreeWalkerWhenNoHandlerManagesTheToolCall() { + final var doc = createDocument("hello", "text/plain", "test.txt"); + final var result = + ToolCallResult.builder() + .id("call_1") + .name("plain_bpmn_tool") + .content(Map.of("file", doc)) + .build(); + + when(registry.handlerForToolDefinition("plain_bpmn_tool")).thenReturn(Optional.empty()); + + final var extracted = extractor.extractDocuments(List.of(result)); + + assertThat(extracted).hasSize(1); + assertThat(extracted.getFirst().documents()).containsExactly(doc); + } + + @Test + void delegatesToHandlerWhenManaged(@Mock GatewayToolHandler handler) { + final var doc = createDocument("typed", "text/plain", "typed.txt"); + final var typedContent = new TypedHandlerContent(doc); + final var result = + ToolCallResult.builder().id("call_1").name("typed_tool").content(typedContent).build(); + + when(registry.handlerForToolDefinition("typed_tool")).thenReturn(Optional.of(handler)); + when(handler.extractDocuments(result)).thenReturn(List.of(doc)); + + final var extracted = extractor.extractDocuments(List.of(result)); + + assertThat(extracted).hasSize(1); + assertThat(extracted.getFirst().documents()).containsExactly(doc); + verify(handler).extractDocuments(result); + } + + @Test + void doesNotConsultHandlerWhenContentHasNoDocumentsAndIsUnmanaged() { + final var result = + ToolCallResult.builder().id("call_1").name("unknown").content("plain text").build(); + when(registry.handlerForToolDefinition("unknown")).thenReturn(Optional.empty()); + + assertThat(extractor.extractDocuments(List.of(result))).isEmpty(); + } + + @Test + void groupsExtractedDocumentsByToolCall() { final var doc1 = createDocument("hello", "text/plain", "test.txt"); final var doc2 = createDocument("", "application/pdf", "report.pdf"); @@ -60,8 +105,7 @@ void groupsDocumentsByToolCall_routedThroughRegistry() { .content(Map.of("report", doc2)) .build(); - when(registry.extractDocuments(result1)).thenReturn(List.of(doc1)); - when(registry.extractDocuments(result2)).thenReturn(List.of(doc2)); + when(registry.handlerForToolDefinition(any())).thenReturn(Optional.empty()); final var extracted = extractor.extractDocuments(List.of(result1, result2)); @@ -91,8 +135,7 @@ void excludesToolCallsWithoutDocuments() { final var withoutDoc = ToolCallResult.builder().id("call_2").name("tool_b").content("plain text result").build(); - when(registry.extractDocuments(withDoc)).thenReturn(List.of(doc)); - when(registry.extractDocuments(withoutDoc)).thenReturn(List.of()); + when(registry.handlerForToolDefinition(any())).thenReturn(Optional.empty()); final var extracted = extractor.extractDocuments(List.of(withDoc, withoutDoc)); @@ -100,20 +143,12 @@ void excludesToolCallsWithoutDocuments() { assertThat(extracted.getFirst().toolCallId()).isEqualTo("call_1"); } - @Test - void returnsEmptyWhenNoToolCallsContainDocuments() { - final var result = - ToolCallResult.builder().id("call_1").name("tool_a").content("text result").build(); - when(registry.extractDocuments(result)).thenReturn(List.of()); - - assertThat(extractor.extractDocuments(List.of(result))).isEmpty(); - } - @Test void handlesNullNameAndId() { final var doc = createDocument("hello", "text/plain", "test.txt"); final var result = ToolCallResult.builder().content(Map.of("file", doc)).build(); - when(registry.extractDocuments(result)).thenReturn(List.of(doc)); + + when(registry.handlerForToolDefinition(null)).thenReturn(Optional.empty()); final var extracted = extractor.extractDocuments(List.of(result)); @@ -127,18 +162,9 @@ void handlesNullNameAndId() { } @Test - void returnsEmptyForEmptyInputWithoutTouchingRegistry() { - final var extracted = extractor.extractDocuments(List.of()); - - assertThat(extracted).isEmpty(); - verifyNoInteractions(registry); - } - - @Test - void integrationWithRealRegistry_fallsBackToContentTreeWalkerWhenNoHandlerMatches() { - // Use a real (empty) registry: no handlers -> default walker fallback. - final var realRegistry = new GatewayToolHandlerRegistryImpl(List.of()); - final var integrationExtractor = new ToolCallResultDocumentExtractor(realRegistry); + void integrationWithRealRegistry_fallsBackToWalkerWhenNoHandlerMatches() { + final var realExtractor = + new ToolCallResultDocumentExtractor(new GatewayToolHandlerRegistryImpl(List.of())); final var doc = createDocument("hello", "text/plain", "test.txt"); final var result = @@ -148,15 +174,14 @@ void integrationWithRealRegistry_fallsBackToContentTreeWalkerWhenNoHandlerMatche .content(Map.of("attachment", doc)) .build(); - final var extracted = integrationExtractor.extractDocuments(List.of(result)); + final var extracted = realExtractor.extractDocuments(List.of(result)); assertThat(extracted).hasSize(1); assertThat(extracted.getFirst().documents()).containsExactly(doc); } @Test - void integrationWithRealRegistry_routesToHandlerOverridingExtraction( - @Mock GatewayToolHandler handler) { + void integrationWithRealRegistry_routesToManagingHandler(@Mock GatewayToolHandler handler) { final var doc = createDocument("typed", "text/plain", "typed.txt"); final var typedContent = new TypedHandlerContent(doc); @@ -164,19 +189,43 @@ void integrationWithRealRegistry_routesToHandlerOverridingExtraction( when(handler.isGatewayManaged("typed_tool")).thenReturn(true); when(handler.extractDocuments(any(ToolCallResult.class))).thenReturn(List.of(doc)); - final var realRegistry = new GatewayToolHandlerRegistryImpl(List.of(handler)); - final var integrationExtractor = new ToolCallResultDocumentExtractor(realRegistry); + final var realExtractor = + new ToolCallResultDocumentExtractor(new GatewayToolHandlerRegistryImpl(List.of(handler))); final var result = ToolCallResult.builder().id("call_1").name("typed_tool").content(typedContent).build(); - final var extracted = integrationExtractor.extractDocuments(List.of(result)); + final var extracted = realExtractor.extractDocuments(List.of(result)); assertThat(extracted).hasSize(1); assertThat(extracted.getFirst().documents()).containsExactly(doc); verify(handler).extractDocuments(result); } + @Test + void integrationWithRealRegistry_doesNotConsultHandlerForUnmanagedTool( + @Mock GatewayToolHandler handler) { + when(handler.type()).thenReturn("typed"); + when(handler.isGatewayManaged("plain_tool")).thenReturn(false); + + final var realExtractor = + new ToolCallResultDocumentExtractor(new GatewayToolHandlerRegistryImpl(List.of(handler))); + + final var doc = createDocument("hello", "text/plain", "test.txt"); + final var result = + ToolCallResult.builder() + .id("call_1") + .name("plain_tool") + .content(Map.of("attachment", doc)) + .build(); + + final var extracted = realExtractor.extractDocuments(List.of(result)); + + assertThat(extracted).hasSize(1); + assertThat(extracted.getFirst().documents()).containsExactly(doc); + verify(handler, never()).extractDocuments(any()); + } + private Document createDocument(String content, String contentType, String filename) { return documentFactory.create( DocumentCreationRequest.from(content.getBytes(StandardCharsets.UTF_8)) From f8e4616001ce5ca9fd55515a18ec80727199eab3 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Apr 2026 13:40:19 +0000 Subject: [PATCH 29/81] refactor(agentic-ai): minimise transform-method changes in MCP/A2A handlers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous refactor commit changed the shape of the transformed ToolCallResult.content() in both gateway handlers more than necessary: * MCP set content to the full McpClientCallToolResult wrapper ({name, content[], isError}) instead of just the content list as it was pre-workaround (List). Reverted to passing callToolResult.content() — same shape the LLM saw before commit 85c6617 introduced the raw-Map workaround. McpClientGatewayToolHandler .extractDocuments now walks List from content(). * A2A had no behavioural change pre-vs-post-workaround beyond the variable naming and builder usage style — the previous commit introduced cosmetic churn. Restored the original method shape with the sendMessageResult variable and explicit toolCallResultBuilder. Tests updated to match the restored List content shape. https://claude.ai/code/session_01SM8HzedSAVWqnDaEKrmCpR --- .../agentic/tool/A2aGatewayToolHandler.java | 15 +-- .../McpClientGatewayToolHandler.java | 10 +- .../McpClientGatewayToolHandlerTest.java | 93 +++++++++---------- 3 files changed, 58 insertions(+), 60 deletions(-) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandler.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandler.java index fd50e48867a..48ac40c384f 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandler.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandler.java @@ -227,15 +227,16 @@ private List getA2aClientIds(AgentContext agentContext) { } private ToolCallResult toolCallResultFromA2aSendMessage(ToolCallResult toolCallResult) { - final var identifier = new A2aToolCallIdentifier(toolCallResult.name()); - final var typedContent = + final var sendMessageResult = objectMapper.convertValue(toolCallResult.content(), A2aSendMessageResult.class); + final var identifier = new A2aToolCallIdentifier(toolCallResult.name()); - return ToolCallResult.builder() - .id(toolCallResult.id()) - .name(identifier.fullyQualifiedName()) - .content(typedContent) - .build(); + final var toolCallResultBuilder = + ToolCallResult.builder().id(toolCallResult.id()).name(identifier.fullyQualifiedName()); + + toolCallResultBuilder.content(sendMessageResult); + + return toolCallResultBuilder.build(); } @Override diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandler.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandler.java index 7f9c259e50f..4275a4f8d10 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandler.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandler.java @@ -17,6 +17,7 @@ import io.camunda.connector.agenticai.mcp.client.model.McpClientOperation; import io.camunda.connector.agenticai.mcp.client.model.McpClientOperationDefinitions; import io.camunda.connector.agenticai.mcp.client.model.McpToolDefinition; +import io.camunda.connector.agenticai.mcp.client.model.content.McpContent; import io.camunda.connector.agenticai.mcp.client.model.content.McpDocumentContent; import io.camunda.connector.agenticai.mcp.client.model.content.McpEmbeddedResourceContent; import io.camunda.connector.agenticai.mcp.client.model.content.McpEmbeddedResourceContent.BlobDocumentResource; @@ -264,7 +265,7 @@ private ToolCallResult toolCallResultFromMcpToolCall(ToolCallResult toolCallResu && callToolResult.content().getFirst() instanceof McpTextContent textContent) { toolCallResultBuilder.content(textContent.text()); } else { - toolCallResultBuilder.content(callToolResult); + toolCallResultBuilder.content(callToolResult.content()); } return toolCallResultBuilder.build(); @@ -272,13 +273,16 @@ private ToolCallResult toolCallResultFromMcpToolCall(ToolCallResult toolCallResu @Override public List extractDocuments(ToolCallResult toolCallResult) { - if (!(toolCallResult.content() instanceof McpClientCallToolResult callToolResult)) { + if (!(toolCallResult.content() instanceof List contents)) { // string-content optimization or unmanaged shape — nothing to walk return List.of(); } final var documents = new ArrayList(); - for (var content : callToolResult.content()) { + for (var entry : contents) { + if (!(entry instanceof McpContent content)) { + continue; + } switch (content) { case McpDocumentContent documentContent -> documents.add(documentContent.document()); case McpEmbeddedResourceContent embeddedResourceContent -> { diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandlerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandlerTest.java index d142cda995b..d746f2e4bdf 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandlerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandlerTest.java @@ -17,6 +17,7 @@ import io.camunda.connector.agenticai.mcp.client.model.McpToolDefinition; import io.camunda.connector.agenticai.mcp.client.model.McpToolDefinitionBuilder; import io.camunda.connector.agenticai.mcp.client.model.content.McpBlobContent; +import io.camunda.connector.agenticai.mcp.client.model.content.McpContent; import io.camunda.connector.agenticai.mcp.client.model.content.McpDocumentContent; import io.camunda.connector.agenticai.mcp.client.model.content.McpEmbeddedResourceContent; import io.camunda.connector.agenticai.mcp.client.model.content.McpEmbeddedResourceContent.BlobDocumentResource; @@ -401,7 +402,7 @@ void transformsMcpClientResults_toMcpToolCallResults() { } @Test - void retainsTypedCallToolResultIfResultIsNotASingleTextBlock() { + void retainsTypedContentListIfResultIsNotASingleTextBlock() { var agentContext = AgentContext.empty().withProperty(PROPERTY_MCP_CLIENTS, List.of("mcp1")); var mcpCallToolResult = new McpClientCallToolResult( @@ -419,15 +420,10 @@ void retainsTypedCallToolResultIfResultIsNotASingleTextBlock() { assertThat(result).hasSize(1); assertThat(result.getFirst().content()) - .isInstanceOfSatisfying( - McpClientCallToolResult.class, - typedResult -> { - assertThat(typedResult.name()).isEqualTo("tool1"); - assertThat(typedResult.content()) - .containsExactly( - McpTextContent.textContent("First content"), - McpTextContent.textContent("Second content")); - }); + .asInstanceOf(InstanceOfAssertFactories.list(McpContent.class)) + .containsExactly( + McpTextContent.textContent("First content"), + McpTextContent.textContent("Second content")); } @Test @@ -591,11 +587,9 @@ class ExtractDocuments { @Test void extractsDocumentFromMcpDocumentContent() { var document = mock(Document.class); - var callToolResult = - new McpClientCallToolResult( - "tool1", List.of(new McpDocumentContent(document, Map.of())), false); var toolCallResult = - createToolCallResultWithContent("call1", "MCP_mcp1___tool1", callToolResult); + createToolCallResultWithContent( + "call1", "MCP_mcp1___tool1", List.of(new McpDocumentContent(document, Map.of()))); var documents = handler.extractDocuments(toolCallResult); @@ -605,16 +599,14 @@ void extractsDocumentFromMcpDocumentContent() { @Test void extractsDocumentFromBlobDocumentResourceInsideEmbeddedResource() { var document = mock(Document.class); - var callToolResult = - new McpClientCallToolResult( - "tool1", + var toolCallResult = + createToolCallResultWithContent( + "call1", + "MCP_mcp1___tool1", List.of( new McpEmbeddedResourceContent( new BlobDocumentResource("uri://doc", "application/pdf", document), - Map.of())), - false); - var toolCallResult = - createToolCallResultWithContent("call1", "MCP_mcp1___tool1", callToolResult); + Map.of()))); var documents = handler.extractDocuments(toolCallResult); @@ -623,48 +615,42 @@ void extractsDocumentFromBlobDocumentResourceInsideEmbeddedResource() { @Test void doesNotExtractFromTextResource() { - var callToolResult = - new McpClientCallToolResult( - "tool1", + var toolCallResult = + createToolCallResultWithContent( + "call1", + "MCP_mcp1___tool1", List.of( new McpEmbeddedResourceContent( - new TextResource("uri://text", "text/plain", "hello"), Map.of())), - false); - var toolCallResult = - createToolCallResultWithContent("call1", "MCP_mcp1___tool1", callToolResult); + new TextResource("uri://text", "text/plain", "hello"), Map.of()))); assertThat(handler.extractDocuments(toolCallResult)).isEmpty(); } @Test void doesNotExtractFromBlobResource() { - var callToolResult = - new McpClientCallToolResult( - "tool1", + var toolCallResult = + createToolCallResultWithContent( + "call1", + "MCP_mcp1___tool1", List.of( new McpEmbeddedResourceContent( new BlobResource("uri://blob", "application/octet-stream", new byte[] {1, 2}), - Map.of())), - false); - var toolCallResult = - createToolCallResultWithContent("call1", "MCP_mcp1___tool1", callToolResult); + Map.of()))); assertThat(handler.extractDocuments(toolCallResult)).isEmpty(); } @Test void doesNotExtractFromTextOrObjectOrBlobOrResourceLinkVariants() { - var callToolResult = - new McpClientCallToolResult( - "tool1", + var toolCallResult = + createToolCallResultWithContent( + "call1", + "MCP_mcp1___tool1", List.of( McpTextContent.textContent("just text"), new McpObjectContent(Map.of("k", "v"), Map.of()), new McpBlobContent(new byte[] {1}, "image/png", Map.of()), - new McpResourceLinkContent("uri://x", "link", "desc", "text/plain", Map.of())), - false); - var toolCallResult = - createToolCallResultWithContent("call1", "MCP_mcp1___tool1", callToolResult); + new McpResourceLinkContent("uri://x", "link", "desc", "text/plain", Map.of()))); assertThat(handler.extractDocuments(toolCallResult)).isEmpty(); } @@ -673,17 +659,15 @@ void doesNotExtractFromTextOrObjectOrBlobOrResourceLinkVariants() { void preservesOrderAndCollectsMultipleDocuments() { var doc1 = mock(Document.class); var doc2 = mock(Document.class); - var callToolResult = - new McpClientCallToolResult( - "tool1", + var toolCallResult = + createToolCallResultWithContent( + "call1", + "MCP_mcp1___tool1", List.of( new McpDocumentContent(doc1, Map.of()), McpTextContent.textContent("between"), new McpEmbeddedResourceContent( - new BlobDocumentResource("uri://2", "application/pdf", doc2), Map.of())), - false); - var toolCallResult = - createToolCallResultWithContent("call1", "MCP_mcp1___tool1", callToolResult); + new BlobDocumentResource("uri://2", "application/pdf", doc2), Map.of()))); assertThat(handler.extractDocuments(toolCallResult)).containsExactly(doc1, doc2); } @@ -698,11 +682,20 @@ void returnsEmptyListWhenContentIsAStringOptimization() { } @Test - void returnsEmptyListWhenContentIsNotMcpClientCallToolResult() { + void returnsEmptyListWhenContentIsNotAList() { var toolCallResult = createToolCallResultWithContent("call1", "MCP_mcp1___tool1", Map.of("foo", "bar")); assertThat(handler.extractDocuments(toolCallResult)).isEmpty(); } + + @Test + void returnsEmptyListWhenListEntriesAreNotMcpContent() { + var toolCallResult = + createToolCallResultWithContent( + "call1", "MCP_mcp1___tool1", List.of(Map.of("type", "text", "text", "raw map"))); + + assertThat(handler.extractDocuments(toolCallResult)).isEmpty(); + } } } From dbefe0a25f9d0b93888df4ef6f09d2c0736f1cc0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Apr 2026 14:54:01 +0000 Subject: [PATCH 30/81] docs(agentic-ai): document new SPI hook + add fidelity test and address review nits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback follow-up: * GatewayToolHandler javadoc: drop stale `ContentTreeDocumentWalker#INSTANCE` link, point at the static `extractDocumentsFromContent` method. * ContentTreeDocumentWalkerTest: static-import the walker method (less noise) and convert the scalar-content test to a parameterised one that also covers null input. * L4J E2E tests (4 sites): drop the regex-based document short-id extraction. `AiAgentTestFixtures.readDocumentReference(String)` parses the tool result JSON, asserts the camunda document discriminator, and returns a typed DocumentReferenceFields record with storeId / documentId / contentType / fileName + shortId(). Tests now assert the parsed contentType against the parameterised mimeType and read shortId from the parsed record. * docs/reference/ai-agent.md §19: add `extractDocuments` to the GatewayToolHandler interface listing and a new "Document Extraction from Tool Call Results" subsection describing the extractor → registry → handler routing with ContentTreeDocumentWalker as the fallback. * docs/reference/mcp.md §7 and a2a.md §7: tool call execution flows now document the typed transformed content shape and the per-handler extractDocuments step. Regression test: * New GatewayToolResultDocumentSerializationTest pins the JSON wire format produced by the connector ObjectMapper for Documents nested inside McpClientCallToolResult and A2aSendMessageResult — they must serialize as `camunda.document.type` references via DocumentSerializer, never as base64 payloads or raw DocumentReference POJOs. Covers root McpDocumentContent, embedded BlobDocumentResource, A2aMessage contents, A2aTask artifacts and recursive history, plus an explicit base64-must-not-appear assertion. https://claude.ai/code/session_01SM8HzedSAVWqnDaEKrmCpR --- .../aiagent/AiAgentTestFixtures.java | 86 +++++-- ...4JAiAgentJobWorkerMcpIntegrationTests.java | 13 +- .../L4JAiAgentJobWorkerToolCallingTests.java | 15 +- ...4JAiAgentConnectorMcpIntegrationTests.java | 13 +- .../L4JAiAgentConnectorToolCallingTests.java | 15 +- connectors/agentic-ai/docs/reference/a2a.md | 8 +- .../agentic-ai/docs/reference/ai-agent.md | 35 +++ connectors/agentic-ai/docs/reference/mcp.md | 8 +- .../aiagent/tool/GatewayToolHandler.java | 3 +- .../agent/ContentTreeDocumentWalkerTest.java | 58 ++--- ...ayToolResultDocumentSerializationTest.java | 220 ++++++++++++++++++ 11 files changed, 389 insertions(+), 85 deletions(-) create mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/GatewayToolResultDocumentSerializationTest.java diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/AiAgentTestFixtures.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/AiAgentTestFixtures.java index 3a04307a14b..0786cf92d6e 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/AiAgentTestFixtures.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/AiAgentTestFixtures.java @@ -19,8 +19,10 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.entry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Map; -import java.util.regex.Pattern; import org.assertj.core.api.InstanceOfAssertFactories; import org.assertj.core.api.ThrowingConsumer; @@ -91,24 +93,76 @@ public interface AiAgentTestFixtures { String FEEDBACK_LOOP_RESPONSE_TEXT = "A very complex calculation only the superflux calculation tool can do."; - Pattern DOCUMENT_ID_PATTERN = Pattern.compile("\"documentId\"\\s*:\\s*\"([^\"]+)\""); + ObjectMapper TOOL_RESULT_OBJECT_MAPPER = new ObjectMapper(); /** - * Extracts the document short ID (first UUID segment) from a tool result text containing a - * serialized document reference. The tool result JSON includes a document reference like: - * - * 

    {@code
-   * {"storeId":"in-memory","documentId":"25ece9fa-aeea-423d-98ed-67c1f08b137b",...}
-   * }
    + * Parsed Camunda document reference fields extracted from a tool call result JSON. The {@link + * #shortId()} returns the first UUID segment of the {@code documentId}, used as the compact + * correlation key in the synthetic {@code } XML tag. + */ + record DocumentReferenceFields( + String storeId, String documentId, String contentType, String fileName) { + public String shortId() { + int dash = documentId.indexOf('-'); + return dash > 0 ? documentId.substring(0, dash) : documentId; + } + } + + /** + * Locates the first Camunda document reference inside a serialized tool call result and reads its + * structural fields. Recursively descends into objects/arrays until it finds an object carrying + * the {@code camunda.document.type} discriminator (set by {@code DocumentSerializer}). * - * This method extracts the first segment of the documentId UUID ("25ece9fa"), which is used as a - * compact correlation key in the document XML tags. + *

    Use this in place of regex-matching the raw text — both for asserting expected reference + * fields (content type, file name) and for extracting the document short ID. */ - static String extractDocumentShortId(String toolResultText) { - var matcher = DOCUMENT_ID_PATTERN.matcher(toolResultText); - assertThat(matcher.find()).as("documentId in tool result text").isTrue(); - var documentId = matcher.group(1); - int dash = documentId.indexOf('-'); - return dash > 0 ? documentId.substring(0, dash) : documentId; + static DocumentReferenceFields readDocumentReference(String toolResultText) { + final JsonNode root; + try { + root = TOOL_RESULT_OBJECT_MAPPER.readTree(toolResultText); + } catch (JsonProcessingException e) { + throw new AssertionError("Failed to parse tool result text as JSON: " + toolResultText, e); + } + + final JsonNode docNode = findFirstCamundaDocumentNode(root); + assertThat(docNode) + .as("Camunda document reference in tool result text: %s", toolResultText) + .isNotNull(); + assertThat(docNode.path("camunda.document.type").asText()) + .as("camunda.document.type discriminator") + .isEqualTo("camunda"); + assertThat(docNode.path("documentId").asText()).as("documentId").isNotBlank(); + + return new DocumentReferenceFields( + docNode.path("storeId").asText(null), + docNode.path("documentId").asText(), + docNode.path("metadata").path("contentType").asText(null), + docNode.path("metadata").path("fileName").asText(null)); + } + + private static JsonNode findFirstCamundaDocumentNode(JsonNode node) { + if (node == null) { + return null; + } + if (node.isObject()) { + if ("camunda".equals(node.path("camunda.document.type").asText(null))) { + return node; + } + final var fields = node.fields(); + while (fields.hasNext()) { + final var found = findFirstCamundaDocumentNode(fields.next().getValue()); + if (found != null) { + return found; + } + } + } else if (node.isArray()) { + for (var element : node) { + final var found = findFirstCamundaDocumentNode(element); + if (found != null) { + return found; + } + } + } + return null; } } diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerMcpIntegrationTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerMcpIntegrationTests.java index f63c664be7a..6de98cb7186 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerMcpIntegrationTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerMcpIntegrationTests.java @@ -17,7 +17,7 @@ package io.camunda.connector.e2e.agenticai.aiagent.langchain4j.jobworker; import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.HAIKU_TEXT; -import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.extractDocumentShortId; +import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.readDocumentReference; import static io.camunda.connector.e2e.agenticai.aiagent.langchain4j.Langchain4JAiAgentToolSpecifications.EXPECTED_MCP_TOOL_SPECIFICATIONS; import static io.camunda.connector.e2e.agenticai.mcp.McpSdkToolSpecifications.MCP_TOOL_SPECIFICATIONS; import static org.assertj.core.api.Assertions.assertThat; @@ -402,18 +402,17 @@ void extractsDocumentsFromMcpImageToolCallResult() throws IOException { assertThat(lastMessages.get(2)).isInstanceOf(AiMessage.class); // tool call // tool result: document serialized as document reference + var toolResultText = ((ToolExecutionResultMessage) lastMessages.get(3)).text(); + var documentReference = readDocumentReference(toolResultText); + assertThat(lastMessages.get(3)) .isInstanceOfSatisfying( ToolExecutionResultMessage.class, msg -> { assertThat(msg.id()).isEqualTo("img111"); assertThat(msg.toolName()).isEqualTo("MCP_A_MCP_Client___toolA"); - assertThat(msg.text()).contains("camunda.document.type"); }); - - // extract the document short ID (first UUID segment) from the serialized document reference - var toolResultText = ((ToolExecutionResultMessage) lastMessages.get(3)).text(); - var documentShortId = extractDocumentShortId(toolResultText); + assertThat(documentReference.contentType()).isEqualTo("image/png"); // document user message: extracted document content assertThat(lastMessages.get(4)) @@ -435,7 +434,7 @@ void extractsDocumentsFromMcpImageToolCallResult() throws IOException { assertThat(tc.text()) .isEqualTo( "" - .formatted(documentShortId))); + .formatted(documentReference.shortId()))); assertThat(contents.get(2)) .isInstanceOfSatisfying( ImageContent.class, diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java index d1158597541..6af97da0cf5 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java @@ -17,7 +17,7 @@ package io.camunda.connector.e2e.agenticai.aiagent.langchain4j.jobworker; import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.FEEDBACK_LOOP_RESPONSE_TEXT; -import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.extractDocumentShortId; +import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.readDocumentReference; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; @@ -151,20 +151,17 @@ void supportsDocumentResponsesFromToolCalls( assertThat(lastMessages.get(2)).isInstanceOf(AiMessage.class); // tool call // tool result: document serialized as document reference + var toolResultText = ((ToolExecutionResultMessage) lastMessages.get(3)).text(); + var documentReference = readDocumentReference(toolResultText); + assertThat(lastMessages.get(3)) .isInstanceOfSatisfying( ToolExecutionResultMessage.class, msg -> { assertThat(msg.id()).isEqualTo("aaa111"); assertThat(msg.toolName()).isEqualTo("Download_A_File"); - assertThat(msg.text()).contains("camunda.document.type"); - assertThat(msg.text()).contains(mimeType); }); - - // extract the document short ID (first UUID segment) from the serialized document reference - // e.g. from {"documentId":"25ece9fa-...", ...} -> "25ece9fa" - var toolResultText = ((ToolExecutionResultMessage) lastMessages.get(3)).text(); - var documentShortId = extractDocumentShortId(toolResultText); + assertThat(documentReference.contentType()).isEqualTo(mimeType); // document user message: extracted document content assertThat(lastMessages.get(4)) @@ -186,7 +183,7 @@ void supportsDocumentResponsesFromToolCalls( assertThat(tc.text()) .isEqualTo( "" - .formatted(documentShortId))); + .formatted(documentReference.shortId()))); assertDocumentContentBlock(contents.get(2), type, mimeType); }); diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorMcpIntegrationTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorMcpIntegrationTests.java index b45dceb335d..fe4fbd7906c 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorMcpIntegrationTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorMcpIntegrationTests.java @@ -17,7 +17,7 @@ package io.camunda.connector.e2e.agenticai.aiagent.langchain4j.outboundconnector; import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.HAIKU_TEXT; -import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.extractDocumentShortId; +import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.readDocumentReference; import static io.camunda.connector.e2e.agenticai.aiagent.langchain4j.Langchain4JAiAgentToolSpecifications.EXPECTED_MCP_TOOL_SPECIFICATIONS; import static io.camunda.connector.e2e.agenticai.mcp.McpSdkToolSpecifications.MCP_TOOL_SPECIFICATIONS; import static org.assertj.core.api.Assertions.assertThat; @@ -404,18 +404,17 @@ void extractsDocumentsFromMcpImageToolCallResult() throws IOException { assertThat(lastMessages.get(2)).isInstanceOf(AiMessage.class); // tool call // tool result: document serialized as document reference + var toolResultText = ((ToolExecutionResultMessage) lastMessages.get(3)).text(); + var documentReference = readDocumentReference(toolResultText); + assertThat(lastMessages.get(3)) .isInstanceOfSatisfying( ToolExecutionResultMessage.class, msg -> { assertThat(msg.id()).isEqualTo("img111"); assertThat(msg.toolName()).isEqualTo("MCP_A_MCP_Client___toolA"); - assertThat(msg.text()).contains("camunda.document.type"); }); - - // extract the document short ID (first UUID segment) from the serialized document reference - var toolResultText = ((ToolExecutionResultMessage) lastMessages.get(3)).text(); - var documentShortId = extractDocumentShortId(toolResultText); + assertThat(documentReference.contentType()).isEqualTo("image/png"); // document user message: extracted document content assertThat(lastMessages.get(4)) @@ -437,7 +436,7 @@ void extractsDocumentsFromMcpImageToolCallResult() throws IOException { assertThat(tc.text()) .isEqualTo( "" - .formatted(documentShortId))); + .formatted(documentReference.shortId()))); assertThat(contents.get(2)) .isInstanceOfSatisfying( ImageContent.class, diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java index 8e6326fa295..b70c308a134 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java @@ -17,7 +17,7 @@ package io.camunda.connector.e2e.agenticai.aiagent.langchain4j.outboundconnector; import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.FEEDBACK_LOOP_RESPONSE_TEXT; -import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.extractDocumentShortId; +import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.readDocumentReference; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; @@ -144,20 +144,17 @@ void supportsDocumentResponsesFromToolCalls( assertThat(lastMessages.get(2)).isInstanceOf(AiMessage.class); // tool call // tool result: document serialized as document reference + var toolResultText = ((ToolExecutionResultMessage) lastMessages.get(3)).text(); + var documentReference = readDocumentReference(toolResultText); + assertThat(lastMessages.get(3)) .isInstanceOfSatisfying( ToolExecutionResultMessage.class, msg -> { assertThat(msg.id()).isEqualTo("aaa111"); assertThat(msg.toolName()).isEqualTo("Download_A_File"); - assertThat(msg.text()).contains("camunda.document.type"); - assertThat(msg.text()).contains(mimeType); }); - - // extract the document short ID (first UUID segment) from the serialized document reference - // e.g. from {"documentId":"25ece9fa-...", ...} -> "25ece9fa" - var toolResultText = ((ToolExecutionResultMessage) lastMessages.get(3)).text(); - var documentShortId = extractDocumentShortId(toolResultText); + assertThat(documentReference.contentType()).isEqualTo(mimeType); // document user message: extracted document content assertThat(lastMessages.get(4)) @@ -179,7 +176,7 @@ void supportsDocumentResponsesFromToolCalls( assertThat(tc.text()) .isEqualTo( "" - .formatted(documentShortId))); + .formatted(documentReference.shortId()))); assertDocumentContentBlock(contents.get(2), type, mimeType); }); diff --git a/connectors/agentic-ai/docs/reference/a2a.md b/connectors/agentic-ai/docs/reference/a2a.md index 71e451c0e36..bdcb53a350b 100644 --- a/connectors/agentic-ai/docs/reference/a2a.md +++ b/connectors/agentic-ai/docs/reference/a2a.md @@ -150,8 +150,14 @@ Simpler than MCP because there's only one "tool" per A2A element (the agent itse 5. Result (A2aSendMessageResult — either A2aTask or A2aMessage) flows back 6. A2aGatewayToolHandler.transformToolCallResults(): - Rebuilds fully qualified name: "A2A_WeatherAgent" - - Returns ToolCallResult with the send message result content + - Deserializes content into the typed A2aSendMessageResult and sets it as the transformed + ToolCallResult content 7. LLM receives the response and decides next action based on task state +8. A2aGatewayToolHandler.extractDocuments() walks the typed A2aSendMessageResult and collects + Camunda Documents from DocumentContent entries: A2aMessage.contents at the root, plus + A2aTask.artifacts and (recursively) A2aTask.history. Documents flow into the synthetic + document UserMessage produced by ToolCallResultDocumentExtractor (see ai-agent.md §19 + "Document Extraction from Tool Call Results"). ``` --- diff --git a/connectors/agentic-ai/docs/reference/ai-agent.md b/connectors/agentic-ai/docs/reference/ai-agent.md index 8054dfceaf6..f9be6200332 100644 --- a/connectors/agentic-ai/docs/reference/ai-agent.md +++ b/connectors/agentic-ai/docs/reference/ai-agent.md @@ -1291,6 +1291,9 @@ public interface GatewayToolHandler extends GatewayToolCallTransformer { // Migration GatewayToolDefinitionUpdates resolveUpdatedGatewayToolDefinitions(agentContext, gatewayToolDefinitions); + + // Document extraction (default falls back to ContentTreeDocumentWalker for raw content trees) + default List extractDocuments(ToolCallResult toolCallResult); } // From GatewayToolCallTransformer: @@ -1359,6 +1362,38 @@ Gateway handlers store per-handler state in `AgentContext.properties`: These are used during discovery checking and tool call result transformation. +### Document Extraction from Tool Call Results + +Tool call results may contain Camunda `Document` instances — at the root, nested in maps/lists, or +embedded inside typed gateway responses (e.g. `McpDocumentContent`, A2A artifacts). The agent +extracts those documents into a synthetic follow-up `UserMessage` with native `DocumentContent` +blocks so LLMs can interpret them; see [ADR-004](../adr/004-document-handling-in-tool-call-results.md). + +`ToolCallResultDocumentExtractor` is the entrypoint, called from `AgentMessagesHandlerImpl` after +the `ToolCallResultMessage` is built. For each result it asks the registry which handler manages +the tool name (`GatewayToolHandlerRegistry.handlerForToolDefinition`) and either: + +- delegates to that handler's `extractDocuments(ToolCallResult)` — handlers walk their own typed + content (sealed-type `switch` over `McpContent` / `A2aSendMessageResult`), so documents inside + typed records remain discoverable; +- falls back to `ContentTreeDocumentWalker` — a stateless static utility that recursively walks + `Map`, `Collection`, `Object[]` and `Document` nodes. Used for plain BPMN tools whose content is + the raw FEEL tree from the engine. + +The default `GatewayToolHandler.extractDocuments` implementation also delegates to +`ContentTreeDocumentWalker`, so third-party handlers that return raw maps work without overriding. +Handlers whose typed content embeds raw user-generated subtrees can call the walker directly on +those subtrees. + +``` +ToolCallResultDocumentExtractor.extractDocuments(results) + ├─ for each result: + │ GatewayToolHandlerRegistry.handlerForToolDefinition(result.name()) + │ ├─ Some(handler) → handler.extractDocuments(result) ── typed walk + │ └─ None → ContentTreeDocumentWalker.extractDocumentsFromContent(content) + └─ groups documents by ToolCallDocuments(toolCallId, toolCallName, documents) +``` + --- diff --git a/connectors/agentic-ai/docs/reference/mcp.md b/connectors/agentic-ai/docs/reference/mcp.md index 1ee04c5f6fb..d75daeaa124 100644 --- a/connectors/agentic-ai/docs/reference/mcp.md +++ b/connectors/agentic-ai/docs/reference/mcp.md @@ -226,9 +226,15 @@ Discovery tool call IDs use a different format: `MCP_toolsList_` — 5. Result flows back as toolCallResult (McpClientCallToolResult) 6. McpClientGatewayToolHandler.transformToolCallResults(): - Extracts tool name from result, rebuilds fully qualified name - - Maps content: if single McpTextContent → use string directly, else use list + - Maps content: if single McpTextContent → use string directly, else use the typed + List as the transformed ToolCallResult content - Returns ToolCallResult with name="MCP_MyFilesystem___readFile" 7. Agent presents result to LLM with the original fully qualified tool name +8. McpClientGatewayToolHandler.extractDocuments() walks the List via a sealed-type + switch and collects Camunda Documents from McpDocumentContent and + McpEmbeddedResourceContent.BlobDocumentResource entries — feeding the synthetic document + UserMessage produced by ToolCallResultDocumentExtractor (see ai-agent.md §19 "Document + Extraction from Tool Call Results"). ``` --- diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandler.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandler.java index c7aba34266f..69c81bf3c31 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandler.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/tool/GatewayToolHandler.java @@ -68,7 +68,8 @@ List handleToolDiscoveryResults( * *

    Handlers that return typed records or POJOs as content must override this method and walk * their own structure (typically via a sealed-type switch). For mixed shapes — typed wrappers - * around nested raw subtrees — call {@link ContentTreeDocumentWalker#INSTANCE} on the raw parts. + * around nested raw subtrees — call {@link + * ContentTreeDocumentWalker#extractDocumentsFromContent(Object)} on the raw parts. */ default List extractDocuments(ToolCallResult toolCallResult) { return ContentTreeDocumentWalker.extractDocumentsFromContent(toolCallResult.content()); diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalkerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalkerTest.java index c8da47b3984..2d0228cb748 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalkerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ContentTreeDocumentWalkerTest.java @@ -6,6 +6,7 @@ */ package io.camunda.connector.agenticai.aiagent.agent; +import static io.camunda.connector.agenticai.aiagent.agent.ContentTreeDocumentWalker.extractDocumentsFromContent; import static org.assertj.core.api.Assertions.assertThat; import io.camunda.connector.api.document.Document; @@ -18,8 +19,12 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; class ContentTreeDocumentWalkerTest { @@ -34,32 +39,27 @@ void setUp() { @Test void extractsRootLevelDocument() { final var doc = createDocument("hello", "text/plain", "test.txt"); - final var result = ContentTreeDocumentWalker.extractDocumentsFromContent(doc); - assertThat(result).containsExactly(doc); + assertThat(extractDocumentsFromContent(doc)).containsExactly(doc); } @Test void extractsDocumentFromMapValue() { final var doc = createDocument("hello", "text/plain", "test.txt"); - final var result = - ContentTreeDocumentWalker.extractDocumentsFromContent(Map.of("file", doc, "key", "value")); - assertThat(result).containsExactly(doc); + assertThat(extractDocumentsFromContent(Map.of("file", doc, "key", "value"))) + .containsExactly(doc); } @Test void extractsDocumentFromList() { final var doc = createDocument("hello", "text/plain", "test.txt"); - final var result = - ContentTreeDocumentWalker.extractDocumentsFromContent(List.of("text", doc, 42)); - assertThat(result).containsExactly(doc); + assertThat(extractDocumentsFromContent(List.of("text", doc, 42))).containsExactly(doc); } @Test void extractsDeeplyNestedDocuments() { final var doc = createDocument("hello", "text/plain", "test.txt"); final var nested = Map.of("level1", Map.of("level2", List.of(Map.of("file", doc)))); - final var result = ContentTreeDocumentWalker.extractDocumentsFromContent(nested); - assertThat(result).containsExactly(doc); + assertThat(extractDocumentsFromContent(nested)).containsExactly(doc); } @Test @@ -71,45 +71,37 @@ void extractsMultipleDocuments() { content.put("report", doc2); content.put("other", "value"); - final var result = ContentTreeDocumentWalker.extractDocumentsFromContent(content); - assertThat(result).containsExactly(doc1, doc2); + assertThat(extractDocumentsFromContent(content)).containsExactly(doc1, doc2); } @Test void returnsEmptyForContentWithoutDocuments() { - final var result = - ContentTreeDocumentWalker.extractDocumentsFromContent( - Map.of("key", "value", "list", List.of(1, 2, 3))); - assertThat(result).isEmpty(); + assertThat(extractDocumentsFromContent(Map.of("key", "value", "list", List.of(1, 2, 3)))) + .isEmpty(); } - @Test - void returnsEmptyForNullContent() { - final var result = ContentTreeDocumentWalker.extractDocumentsFromContent(null); - assertThat(result).isEmpty(); + @ParameterizedTest + @MethodSource("nullAndScalars") + void returnsEmptyForNullOrScalarContent(Object content) { + assertThat(extractDocumentsFromContent(content)).isEmpty(); } - @Test - void returnsEmptyForScalarContent() { - assertThat(ContentTreeDocumentWalker.extractDocumentsFromContent("text")).isEmpty(); - assertThat(ContentTreeDocumentWalker.extractDocumentsFromContent(42)).isEmpty(); - assertThat(ContentTreeDocumentWalker.extractDocumentsFromContent(true)).isEmpty(); + static Stream nullAndScalars() { + return Stream.of( + Arguments.of((Object) null), Arguments.of("text"), Arguments.of(42), Arguments.of(true)); } @Test void extractsDocumentFromArray() { final var doc = createDocument("hello", "text/plain", "test.txt"); - final var result = - ContentTreeDocumentWalker.extractDocumentsFromContent(new Object[] {"text", doc, 42}); - assertThat(result).containsExactly(doc); + assertThat(extractDocumentsFromContent(new Object[] {"text", doc, 42})).containsExactly(doc); } @Test void extractsDocumentFromNestedArray() { final var doc = createDocument("hello", "text/plain", "test.txt"); final var nested = Map.of("items", new Object[] {doc}); - final var result = ContentTreeDocumentWalker.extractDocumentsFromContent(nested); - assertThat(result).containsExactly(doc); + assertThat(extractDocumentsFromContent(nested)).containsExactly(doc); } @Test @@ -119,8 +111,7 @@ void handlesNullValuesInMap() { content.put("file", doc); content.put("missing", null); - final var result = ContentTreeDocumentWalker.extractDocumentsFromContent(content); - assertThat(result).containsExactly(doc); + assertThat(extractDocumentsFromContent(content)).containsExactly(doc); } @Test @@ -131,8 +122,7 @@ void handlesNullElementsInList() { content.add(null); content.add("text"); - final var result = ContentTreeDocumentWalker.extractDocumentsFromContent(content); - assertThat(result).containsExactly(doc); + assertThat(extractDocumentsFromContent(content)).containsExactly(doc); } private Document createDocument(String content, String contentType, String filename) { diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/GatewayToolResultDocumentSerializationTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/GatewayToolResultDocumentSerializationTest.java new file mode 100644 index 00000000000..0f69717b19a --- /dev/null +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/GatewayToolResultDocumentSerializationTest.java @@ -0,0 +1,220 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.agent; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.connector.agenticai.a2a.client.common.model.result.A2aArtifact; +import io.camunda.connector.agenticai.a2a.client.common.model.result.A2aMessage; +import io.camunda.connector.agenticai.a2a.client.common.model.result.A2aSendMessageResult; +import io.camunda.connector.agenticai.a2a.client.common.model.result.A2aTask; +import io.camunda.connector.agenticai.a2a.client.common.model.result.A2aTaskStatus; +import io.camunda.connector.agenticai.mcp.client.model.content.McpDocumentContent; +import io.camunda.connector.agenticai.mcp.client.model.content.McpEmbeddedResourceContent; +import io.camunda.connector.agenticai.mcp.client.model.content.McpEmbeddedResourceContent.BlobDocumentResource; +import io.camunda.connector.agenticai.mcp.client.model.content.McpTextContent; +import io.camunda.connector.agenticai.mcp.client.model.result.McpClientCallToolResult; +import io.camunda.connector.agenticai.model.message.content.DocumentContent; +import io.camunda.connector.agenticai.model.message.content.TextContent; +import io.camunda.connector.agenticai.util.TestObjectMapperSupplier; +import io.camunda.connector.api.document.Document; +import io.camunda.connector.api.document.DocumentCreationRequest; +import io.camunda.connector.api.document.DocumentFactory; +import io.camunda.connector.api.document.DocumentReference.CamundaDocumentReference; +import io.camunda.connector.runtime.core.document.DocumentFactoryImpl; +import io.camunda.connector.runtime.core.document.store.InMemoryDocumentStore; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Regression test pinning the JSON wire format that the connector ObjectMapper produces for + * Documents nested inside the typed gateway result objects that {@code McpClientGatewayToolHandler} + * and {@code A2aGatewayToolHandler} put on the transformed {@code ToolCallResult.content()}. + * + *

    The contract: every {@link Document} reachable from {@code McpClientCallToolResult} or {@code + * A2aSendMessageResult} must serialize through the standard {@code DocumentSerializer} into a + * {@code camunda.document.type} reference — never as base64 blob data and never as a raw {@code + * DocumentReference} POJO. This is the property that lets the LLM correlate references it sees in + * the tool result text with the actual document content delivered in the synthetic user message + * (see ADR-004) and that lets the message be persisted/replayed losslessly. + */ +class GatewayToolResultDocumentSerializationTest { + + private final ObjectMapper objectMapper = TestObjectMapperSupplier.INSTANCE; + private final InMemoryDocumentStore documentStore = InMemoryDocumentStore.INSTANCE; + private final DocumentFactory documentFactory = new DocumentFactoryImpl(documentStore); + + @BeforeEach + void resetStore() { + documentStore.clear(); + } + + @Test + void mcpDocumentContent_serializesAsCamundaDocumentReference() throws Exception { + final var document = createDocument("hello", "text/plain", "test.txt"); + final var callToolResult = + new McpClientCallToolResult( + "tool1", + List.of( + McpTextContent.textContent("intro"), new McpDocumentContent(document, Map.of())), + false); + + final var json = objectMapper.writeValueAsString(callToolResult); + final var docNode = findFirstCamundaDocumentNode(objectMapper.readTree(json)); + + assertCamundaDocumentReference(docNode, document, "text/plain"); + } + + @Test + void mcpEmbeddedBlobDocumentResource_serializesAsCamundaDocumentReference() throws Exception { + final var document = createDocument("", "application/pdf", "report.pdf"); + final var callToolResult = + new McpClientCallToolResult( + "tool1", + List.of( + new McpEmbeddedResourceContent( + new BlobDocumentResource("uri://doc", "application/pdf", document), Map.of())), + false); + + final var json = objectMapper.writeValueAsString(callToolResult); + final var docNode = findFirstCamundaDocumentNode(objectMapper.readTree(json)); + + assertCamundaDocumentReference(docNode, document, "application/pdf"); + } + + @Test + void a2aMessageDocumentContent_serializesAsCamundaDocumentReference() throws Exception { + final var document = createDocument("hello", "text/plain", "msg.txt"); + final A2aSendMessageResult message = + A2aMessage.builder() + .role(A2aMessage.Role.AGENT) + .messageId("msg-1") + .contextId("ctx-1") + .contents( + List.of( + TextContent.textContent("intro"), DocumentContent.documentContent(document))) + .build(); + + final var json = objectMapper.writeValueAsString(message); + final var docNode = findFirstCamundaDocumentNode(objectMapper.readTree(json)); + + assertCamundaDocumentReference(docNode, document, "text/plain"); + } + + @Test + void a2aTaskArtifactsAndHistoryDocuments_serializeAsCamundaDocumentReferences() throws Exception { + final var artifactDoc = createDocument("art", "image/png", "image.png"); + final var historyDoc = createDocument("hist", "application/pdf", "history.pdf"); + + final var artifact = + A2aArtifact.builder() + .artifactId("art-1") + .contents(List.of(DocumentContent.documentContent(artifactDoc))) + .build(); + final var historyMessage = + A2aMessage.builder() + .role(A2aMessage.Role.AGENT) + .messageId("msg-1") + .contextId("ctx-1") + .contents(List.of(DocumentContent.documentContent(historyDoc))) + .build(); + final A2aSendMessageResult task = + A2aTask.builder() + .id("task-1") + .contextId("ctx-1") + .status(A2aTaskStatus.builder().state(A2aTaskStatus.TaskState.COMPLETED).build()) + .artifacts(List.of(artifact)) + .history(List.of(historyMessage)) + .build(); + + final var json = objectMapper.writeValueAsString(task); + final var rootNode = objectMapper.readTree(json); + + final var artifactDocNode = findFirstCamundaDocumentNode(rootNode.path("artifacts")); + final var historyDocNode = findFirstCamundaDocumentNode(rootNode.path("history")); + + assertCamundaDocumentReference(artifactDocNode, artifactDoc, "image/png"); + assertCamundaDocumentReference(historyDocNode, historyDoc, "application/pdf"); + } + + @Test + void serializedJsonContainsNoBase64BlobForCamundaDocuments() throws Exception { + final var document = createDocument("payload bytes", "application/octet-stream", "blob.bin"); + final var callToolResult = + new McpClientCallToolResult( + "tool1", List.of(new McpDocumentContent(document, Map.of())), false); + + final var json = objectMapper.writeValueAsString(callToolResult); + + // base64 of "payload bytes" — must NOT appear in serialized output (would mean the document + // was inlined as raw blob data instead of a reference). + final var base64 = + java.util.Base64.getEncoder() + .encodeToString("payload bytes".getBytes(StandardCharsets.UTF_8)); + assertThat(json) + .doesNotContain(base64) + .contains("camunda.document.type") + .contains("\"documentId\""); + } + + private void assertCamundaDocumentReference( + JsonNode docNode, Document expectedDocument, String expectedContentType) { + assertThat(docNode).as("camunda document reference node found in JSON").isNotNull(); + assertThat(docNode.path("camunda.document.type").asText()) + .as("camunda.document.type discriminator") + .isEqualTo("camunda"); + assertThat(expectedDocument.reference()).isInstanceOf(CamundaDocumentReference.class); + final var camundaReference = (CamundaDocumentReference) expectedDocument.reference(); + assertThat(docNode.path("documentId").asText()) + .as("documentId") + .isEqualTo(camundaReference.getDocumentId()); + if (expectedContentType != null) { + assertThat(docNode.path("metadata").path("contentType").asText()) + .as("metadata.contentType") + .isEqualTo(expectedContentType); + } + } + + private static JsonNode findFirstCamundaDocumentNode(JsonNode node) { + if (node == null || node.isMissingNode()) { + return null; + } + if (node.isObject()) { + if ("camunda".equals(node.path("camunda.document.type").asText(null))) { + return node; + } + final var fields = node.fields(); + while (fields.hasNext()) { + final var found = findFirstCamundaDocumentNode(fields.next().getValue()); + if (found != null) { + return found; + } + } + } else if (node.isArray()) { + for (var element : node) { + final var found = findFirstCamundaDocumentNode(element); + if (found != null) { + return found; + } + } + } + return null; + } + + private Document createDocument(String content, String contentType, String filename) { + return documentFactory.create( + DocumentCreationRequest.from(content.getBytes(StandardCharsets.UTF_8)) + .contentType(contentType) + .fileName(filename) + .build()); + } +} From 1943df9acf9ce98861938e622c1d327d07ce8303 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Apr 2026 15:03:30 +0000 Subject: [PATCH 31/81] test(agentic-ai): drop redundant GatewayToolResultDocumentSerializationTest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SDK already covers Camunda document serialization fidelity in DocumentSerializerTest (connector-runtime/jackson-datatype-document). Once a Document field is reachable by Jackson and the document module is registered, DocumentSerializer is invoked regardless of the parent type — neither McpDocumentContent, BlobDocumentResource, nor A2A DocumentContent overrides this. The test added in 843e2e2 only re-ran the SDK serializer through different wrapper objects and would have failed in lockstep with the SDK tests, providing no additional signal. It also mixed MCP and A2A concerns in one test class, which should have been split per handler if anything. https://claude.ai/code/session_01SM8HzedSAVWqnDaEKrmCpR --- ...ayToolResultDocumentSerializationTest.java | 220 ------------------ 1 file changed, 220 deletions(-) delete mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/GatewayToolResultDocumentSerializationTest.java diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/GatewayToolResultDocumentSerializationTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/GatewayToolResultDocumentSerializationTest.java deleted file mode 100644 index 0f69717b19a..00000000000 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/GatewayToolResultDocumentSerializationTest.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH - * under one or more contributor license agreements. Licensed under a proprietary license. - * See the License.txt file for more information. You may not use this file - * except in compliance with the proprietary license. - */ -package io.camunda.connector.agenticai.aiagent.agent; - -import static org.assertj.core.api.Assertions.assertThat; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.camunda.connector.agenticai.a2a.client.common.model.result.A2aArtifact; -import io.camunda.connector.agenticai.a2a.client.common.model.result.A2aMessage; -import io.camunda.connector.agenticai.a2a.client.common.model.result.A2aSendMessageResult; -import io.camunda.connector.agenticai.a2a.client.common.model.result.A2aTask; -import io.camunda.connector.agenticai.a2a.client.common.model.result.A2aTaskStatus; -import io.camunda.connector.agenticai.mcp.client.model.content.McpDocumentContent; -import io.camunda.connector.agenticai.mcp.client.model.content.McpEmbeddedResourceContent; -import io.camunda.connector.agenticai.mcp.client.model.content.McpEmbeddedResourceContent.BlobDocumentResource; -import io.camunda.connector.agenticai.mcp.client.model.content.McpTextContent; -import io.camunda.connector.agenticai.mcp.client.model.result.McpClientCallToolResult; -import io.camunda.connector.agenticai.model.message.content.DocumentContent; -import io.camunda.connector.agenticai.model.message.content.TextContent; -import io.camunda.connector.agenticai.util.TestObjectMapperSupplier; -import io.camunda.connector.api.document.Document; -import io.camunda.connector.api.document.DocumentCreationRequest; -import io.camunda.connector.api.document.DocumentFactory; -import io.camunda.connector.api.document.DocumentReference.CamundaDocumentReference; -import io.camunda.connector.runtime.core.document.DocumentFactoryImpl; -import io.camunda.connector.runtime.core.document.store.InMemoryDocumentStore; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Map; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -/** - * Regression test pinning the JSON wire format that the connector ObjectMapper produces for - * Documents nested inside the typed gateway result objects that {@code McpClientGatewayToolHandler} - * and {@code A2aGatewayToolHandler} put on the transformed {@code ToolCallResult.content()}. - * - *

    The contract: every {@link Document} reachable from {@code McpClientCallToolResult} or {@code - * A2aSendMessageResult} must serialize through the standard {@code DocumentSerializer} into a - * {@code camunda.document.type} reference — never as base64 blob data and never as a raw {@code - * DocumentReference} POJO. This is the property that lets the LLM correlate references it sees in - * the tool result text with the actual document content delivered in the synthetic user message - * (see ADR-004) and that lets the message be persisted/replayed losslessly. - */ -class GatewayToolResultDocumentSerializationTest { - - private final ObjectMapper objectMapper = TestObjectMapperSupplier.INSTANCE; - private final InMemoryDocumentStore documentStore = InMemoryDocumentStore.INSTANCE; - private final DocumentFactory documentFactory = new DocumentFactoryImpl(documentStore); - - @BeforeEach - void resetStore() { - documentStore.clear(); - } - - @Test - void mcpDocumentContent_serializesAsCamundaDocumentReference() throws Exception { - final var document = createDocument("hello", "text/plain", "test.txt"); - final var callToolResult = - new McpClientCallToolResult( - "tool1", - List.of( - McpTextContent.textContent("intro"), new McpDocumentContent(document, Map.of())), - false); - - final var json = objectMapper.writeValueAsString(callToolResult); - final var docNode = findFirstCamundaDocumentNode(objectMapper.readTree(json)); - - assertCamundaDocumentReference(docNode, document, "text/plain"); - } - - @Test - void mcpEmbeddedBlobDocumentResource_serializesAsCamundaDocumentReference() throws Exception { - final var document = createDocument("", "application/pdf", "report.pdf"); - final var callToolResult = - new McpClientCallToolResult( - "tool1", - List.of( - new McpEmbeddedResourceContent( - new BlobDocumentResource("uri://doc", "application/pdf", document), Map.of())), - false); - - final var json = objectMapper.writeValueAsString(callToolResult); - final var docNode = findFirstCamundaDocumentNode(objectMapper.readTree(json)); - - assertCamundaDocumentReference(docNode, document, "application/pdf"); - } - - @Test - void a2aMessageDocumentContent_serializesAsCamundaDocumentReference() throws Exception { - final var document = createDocument("hello", "text/plain", "msg.txt"); - final A2aSendMessageResult message = - A2aMessage.builder() - .role(A2aMessage.Role.AGENT) - .messageId("msg-1") - .contextId("ctx-1") - .contents( - List.of( - TextContent.textContent("intro"), DocumentContent.documentContent(document))) - .build(); - - final var json = objectMapper.writeValueAsString(message); - final var docNode = findFirstCamundaDocumentNode(objectMapper.readTree(json)); - - assertCamundaDocumentReference(docNode, document, "text/plain"); - } - - @Test - void a2aTaskArtifactsAndHistoryDocuments_serializeAsCamundaDocumentReferences() throws Exception { - final var artifactDoc = createDocument("art", "image/png", "image.png"); - final var historyDoc = createDocument("hist", "application/pdf", "history.pdf"); - - final var artifact = - A2aArtifact.builder() - .artifactId("art-1") - .contents(List.of(DocumentContent.documentContent(artifactDoc))) - .build(); - final var historyMessage = - A2aMessage.builder() - .role(A2aMessage.Role.AGENT) - .messageId("msg-1") - .contextId("ctx-1") - .contents(List.of(DocumentContent.documentContent(historyDoc))) - .build(); - final A2aSendMessageResult task = - A2aTask.builder() - .id("task-1") - .contextId("ctx-1") - .status(A2aTaskStatus.builder().state(A2aTaskStatus.TaskState.COMPLETED).build()) - .artifacts(List.of(artifact)) - .history(List.of(historyMessage)) - .build(); - - final var json = objectMapper.writeValueAsString(task); - final var rootNode = objectMapper.readTree(json); - - final var artifactDocNode = findFirstCamundaDocumentNode(rootNode.path("artifacts")); - final var historyDocNode = findFirstCamundaDocumentNode(rootNode.path("history")); - - assertCamundaDocumentReference(artifactDocNode, artifactDoc, "image/png"); - assertCamundaDocumentReference(historyDocNode, historyDoc, "application/pdf"); - } - - @Test - void serializedJsonContainsNoBase64BlobForCamundaDocuments() throws Exception { - final var document = createDocument("payload bytes", "application/octet-stream", "blob.bin"); - final var callToolResult = - new McpClientCallToolResult( - "tool1", List.of(new McpDocumentContent(document, Map.of())), false); - - final var json = objectMapper.writeValueAsString(callToolResult); - - // base64 of "payload bytes" — must NOT appear in serialized output (would mean the document - // was inlined as raw blob data instead of a reference). - final var base64 = - java.util.Base64.getEncoder() - .encodeToString("payload bytes".getBytes(StandardCharsets.UTF_8)); - assertThat(json) - .doesNotContain(base64) - .contains("camunda.document.type") - .contains("\"documentId\""); - } - - private void assertCamundaDocumentReference( - JsonNode docNode, Document expectedDocument, String expectedContentType) { - assertThat(docNode).as("camunda document reference node found in JSON").isNotNull(); - assertThat(docNode.path("camunda.document.type").asText()) - .as("camunda.document.type discriminator") - .isEqualTo("camunda"); - assertThat(expectedDocument.reference()).isInstanceOf(CamundaDocumentReference.class); - final var camundaReference = (CamundaDocumentReference) expectedDocument.reference(); - assertThat(docNode.path("documentId").asText()) - .as("documentId") - .isEqualTo(camundaReference.getDocumentId()); - if (expectedContentType != null) { - assertThat(docNode.path("metadata").path("contentType").asText()) - .as("metadata.contentType") - .isEqualTo(expectedContentType); - } - } - - private static JsonNode findFirstCamundaDocumentNode(JsonNode node) { - if (node == null || node.isMissingNode()) { - return null; - } - if (node.isObject()) { - if ("camunda".equals(node.path("camunda.document.type").asText(null))) { - return node; - } - final var fields = node.fields(); - while (fields.hasNext()) { - final var found = findFirstCamundaDocumentNode(fields.next().getValue()); - if (found != null) { - return found; - } - } - } else if (node.isArray()) { - for (var element : node) { - final var found = findFirstCamundaDocumentNode(element); - if (found != null) { - return found; - } - } - } - return null; - } - - private Document createDocument(String content, String contentType, String filename) { - return documentFactory.create( - DocumentCreationRequest.from(content.getBytes(StandardCharsets.UTF_8)) - .contentType(contentType) - .fileName(filename) - .build()); - } -} From fc68c58f36d0a3ba6ebcaa7b442501705a25fce8 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Apr 2026 15:34:23 +0000 Subject: [PATCH 32/81] =?UTF-8?q?chore(agentic-ai):=20address=20review=20n?= =?UTF-8?q?its=20=E2=80=94=20JsonNode.fields()=20deprecation,=20stale=20AD?= =?UTF-8?q?R=20reference?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * AiAgentTestFixtures.findFirstCamundaDocumentNode: replace deprecated JsonNode.fields() with JsonNode.properties() (CodeQL: deprecated method invocation). * ADR-004: the "Per-handler document extraction" subsection still pointed at ContentTreeDocumentWalker.INSTANCE. The walker is now a static utility — point readers at extractDocumentsFromContent(...) instead. The Java javadoc on GatewayToolHandler was already fixed; this brings the ADR in line. https://claude.ai/code/session_01SM8HzedSAVWqnDaEKrmCpR --- .../e2e/agenticai/aiagent/AiAgentTestFixtures.java | 6 +++--- .../docs/adr/004-document-handling-in-tool-call-results.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/AiAgentTestFixtures.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/AiAgentTestFixtures.java index 0786cf92d6e..1fcc78c1938 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/AiAgentTestFixtures.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/AiAgentTestFixtures.java @@ -148,9 +148,9 @@ private static JsonNode findFirstCamundaDocumentNode(JsonNode node) { if ("camunda".equals(node.path("camunda.document.type").asText(null))) { return node; } - final var fields = node.fields(); - while (fields.hasNext()) { - final var found = findFirstCamundaDocumentNode(fields.next().getValue()); + final var properties = node.properties(); + for (var property : properties) { + final var found = findFirstCamundaDocumentNode(property.getValue()); if (found != null) { return found; } diff --git a/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md b/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md index a76723931a5..d7159c7c995 100644 --- a/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md +++ b/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md @@ -135,7 +135,7 @@ BPMN tool calls whose `content()` is a raw `Map`/`List`/`Document` tree from the `Object[]`, and `Document`. The walker is also public: handler implementations whose typed content embeds raw user-generated subtrees (e.g. opaque -`Map` payloads from a downstream system) can call `ContentTreeDocumentWalker.INSTANCE` for those parts. +`Map` payloads from a downstream system) can call `ContentTreeDocumentWalker.extractDocumentsFromContent(...)` for those parts. The default `GatewayToolHandler.extractDocuments` implementation delegates to the walker, so third-party handlers that return raw content do not need to override anything. From 81ba14cc291661ae291cc9cf05156d0acea8c608 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Mon, 4 May 2026 14:56:04 +0200 Subject: [PATCH 33/81] chore(agentic-ai): reformat tests --- .../e2e/agenticai/aiagent/DocumentToolCallResultsIT.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/DocumentToolCallResultsIT.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/DocumentToolCallResultsIT.java index c5e44d4be2c..d9f4497dbfa 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/DocumentToolCallResultsIT.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/DocumentToolCallResultsIT.java @@ -27,6 +27,7 @@ import io.camunda.client.CamundaClient; import io.camunda.connector.e2e.BpmnFile; import io.camunda.connector.e2e.ElementTemplate; +import io.camunda.connector.e2e.ZeebeTest; import io.camunda.connector.e2e.agenticai.CamundaDocumentTestConfiguration; import io.camunda.connector.e2e.app.TestConnectorRuntimeApplication; import io.camunda.connector.runtime.core.document.store.InMemoryDocumentStore; @@ -225,6 +226,8 @@ void nestedStructureDocumentsFromToolCallResult( static Stream providers() { List> modelFilters = new ArrayList<>(); + + // sample filter // modelFilters.add(p -> p.label().contains("gpt-4.1")); return Stream.of( @@ -339,7 +342,7 @@ private io.camunda.client.api.response.ProcessInstanceEvent startProcess( var model = buildModel(provider); // deploy and wait for process definition to be available - io.camunda.connector.e2e.ZeebeTest.with(camundaClient).awaitCompleteTopology().deploy(model); + ZeebeTest.with(camundaClient).awaitCompleteTopology().deploy(model); return camundaClient .newCreateInstanceCommand() From 6c0009c07c4be3d540957419213cf9b50f6fa952 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 11:14:14 +0200 Subject: [PATCH 34/81] docs(agentic-ai): reframe ADR-004 multi-content limitation as provider + L4J adapter combo --- .../004-document-handling-in-tool-call-results.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md b/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md index d7159c7c995..36ef86513e5 100644 --- a/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md +++ b/connectors/agentic-ai/docs/adr/004-document-handling-in-tool-call-results.md @@ -16,7 +16,12 @@ the documents need to be represented in a way the model can actually interpret. The current implementation converts `ToolCallResult.content()` to a single JSON text string for LangChain4J's `ToolExecutionResultMessage`. Documents encountered during Jackson serialization are converted to a Claude-specific content block format (`DocumentToContentSerializer`) containing base64-encoded data embedded within the text. This -approach does not work -- models cannot meaningfully interpret large base64 blobs embedded in tool result text. +approach does not work -- models cannot meaningfully interpret large base64 blobs embedded in tool result text. The +natural alternative — passing documents as native multi-modal content on the tool result — is also blocked today by +how providers and LangChain4J interact: providers vary in what they accept as tool result content (e.g. the OpenAI +Responses API and Anthropic accept rich content blocks; the OpenAI Chat Completions API is narrower), and LangChain4J's +provider adapters only partially expose that capability (e.g. `OpenAiChatModel` surfaces images but not other binary +types). See "Option 1" below for the full rationale. In contrast, documents provided via user messages already work correctly: they are converted to proper LangChain4J content types (`ImageContent`, `PdfFileContent`, `TextContent`) through `DocumentToContentConverterImpl` and arrive at @@ -41,7 +46,11 @@ Extract documents from tool call results and add them as separate `Content` bloc **Rejected** because: -- LangChain4J provider adapters have inconsistent support for multi-content tool results across providers. +- Native multi-content tool results are limited by a combination of factors: providers themselves vary in what + they accept as tool result content (e.g. the OpenAI Responses API and Anthropic accept rich content blocks; the + OpenAI Chat Completions API is narrower), and LangChain4J's provider adapters only partially expose that + capability — `OpenAiChatModel`, for instance, surfaces images but not other binary types, and broader coverage + would require migrating to richer adapters such as `OpenAiResponsesChatModel`. - Changes would be invisible in the conversation history (only visible at the L4J wire format level). - Tightly couples the solution to LangChain4J capabilities. From 71456c5a9db1efdb8173caeecc07a541cd6b48d3 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 12:11:34 +0200 Subject: [PATCH 35/81] test(e2e): extract document tool-call assertions into reusable helper Moves the synthetic UserMessage assertion logic, document reference parsing, and the content-block helper out of AiAgentTestFixtures and the per-test duplicates into a dedicated ToolCallResultDocumentAssertions class. assertExtractedDocumentsUserMessage takes ExtractedDocument varargs so it can also assert UserMessages carrying multiple tool call results, and builds expected XML tags via the production DocumentXmlTag. Parsed references use the production DocumentReferenceModel.CamundaDocumentReferenceModel, keeping the test in lockstep with the on-the-wire format. --- .../aiagent/AiAgentTestFixtures.java | 76 ------ .../ToolCallResultDocumentAssertions.java | 216 ++++++++++++++++++ ...4JAiAgentJobWorkerMcpIntegrationTests.java | 52 ++--- .../L4JAiAgentJobWorkerToolCallingTests.java | 62 ++--- ...4JAiAgentConnectorMcpIntegrationTests.java | 52 ++--- .../L4JAiAgentConnectorToolCallingTests.java | 62 ++--- 6 files changed, 280 insertions(+), 240 deletions(-) create mode 100644 connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/ToolCallResultDocumentAssertions.java diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/AiAgentTestFixtures.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/AiAgentTestFixtures.java index 1fcc78c1938..80831fdfbe3 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/AiAgentTestFixtures.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/AiAgentTestFixtures.java @@ -19,9 +19,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.entry; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Map; import org.assertj.core.api.InstanceOfAssertFactories; import org.assertj.core.api.ThrowingConsumer; @@ -92,77 +89,4 @@ public interface AiAgentTestFixtures { String FEEDBACK_LOOP_RESPONSE_TEXT = "A very complex calculation only the superflux calculation tool can do."; - - ObjectMapper TOOL_RESULT_OBJECT_MAPPER = new ObjectMapper(); - - /** - * Parsed Camunda document reference fields extracted from a tool call result JSON. The {@link - * #shortId()} returns the first UUID segment of the {@code documentId}, used as the compact - * correlation key in the synthetic {@code } XML tag. - */ - record DocumentReferenceFields( - String storeId, String documentId, String contentType, String fileName) { - public String shortId() { - int dash = documentId.indexOf('-'); - return dash > 0 ? documentId.substring(0, dash) : documentId; - } - } - - /** - * Locates the first Camunda document reference inside a serialized tool call result and reads its - * structural fields. Recursively descends into objects/arrays until it finds an object carrying - * the {@code camunda.document.type} discriminator (set by {@code DocumentSerializer}). - * - *

    Use this in place of regex-matching the raw text — both for asserting expected reference - * fields (content type, file name) and for extracting the document short ID. - */ - static DocumentReferenceFields readDocumentReference(String toolResultText) { - final JsonNode root; - try { - root = TOOL_RESULT_OBJECT_MAPPER.readTree(toolResultText); - } catch (JsonProcessingException e) { - throw new AssertionError("Failed to parse tool result text as JSON: " + toolResultText, e); - } - - final JsonNode docNode = findFirstCamundaDocumentNode(root); - assertThat(docNode) - .as("Camunda document reference in tool result text: %s", toolResultText) - .isNotNull(); - assertThat(docNode.path("camunda.document.type").asText()) - .as("camunda.document.type discriminator") - .isEqualTo("camunda"); - assertThat(docNode.path("documentId").asText()).as("documentId").isNotBlank(); - - return new DocumentReferenceFields( - docNode.path("storeId").asText(null), - docNode.path("documentId").asText(), - docNode.path("metadata").path("contentType").asText(null), - docNode.path("metadata").path("fileName").asText(null)); - } - - private static JsonNode findFirstCamundaDocumentNode(JsonNode node) { - if (node == null) { - return null; - } - if (node.isObject()) { - if ("camunda".equals(node.path("camunda.document.type").asText(null))) { - return node; - } - final var properties = node.properties(); - for (var property : properties) { - final var found = findFirstCamundaDocumentNode(property.getValue()); - if (found != null) { - return found; - } - } - } else if (node.isArray()) { - for (var element : node) { - final var found = findFirstCamundaDocumentNode(element); - if (found != null) { - return found; - } - } - } - return null; - } } diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/ToolCallResultDocumentAssertions.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/ToolCallResultDocumentAssertions.java new file mode 100644 index 00000000000..bab4f82bde8 --- /dev/null +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/ToolCallResultDocumentAssertions.java @@ -0,0 +1,216 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. Camunda licenses this file to you under the Apache License, + * Version 2.0; you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.camunda.connector.e2e.agenticai.aiagent; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.langchain4j.data.message.ChatMessage; +import dev.langchain4j.data.message.Content; +import dev.langchain4j.data.message.ImageContent; +import dev.langchain4j.data.message.PdfFileContent; +import dev.langchain4j.data.message.TextContent; +import dev.langchain4j.data.message.UserMessage; +import io.camunda.connector.agenticai.model.message.DocumentXmlTag; +import io.camunda.connector.document.jackson.DocumentReferenceModel; +import io.camunda.connector.document.jackson.DocumentReferenceModel.CamundaDocumentReferenceModel; +import java.util.function.Consumer; + +/** + * Assertion helpers for the synthetic {@link UserMessage} that carries documents extracted from + * tool call results. + * + *

    The user message has the structure {@code preamble + (xml-tag, content-block)*}, so it can + * carry documents from one or many tool call results. Use {@link + * #assertExtractedDocumentsUserMessage(ChatMessage, ExtractedDocument...)} to assert against any + * number of documents in one go. + */ +public final class ToolCallResultDocumentAssertions { + + static final String EXTRACTED_DOCUMENTS_PREAMBLE = "Documents extracted from tool call results:"; + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private ToolCallResultDocumentAssertions() {} + + /** + * Locates the first Camunda document reference inside a serialized tool call result and + * deserializes it into the production {@link CamundaDocumentReferenceModel}. Recursively descends + * into objects/arrays until it finds an object carrying the {@code camunda.document.type} + * discriminator (set by {@code DocumentSerializer}). + * + *

    Throws {@link AssertionError} if the text cannot be parsed as JSON or no document reference + * is present. + */ + public static CamundaDocumentReferenceModel parseDocumentReference(String toolResultText) { + final JsonNode root; + try { + root = OBJECT_MAPPER.readTree(toolResultText); + } catch (JsonProcessingException e) { + throw new AssertionError("Failed to parse tool result text as JSON: " + toolResultText, e); + } + + final JsonNode docNode = findFirstCamundaDocumentNode(root); + if (docNode == null) { + throw new AssertionError( + "No Camunda document reference found in tool result text: " + toolResultText); + } + + try { + return OBJECT_MAPPER.treeToValue(docNode, CamundaDocumentReferenceModel.class); + } catch (JsonProcessingException e) { + throw new AssertionError("Failed to deserialize Camunda document reference: " + docNode, e); + } + } + + /** + * Asserts that {@code message} is a {@link UserMessage} with the standard "extracted documents" + * structure: a preamble {@link TextContent} followed by an {@code (xml-tag, content-block)} pair + * per expected document. + * + *

    The XML tag is built using the production {@link DocumentXmlTag}, so the assertion stays in + * sync with the production format. + */ + public static void assertExtractedDocumentsUserMessage( + ChatMessage message, ExtractedDocument... expectedDocuments) { + assertThat(message).isInstanceOf(UserMessage.class); + final var contents = ((UserMessage) message).contents(); + + assertThat(contents) + .as("expected preamble + 2 contents per document (%d documents)", expectedDocuments.length) + .hasSize(1 + 2 * expectedDocuments.length); + + assertThat(contents.get(0)) + .as("preamble TextContent") + .isInstanceOfSatisfying( + TextContent.class, tc -> assertThat(tc.text()).isEqualTo(EXTRACTED_DOCUMENTS_PREAMBLE)); + + for (int i = 0; i < expectedDocuments.length; i++) { + final var expected = expectedDocuments[i]; + final var expectedXml = expected.expectedXmlTag(); + final var tagIndex = 1 + 2 * i; + final var contentIndex = tagIndex + 1; + + assertThat(contents.get(tagIndex)) + .as("XML tag at index %d (document %d)", tagIndex, i) + .isInstanceOfSatisfying( + TextContent.class, tc -> assertThat(tc.text()).isEqualTo(expectedXml)); + + final var contentBlock = contents.get(contentIndex); + try { + expected.contentBlockAssertion().accept(contentBlock); + } catch (AssertionError e) { + throw new AssertionError( + "Content block at index %d (document %d) failed: %s" + .formatted(contentIndex, i, e.getMessage()), + e); + } + } + } + + /** + * Asserts a content block based on a coarse type/mime classification used by tool calling tests: + * + *

      + *
    • {@code "text"} → {@link TextContent} + *
    • {@code "application/pdf"} → {@link PdfFileContent} with non-blank base64 + *
    • otherwise → {@link ImageContent} with matching mime type and non-blank base64 + *
    + */ + public static void assertDocumentContentBlock( + Content content, String expectedType, String expectedMimeType) { + if ("text".equals(expectedType)) { + assertThat(content).isInstanceOf(TextContent.class); + } else if ("application/pdf".equals(expectedMimeType)) { + assertThat(content) + .isInstanceOfSatisfying( + PdfFileContent.class, pdf -> assertThat(pdf.pdfFile().base64Data()).isNotBlank()); + } else { + assertThat(content) + .isInstanceOfSatisfying( + ImageContent.class, + img -> { + assertThat(img.image().mimeType()).isEqualTo(expectedMimeType); + assertThat(img.image().base64Data()).isNotBlank(); + }); + } + } + + private static JsonNode findFirstCamundaDocumentNode(JsonNode node) { + if (node == null) { + return null; + } + if (node.isObject()) { + if ("camunda".equals(node.path(DocumentReferenceModel.DISCRIMINATOR_KEY).asText(null))) { + return node; + } + final var properties = node.properties(); + for (var property : properties) { + final var found = findFirstCamundaDocumentNode(property.getValue()); + if (found != null) { + return found; + } + } + } else if (node.isArray()) { + for (var element : node) { + final var found = findFirstCamundaDocumentNode(element); + if (found != null) { + return found; + } + } + } + return null; + } + + /** + * Specification of one expected extracted document slot in the synthetic user message: an XML tag + * with optional tool call correlation attributes plus a content block check. + */ + public record ExtractedDocument( + String toolCallId, + String toolName, + CamundaDocumentReferenceModel reference, + Consumer contentBlockAssertion) { + + /** Document extracted from a tool call result. */ + public static ExtractedDocument forToolCall( + String toolCallId, + String toolName, + CamundaDocumentReferenceModel reference, + Consumer contentBlockAssertion) { + return new ExtractedDocument(toolCallId, toolName, reference, contentBlockAssertion); + } + + /** The XML tag string this document is expected to render to in the synthetic user message. */ + String expectedXmlTag() { + return new DocumentXmlTag( + toolName, + toolCallId, + documentShortId(), + reference.metadata() != null ? reference.metadata().fileName() : null) + .toXml(); + } + + private String documentShortId() { + final var documentId = reference.documentId(); + final int dash = documentId.indexOf('-'); + return dash > 0 ? documentId.substring(0, dash) : documentId; + } + } +} diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerMcpIntegrationTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerMcpIntegrationTests.java index 6de98cb7186..03ecba8f73f 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerMcpIntegrationTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerMcpIntegrationTests.java @@ -17,7 +17,8 @@ package io.camunda.connector.e2e.agenticai.aiagent.langchain4j.jobworker; import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.HAIKU_TEXT; -import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.readDocumentReference; +import static io.camunda.connector.e2e.agenticai.aiagent.ToolCallResultDocumentAssertions.assertExtractedDocumentsUserMessage; +import static io.camunda.connector.e2e.agenticai.aiagent.ToolCallResultDocumentAssertions.parseDocumentReference; import static io.camunda.connector.e2e.agenticai.aiagent.langchain4j.Langchain4JAiAgentToolSpecifications.EXPECTED_MCP_TOOL_SPECIFICATIONS; import static io.camunda.connector.e2e.agenticai.mcp.McpSdkToolSpecifications.MCP_TOOL_SPECIFICATIONS; import static org.assertj.core.api.Assertions.assertThat; @@ -34,10 +35,8 @@ import dev.langchain4j.agent.tool.ToolExecutionRequest; import dev.langchain4j.agent.tool.ToolSpecification; import dev.langchain4j.data.message.AiMessage; -import dev.langchain4j.data.message.Content; import dev.langchain4j.data.message.ImageContent; import dev.langchain4j.data.message.SystemMessage; -import dev.langchain4j.data.message.TextContent; import dev.langchain4j.data.message.ToolExecutionResultMessage; import dev.langchain4j.data.message.UserMessage; import dev.langchain4j.model.chat.request.ChatRequest; @@ -53,6 +52,7 @@ import io.camunda.connector.agenticai.mcp.client.model.McpRemoteClientTransportConfiguration; import io.camunda.connector.agenticai.mcp.client.model.McpRemoteClientTransportConfiguration.SseHttpMcpRemoteClientTransportConfiguration; import io.camunda.connector.agenticai.mcp.client.model.McpRemoteClientTransportConfiguration.StreamableHttpMcpRemoteClientTransportConfiguration; +import io.camunda.connector.e2e.agenticai.aiagent.ToolCallResultDocumentAssertions.ExtractedDocument; import io.camunda.connector.e2e.agenticai.assertj.JobWorkerAgentResponseAssert; import io.camunda.connector.test.utils.annotation.SlowTest; import io.camunda.process.test.api.CamundaAssert; @@ -403,7 +403,7 @@ void extractsDocumentsFromMcpImageToolCallResult() throws IOException { // tool result: document serialized as document reference var toolResultText = ((ToolExecutionResultMessage) lastMessages.get(3)).text(); - var documentReference = readDocumentReference(toolResultText); + var documentReference = parseDocumentReference(toolResultText); assertThat(lastMessages.get(3)) .isInstanceOfSatisfying( @@ -412,37 +412,23 @@ void extractsDocumentsFromMcpImageToolCallResult() throws IOException { assertThat(msg.id()).isEqualTo("img111"); assertThat(msg.toolName()).isEqualTo("MCP_A_MCP_Client___toolA"); }); - assertThat(documentReference.contentType()).isEqualTo("image/png"); + assertThat(documentReference.metadata().contentType()).isEqualTo("image/png"); // document user message: extracted document content - assertThat(lastMessages.get(4)) - .isInstanceOfSatisfying( - UserMessage.class, - msg -> { - List contents = msg.contents(); - assertThat(contents).hasSize(3); - assertThat(contents.get(0)) - .isInstanceOfSatisfying( - TextContent.class, - tc -> - assertThat(tc.text()) - .isEqualTo("Documents extracted from tool call results:")); - assertThat(contents.get(1)) - .isInstanceOfSatisfying( - TextContent.class, - tc -> - assertThat(tc.text()) - .isEqualTo( - "" - .formatted(documentReference.shortId()))); - assertThat(contents.get(2)) - .isInstanceOfSatisfying( - ImageContent.class, - img -> { - assertThat(img.image().mimeType()).isEqualTo("image/png"); - assertThat(img.image().base64Data()).isEqualTo(imageBase64); - }); - }); + assertExtractedDocumentsUserMessage( + lastMessages.get(4), + ExtractedDocument.forToolCall( + "img111", + "MCP_A_MCP_Client___toolA", + documentReference, + content -> + assertThat(content) + .isInstanceOfSatisfying( + ImageContent.class, + img -> { + assertThat(img.image().mimeType()).isEqualTo("image/png"); + assertThat(img.image().base64Data()).isEqualTo(imageBase64); + }))); assertAgentResponse( zeebeTest, diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java index 6af97da0cf5..0b7318377db 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/jobworker/L4JAiAgentJobWorkerToolCallingTests.java @@ -17,18 +17,16 @@ package io.camunda.connector.e2e.agenticai.aiagent.langchain4j.jobworker; import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.FEEDBACK_LOOP_RESPONSE_TEXT; -import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.readDocumentReference; +import static io.camunda.connector.e2e.agenticai.aiagent.ToolCallResultDocumentAssertions.assertDocumentContentBlock; +import static io.camunda.connector.e2e.agenticai.aiagent.ToolCallResultDocumentAssertions.assertExtractedDocumentsUserMessage; +import static io.camunda.connector.e2e.agenticai.aiagent.ToolCallResultDocumentAssertions.parseDocumentReference; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import dev.langchain4j.agent.tool.ToolExecutionRequest; import dev.langchain4j.data.message.AiMessage; -import dev.langchain4j.data.message.Content; -import dev.langchain4j.data.message.ImageContent; -import dev.langchain4j.data.message.PdfFileContent; import dev.langchain4j.data.message.SystemMessage; -import dev.langchain4j.data.message.TextContent; import dev.langchain4j.data.message.ToolExecutionResultMessage; import dev.langchain4j.data.message.UserMessage; import dev.langchain4j.model.chat.response.ChatResponse; @@ -36,6 +34,7 @@ import dev.langchain4j.model.output.FinishReason; import dev.langchain4j.model.output.TokenUsage; import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; +import io.camunda.connector.e2e.agenticai.aiagent.ToolCallResultDocumentAssertions.ExtractedDocument; import io.camunda.connector.e2e.agenticai.assertj.JobWorkerAgentResponseAssert; import io.camunda.connector.test.utils.annotation.SlowTest; import java.util.List; @@ -152,7 +151,7 @@ void supportsDocumentResponsesFromToolCalls( // tool result: document serialized as document reference var toolResultText = ((ToolExecutionResultMessage) lastMessages.get(3)).text(); - var documentReference = readDocumentReference(toolResultText); + var documentReference = parseDocumentReference(toolResultText); assertThat(lastMessages.get(3)) .isInstanceOfSatisfying( @@ -161,31 +160,16 @@ void supportsDocumentResponsesFromToolCalls( assertThat(msg.id()).isEqualTo("aaa111"); assertThat(msg.toolName()).isEqualTo("Download_A_File"); }); - assertThat(documentReference.contentType()).isEqualTo(mimeType); + assertThat(documentReference.metadata().contentType()).isEqualTo(mimeType); // document user message: extracted document content - assertThat(lastMessages.get(4)) - .isInstanceOfSatisfying( - UserMessage.class, - msg -> { - List contents = msg.contents(); - assertThat(contents).hasSize(3); - assertThat(contents.get(0)) - .isInstanceOfSatisfying( - TextContent.class, - tc -> - assertThat(tc.text()) - .isEqualTo("Documents extracted from tool call results:")); - assertThat(contents.get(1)) - .isInstanceOfSatisfying( - TextContent.class, - tc -> - assertThat(tc.text()) - .isEqualTo( - "" - .formatted(documentReference.shortId()))); - assertDocumentContentBlock(contents.get(2), type, mimeType); - }); + assertExtractedDocumentsUserMessage( + lastMessages.get(4), + ExtractedDocument.forToolCall( + "aaa111", + "Download_A_File", + documentReference, + content -> assertDocumentContentBlock(content, type, mimeType))); assertThat(lastMessages.get(5)).isInstanceOf(AiMessage.class); // response after tool @@ -205,24 +189,4 @@ void supportsDocumentResponsesFromToolCalls( assertThat(userFeedbackJobWorkerCounter.get()).isEqualTo(2); } - - private void assertDocumentContentBlock( - Content content, String expectedType, String expectedMimeType) { - if (expectedType.equals("text")) { - assertThat(content).isInstanceOf(TextContent.class); - } else if (expectedMimeType.equals("application/pdf")) { - assertThat(content) - .isInstanceOfSatisfying( - PdfFileContent.class, pdf -> assertThat(pdf.pdfFile().base64Data()).isNotBlank()); - } else { - // image types - assertThat(content) - .isInstanceOfSatisfying( - ImageContent.class, - img -> { - assertThat(img.image().mimeType()).isEqualTo(expectedMimeType); - assertThat(img.image().base64Data()).isNotBlank(); - }); - } - } } diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorMcpIntegrationTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorMcpIntegrationTests.java index fe4fbd7906c..58d9fe30f65 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorMcpIntegrationTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorMcpIntegrationTests.java @@ -17,7 +17,8 @@ package io.camunda.connector.e2e.agenticai.aiagent.langchain4j.outboundconnector; import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.HAIKU_TEXT; -import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.readDocumentReference; +import static io.camunda.connector.e2e.agenticai.aiagent.ToolCallResultDocumentAssertions.assertExtractedDocumentsUserMessage; +import static io.camunda.connector.e2e.agenticai.aiagent.ToolCallResultDocumentAssertions.parseDocumentReference; import static io.camunda.connector.e2e.agenticai.aiagent.langchain4j.Langchain4JAiAgentToolSpecifications.EXPECTED_MCP_TOOL_SPECIFICATIONS; import static io.camunda.connector.e2e.agenticai.mcp.McpSdkToolSpecifications.MCP_TOOL_SPECIFICATIONS; import static org.assertj.core.api.Assertions.assertThat; @@ -34,10 +35,8 @@ import dev.langchain4j.agent.tool.ToolExecutionRequest; import dev.langchain4j.agent.tool.ToolSpecification; import dev.langchain4j.data.message.AiMessage; -import dev.langchain4j.data.message.Content; import dev.langchain4j.data.message.ImageContent; import dev.langchain4j.data.message.SystemMessage; -import dev.langchain4j.data.message.TextContent; import dev.langchain4j.data.message.ToolExecutionResultMessage; import dev.langchain4j.data.message.UserMessage; import dev.langchain4j.model.chat.request.ChatRequest; @@ -53,6 +52,7 @@ import io.camunda.connector.agenticai.mcp.client.model.McpRemoteClientTransportConfiguration; import io.camunda.connector.agenticai.mcp.client.model.McpRemoteClientTransportConfiguration.SseHttpMcpRemoteClientTransportConfiguration; import io.camunda.connector.agenticai.mcp.client.model.McpRemoteClientTransportConfiguration.StreamableHttpMcpRemoteClientTransportConfiguration; +import io.camunda.connector.e2e.agenticai.aiagent.ToolCallResultDocumentAssertions.ExtractedDocument; import io.camunda.connector.e2e.agenticai.aiagent.langchain4j.Langchain4JAiAgentToolSpecifications; import io.camunda.connector.e2e.agenticai.assertj.AgentResponseAssert; import io.camunda.connector.test.utils.annotation.SlowTest; @@ -405,7 +405,7 @@ void extractsDocumentsFromMcpImageToolCallResult() throws IOException { // tool result: document serialized as document reference var toolResultText = ((ToolExecutionResultMessage) lastMessages.get(3)).text(); - var documentReference = readDocumentReference(toolResultText); + var documentReference = parseDocumentReference(toolResultText); assertThat(lastMessages.get(3)) .isInstanceOfSatisfying( @@ -414,37 +414,23 @@ void extractsDocumentsFromMcpImageToolCallResult() throws IOException { assertThat(msg.id()).isEqualTo("img111"); assertThat(msg.toolName()).isEqualTo("MCP_A_MCP_Client___toolA"); }); - assertThat(documentReference.contentType()).isEqualTo("image/png"); + assertThat(documentReference.metadata().contentType()).isEqualTo("image/png"); // document user message: extracted document content - assertThat(lastMessages.get(4)) - .isInstanceOfSatisfying( - UserMessage.class, - msg -> { - List contents = msg.contents(); - assertThat(contents).hasSize(3); - assertThat(contents.get(0)) - .isInstanceOfSatisfying( - TextContent.class, - tc -> - assertThat(tc.text()) - .isEqualTo("Documents extracted from tool call results:")); - assertThat(contents.get(1)) - .isInstanceOfSatisfying( - TextContent.class, - tc -> - assertThat(tc.text()) - .isEqualTo( - "" - .formatted(documentReference.shortId()))); - assertThat(contents.get(2)) - .isInstanceOfSatisfying( - ImageContent.class, - img -> { - assertThat(img.image().mimeType()).isEqualTo("image/png"); - assertThat(img.image().base64Data()).isEqualTo(imageBase64); - }); - }); + assertExtractedDocumentsUserMessage( + lastMessages.get(4), + ExtractedDocument.forToolCall( + "img111", + "MCP_A_MCP_Client___toolA", + documentReference, + content -> + assertThat(content) + .isInstanceOfSatisfying( + ImageContent.class, + img -> { + assertThat(img.image().mimeType()).isEqualTo("image/png"); + assertThat(img.image().base64Data()).isEqualTo(imageBase64); + }))); assertAgentResponse( zeebeTest, diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java index b70c308a134..7925eb15ed4 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/langchain4j/outboundconnector/L4JAiAgentConnectorToolCallingTests.java @@ -17,18 +17,16 @@ package io.camunda.connector.e2e.agenticai.aiagent.langchain4j.outboundconnector; import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.FEEDBACK_LOOP_RESPONSE_TEXT; -import static io.camunda.connector.e2e.agenticai.aiagent.AiAgentTestFixtures.readDocumentReference; +import static io.camunda.connector.e2e.agenticai.aiagent.ToolCallResultDocumentAssertions.assertDocumentContentBlock; +import static io.camunda.connector.e2e.agenticai.aiagent.ToolCallResultDocumentAssertions.assertExtractedDocumentsUserMessage; +import static io.camunda.connector.e2e.agenticai.aiagent.ToolCallResultDocumentAssertions.parseDocumentReference; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import dev.langchain4j.agent.tool.ToolExecutionRequest; import dev.langchain4j.data.message.AiMessage; -import dev.langchain4j.data.message.Content; -import dev.langchain4j.data.message.ImageContent; -import dev.langchain4j.data.message.PdfFileContent; import dev.langchain4j.data.message.SystemMessage; -import dev.langchain4j.data.message.TextContent; import dev.langchain4j.data.message.ToolExecutionResultMessage; import dev.langchain4j.data.message.UserMessage; import dev.langchain4j.model.chat.response.ChatResponse; @@ -36,6 +34,7 @@ import dev.langchain4j.model.output.FinishReason; import dev.langchain4j.model.output.TokenUsage; import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; +import io.camunda.connector.e2e.agenticai.aiagent.ToolCallResultDocumentAssertions.ExtractedDocument; import io.camunda.connector.e2e.agenticai.assertj.AgentResponseAssert; import io.camunda.connector.test.utils.annotation.SlowTest; import java.util.List; @@ -145,7 +144,7 @@ void supportsDocumentResponsesFromToolCalls( // tool result: document serialized as document reference var toolResultText = ((ToolExecutionResultMessage) lastMessages.get(3)).text(); - var documentReference = readDocumentReference(toolResultText); + var documentReference = parseDocumentReference(toolResultText); assertThat(lastMessages.get(3)) .isInstanceOfSatisfying( @@ -154,31 +153,16 @@ void supportsDocumentResponsesFromToolCalls( assertThat(msg.id()).isEqualTo("aaa111"); assertThat(msg.toolName()).isEqualTo("Download_A_File"); }); - assertThat(documentReference.contentType()).isEqualTo(mimeType); + assertThat(documentReference.metadata().contentType()).isEqualTo(mimeType); // document user message: extracted document content - assertThat(lastMessages.get(4)) - .isInstanceOfSatisfying( - UserMessage.class, - msg -> { - List contents = msg.contents(); - assertThat(contents).hasSize(3); - assertThat(contents.get(0)) - .isInstanceOfSatisfying( - TextContent.class, - tc -> - assertThat(tc.text()) - .isEqualTo("Documents extracted from tool call results:")); - assertThat(contents.get(1)) - .isInstanceOfSatisfying( - TextContent.class, - tc -> - assertThat(tc.text()) - .isEqualTo( - "" - .formatted(documentReference.shortId()))); - assertDocumentContentBlock(contents.get(2), type, mimeType); - }); + assertExtractedDocumentsUserMessage( + lastMessages.get(4), + ExtractedDocument.forToolCall( + "aaa111", + "Download_A_File", + documentReference, + content -> assertDocumentContentBlock(content, type, mimeType))); assertThat(lastMessages.get(5)).isInstanceOf(AiMessage.class); // response after tool @@ -199,24 +183,4 @@ void supportsDocumentResponsesFromToolCalls( assertThat(userFeedbackJobWorkerCounter.get()).isEqualTo(2); } - - private void assertDocumentContentBlock( - Content content, String expectedType, String expectedMimeType) { - if (expectedType.equals("text")) { - assertThat(content).isInstanceOf(TextContent.class); - } else if (expectedMimeType.equals("application/pdf")) { - assertThat(content) - .isInstanceOfSatisfying( - PdfFileContent.class, pdf -> assertThat(pdf.pdfFile().base64Data()).isNotBlank()); - } else { - // image types - assertThat(content) - .isInstanceOfSatisfying( - ImageContent.class, - img -> { - assertThat(img.image().mimeType()).isEqualTo(expectedMimeType); - assertThat(img.image().base64Data()).isNotBlank(); - }); - } - } } From 6b664f8f3b14411e8d284c8f1fc6c083087cf380 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 13:24:26 +0200 Subject: [PATCH 36/81] refactor(agentic-ai): drop "unknown" fallback in tool call document extractor Tool call result name is set on every production path (MCP/A2A handlers, forCancelledToolCall, BPMN _meta.name) and events are filtered out before the extractor runs. Propagating null produces a tag without the tool-name attribute via DocumentXmlTag.appendAttribute, which is the right shape for malformed inputs anyway. --- .../aiagent/agent/ToolCallResultDocumentExtractor.java | 7 +------ .../aiagent/agent/ToolCallResultDocumentExtractorTest.java | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java index d8f45efb36b..80a698b081a 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java @@ -11,7 +11,6 @@ import io.camunda.connector.api.document.Document; import java.util.ArrayList; import java.util.List; -import org.apache.commons.lang3.StringUtils; /** * Extracts {@link Document} instances from a list of tool call results, grouped by tool call. @@ -45,11 +44,7 @@ public List extractDocuments(List toolCallRes for (ToolCallResult toolCallResult : toolCallResults) { final var documents = extractFromToolCallResult(toolCallResult); if (!documents.isEmpty()) { - result.add( - new ToolCallDocuments( - StringUtils.defaultString(toolCallResult.id()), - StringUtils.defaultIfBlank(toolCallResult.name(), "unknown"), - documents)); + result.add(new ToolCallDocuments(toolCallResult.id(), toolCallResult.name(), documents)); } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractorTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractorTest.java index 340fdc68601..0a8f632eb91 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractorTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractorTest.java @@ -156,8 +156,8 @@ void handlesNullNameAndId() { assertThat(extracted.getFirst()) .satisfies( e -> { - assertThat(e.toolCallId()).isEmpty(); - assertThat(e.toolCallName()).isEqualTo("unknown"); + assertThat(e.toolCallId()).isNull(); + assertThat(e.toolCallName()).isNull(); }); } From 574d3228bfd3fb17334a3110fda0a4aeb558d1fb Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 13:24:44 +0200 Subject: [PATCH 37/81] refactor(agentic-ai): swap DocumentXmlTag parameter order to (toolCallId, toolName) Aligns DocumentXmlTag's record fields and factory with the rest of the new code (ToolCallDocuments, ExtractedDocument), which all put toolCallId before toolName/toolCallName. The XML attribute order in toXml() is unchanged. --- .../agenticai/aiagent/ToolCallResultDocumentAssertions.java | 2 +- .../agenticai/aiagent/agent/AgentMessagesHandlerImpl.java | 2 +- .../connector/agenticai/model/message/DocumentXmlTag.java | 6 +++--- .../agenticai/model/message/DocumentXmlTagTest.java | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/ToolCallResultDocumentAssertions.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/ToolCallResultDocumentAssertions.java index bab4f82bde8..49f7cee3c9e 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/ToolCallResultDocumentAssertions.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/ToolCallResultDocumentAssertions.java @@ -200,8 +200,8 @@ public static ExtractedDocument forToolCall( /** The XML tag string this document is expected to render to in the synthetic user message. */ String expectedXmlTag() { return new DocumentXmlTag( - toolName, toolCallId, + toolName, documentShortId(), reference.metadata() != null ? reference.metadata().fileName() : null) .toXml(); diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java index 78fdf288f23..41856c51bd9 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java @@ -227,7 +227,7 @@ private UserMessage createDocumentMessageForToolResults(List res for (var doc : entry.documents()) { content.add( textContent( - DocumentXmlTag.from(doc, entry.toolCallName(), entry.toolCallId()).toXml())); + DocumentXmlTag.from(doc, entry.toolCallId(), entry.toolCallName()).toXml())); content.add(DocumentContent.documentContent(doc)); } } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/DocumentXmlTag.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/DocumentXmlTag.java index 66d94445d66..4579ccffe66 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/DocumentXmlTag.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/DocumentXmlTag.java @@ -22,8 +22,8 @@ * the document reference in the tool result JSON. */ public record DocumentXmlTag( - @Nullable String toolName, @Nullable String toolCallId, + @Nullable String toolName, @Nullable String documentShortId, @Nullable String filename) { @@ -32,9 +32,9 @@ public record DocumentXmlTag( * the first segment of the document's UUID identifier (e.g. "25ece9fa" from * "25ece9fa-aeea-423d-98ed-67c1f08b137b"). */ - public static DocumentXmlTag from(Document document, String toolName, String toolCallId) { + public static DocumentXmlTag from(Document document, String toolCallId, String toolName) { return new DocumentXmlTag( - toolName, toolCallId, extractDocumentShortId(document), extractFileName(document)); + toolCallId, toolName, extractDocumentShortId(document), extractFileName(document)); } /** Creates a tag from a document without tool call context (e.g. for event documents). */ diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/model/message/DocumentXmlTagTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/model/message/DocumentXmlTagTest.java index 33913e05eb9..126e49d3c24 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/model/message/DocumentXmlTagTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/model/message/DocumentXmlTagTest.java @@ -31,7 +31,7 @@ void generatesFullTagWithAllAttributes() { when(doc.metadata()).thenReturn(metadata); when(metadata.getFileName()).thenReturn("report.pdf"); - assertThat(DocumentXmlTag.from(doc, "search", "call_abc").toXml()) + assertThat(DocumentXmlTag.from(doc, "call_abc", "search").toXml()) .isEqualTo( ""); } @@ -85,7 +85,7 @@ void escapesSpecialCharactersInToolName() { when(doc.reference()).thenReturn(ref); when(ref.getDocumentId()).thenReturn("abc12345-0000-0000-0000-000000000000"); - assertThat(DocumentXmlTag.from(doc, "tool", "call_1").toXml()) + assertThat(DocumentXmlTag.from(doc, "call_1", "tool").toXml()) .isEqualTo( ""); } From 5707929cf3f1a6a4ab1f35a0a55e869bd195ecfd Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 13:31:04 +0200 Subject: [PATCH 38/81] feat(agentic-ai): add debug logging for tool call document extraction Adds a single summary log per extractor invocation (results processed, results with documents, total document count), plus targeted DEBUG logs in the MCP and A2A gateway handlers when their extractDocuments path silently returns an empty list because the content has an unexpected shape. --- .../client/agentic/tool/A2aGatewayToolHandler.java | 7 +++++++ .../agent/ToolCallResultDocumentExtractor.java | 13 +++++++++++++ .../mcp/discovery/McpClientGatewayToolHandler.java | 7 +++++++ 3 files changed, 27 insertions(+) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandler.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandler.java index 48ac40c384f..2dfb11c10ac 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandler.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/a2a/client/agentic/tool/A2aGatewayToolHandler.java @@ -242,6 +242,13 @@ private ToolCallResult toolCallResultFromA2aSendMessage(ToolCallResult toolCallR @Override public List extractDocuments(ToolCallResult toolCallResult) { if (!(toolCallResult.content() instanceof A2aSendMessageResult result)) { + LOGGER.debug( + "A2A tool call result content is not an A2aSendMessageResult ({}), skipping document extraction. toolCallId={}, toolName={}", + toolCallResult.content() != null + ? toolCallResult.content().getClass().getSimpleName() + : null, + toolCallResult.id(), + toolCallResult.name()); return List.of(); } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java index 80a698b081a..88fe4647270 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/ToolCallResultDocumentExtractor.java @@ -11,6 +11,8 @@ import io.camunda.connector.api.document.Document; import java.util.ArrayList; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Extracts {@link Document} instances from a list of tool call results, grouped by tool call. @@ -24,6 +26,9 @@ */ public class ToolCallResultDocumentExtractor { + private static final Logger LOGGER = + LoggerFactory.getLogger(ToolCallResultDocumentExtractor.class); + /** Documents extracted from a single tool call result, grouped with the tool call identity. */ public record ToolCallDocuments( String toolCallId, String toolCallName, List documents) {} @@ -40,14 +45,22 @@ public ToolCallResultDocumentExtractor(GatewayToolHandlerRegistry gatewayToolHan */ public List extractDocuments(List toolCallResults) { final var result = new ArrayList(); + int totalDocuments = 0; for (ToolCallResult toolCallResult : toolCallResults) { final var documents = extractFromToolCallResult(toolCallResult); if (!documents.isEmpty()) { result.add(new ToolCallDocuments(toolCallResult.id(), toolCallResult.name(), documents)); + totalDocuments += documents.size(); } } + LOGGER.debug( + "Tool call document extraction: processed {} result(s), extracted documents from {} ({} document(s) total)", + toolCallResults.size(), + result.size(), + totalDocuments); + return result; } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandler.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandler.java index 4275a4f8d10..3ef02758912 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandler.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/mcp/discovery/McpClientGatewayToolHandler.java @@ -275,6 +275,13 @@ private ToolCallResult toolCallResultFromMcpToolCall(ToolCallResult toolCallResu public List extractDocuments(ToolCallResult toolCallResult) { if (!(toolCallResult.content() instanceof List contents)) { // string-content optimization or unmanaged shape — nothing to walk + LOGGER.debug( + "MCP tool call result content is not a List ({}), skipping document extraction. toolCallId={}, toolName={}", + toolCallResult.content() != null + ? toolCallResult.content().getClass().getSimpleName() + : null, + toolCallResult.id(), + toolCallResult.name()); return List.of(); } From b1083d4e2f6e6563400b48cff78462982835ada6 Mon Sep 17 00:00:00 2001 From: Dmitri Nikonov Date: Wed, 6 May 2026 13:32:54 +0200 Subject: [PATCH 39/81] docs(agentic-ai): Add ADR to replace langchain4j --- .../adr/004-replace-langchain4j-framework.md | 599 ++++++++++++++++++ 1 file changed, 599 insertions(+) create mode 100644 connectors/agentic-ai/docs/adr/004-replace-langchain4j-framework.md diff --git a/connectors/agentic-ai/docs/adr/004-replace-langchain4j-framework.md b/connectors/agentic-ai/docs/adr/004-replace-langchain4j-framework.md new file mode 100644 index 00000000000..7ca8c9a3896 --- /dev/null +++ b/connectors/agentic-ai/docs/adr/004-replace-langchain4j-framework.md @@ -0,0 +1,599 @@ +# Replace LangChain4j with Native Provider Layer + +* Deciders: Agentic AI Team +* Date: May 5, 2026 + +## Status + +**Proposed** + +## Context and Problem Statement + +The agentic-ai module abstracts LLM access behind a single global `AiFrameworkAdapter` whose only +production implementation wraps LangChain4j (`Langchain4JAiFrameworkAdapter`). Six per-vendor +`ChatModelProvider` beans construct LangChain4j `ChatModel` instances; converters translate our +domain `Message`/`Content` types to and from LangChain4j's `ChatMessage` types. + +This abstraction worked while LangChain4j tracked provider features closely. It increasingly does +not. Concrete blockers we have hit or will imminently hit: + +* `AssistantMessage` / `SystemMessage` content lists are restricted to a single `TextContent` + block at conversion time + ([`ChatMessageConverterImpl.java:104-110`](../../src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterImpl.java)). + Reasoning blocks and multi-block assistant content cannot be represented even structurally. +* `ToolCallResultMessage` carries content as a single `Object` flattened to a JSON string. Native + multimodal tool results (images and PDFs inside `tool_result` blocks, supported by Anthropic and + AWS Bedrock at the wire level) cannot be expressed; the workaround in PR #6999 synthesizes a + user message with `DocumentContent` blocks because the framework cannot carry documents in tool + results. +* `AgentMetrics.TokenUsage` records only input + output token counts. Cache attribution + (`cache_creation` / `cache_read`) and reasoning-token spend are not surfaced. +* Reasoning models are not supported. Anthropic extended thinking, OpenAI Responses-API reasoning, + and Gemini thinking budgets all require carrying signed reasoning blocks across turns; the + framework discards them. +* Prompt caching is not configurable. Every provider exposes some form of cache control + (Anthropic per-block ephemeral markers, Bedrock cache points, OpenAI prompt cache key, Gemini + context caching); the framework exposes none. +* For Claude on AWS Bedrock and Google Vertex AI, the framework routes through Bedrock Converse / + Vertex generic adapters, which are lowest-common-denominator and lose Claude-specific features + even though Anthropic publishes platform-backend SDKs that preserve full feature parity. +* Per-block cache control, server-side tool blocks (web search, code execution), and beta-API + flags all require escape hatches the framework does not expose. + +Should we extend LangChain4j upstream and continue building on it, or replace the framework layer +with our own provider abstraction? + +## Decision Drivers + +* **Feature surface**: We need access to provider features (multimodal tool results, reasoning + with signature roundtrip, granular cache control, cache and reasoning token attribution) that + the framework's lowest-common-denominator types structurally cannot represent. +* **Release cadence**: Closing each gap upstream requires waiting for the framework's release + cycle; some of the gaps above are structural (data-model restrictions) and unlikely to be + closed without breaking changes. +* **Maintenance burden**: A provider abstraction that we own lets us pick official vendor SDKs, + benefit from their auth / retry / signing handling, and surface their richer types directly. +* **Predictable rollout**: A complete replacement avoids a long-lived hybrid code path with two + sets of converters and conditional behavior. + +## Considered Options + +1. Continue with LangChain4j; contribute upstream PRs for missing features. +2. Build a thin custom HTTP/JSON layer per provider, owning the wire format end-to-end. +3. Replace the framework abstraction with a custom provider layer built on the official vendor + Java SDKs (Anthropic, AWS Bedrock, OpenAI, Google GenAI). + +## Decision Outcome + +Chosen option: **Option 3 — Native provider layer over official vendor SDKs**. + +The vendor SDKs already handle authentication (including SigV4, OAuth refresh, Bearer + Azure +credentials, GCP ADC), retries, timeouts, signing, SSE / event-stream parsing, and tool-schema +derivation. Reimplementing these in-house (Option 2) is unnecessary duplication. Continuing on +LangChain4j (Option 1) does not address the structural data-model restrictions. + +The replacement is delivered as one shipping unit (after a small additive Phase 0). LangChain4j +support is preserved as an opt-in bridge implementation for users running custom bundles with +provider models we have not yet covered natively, but is no longer registered by default. + +### Positive Consequences + +* Direct access to provider-native content blocks (multimodal tool results, signed reasoning, + cache control breakpoints, server-side tool blocks). +* Full token-usage attribution including cache and reasoning tokens. +* Cleaner cloud Claude story: a single Anthropic implementation serves direct, AWS Bedrock, + Google Vertex, and Azure Foundry deployments without lossy adapter routing. +* Opt-in SDK dependencies via `@ConditionalOnClass` — deployments only pull in the providers + they use. +* Discriminated streaming events enable accurate tool-argument accumulation with mid-stream + repair, partial-content preservation on errors, and clean abort propagation. + +### Negative Consequences + +* Larger per-provider implementation than today's LangChain4j wrappers. +* One-time migration effort across the framework, converters, and provider configuration shapes. +* Element template version bump for provider configuration restructuring (Task and Sub-process + flavors). +* Capability matrix YAML to maintain alongside the supported model set. + +## Architecture + +The framework abstraction is replaced with a layered design: + +``` +┌──────────────────────────────────────────────────────────────┐ +│ ChatClient (facade — what BaseAgentRequestHandler calls) │ +│ resolves model + capabilities, applies tool-result │ +│ strategy, dispatches to ChatModelApi. │ +└──────────────────────────────────────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────────────────┐ +│ ChatModelApi (per-job instance, configuration baked in) │ +│ capabilities() : ModelCapabilities │ +│ complete(request, options, listener) : CompletableFuture │ +│ │ +│ Constructed by ChatModelApiFactory — one bean per │ +│ wire-protocol family. │ +└──────────────────────────────────────────────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────────────────────────┐ +│ Vendor SDKs │ +│ anthropic-sdk-java | aws-sdk-java-v2 bedrockruntime | │ +│ openai-java | google-genai-java │ +└──────────────────────────────────────────────────────────────┘ +``` + +Five native API families ship in scope: + +| Family | Wire protocol | Covers | +|--------------------|--------------------------|-------------------------------------------------------------------------------| +| `anthropic-messages` | Anthropic Messages API | Claude direct, Claude on Bedrock, Claude on Vertex AI, Claude on Azure Foundry| +| `bedrock-converse` | AWS Bedrock Converse | Nova, Mistral, Llama, Cohere on Bedrock | +| `openai-responses` | OpenAI Responses API | GPT-5, o-series, Azure deployments thereof | +| `openai-completions` | OpenAI Chat Completions | Legacy OpenAI Chat models, OpenAI-compatible gateways (Ollama, vLLM, etc.) | +| `google-genai` | Google GenAI | Gemini Developer API + Vertex AI (single client, backend toggle) | + +For Claude on Bedrock / Vertex / Foundry, the Anthropic SDK exposes platform-backend modules +(`anthropic-java-bedrock`, `anthropic-java-vertex`, `anthropic-java-foundry`) that preserve the +Anthropic wire format end-to-end. Routing Claude through these backends instead of through +generic platform adapters is a key reason for the restructure. + +### Per-job instance pattern + +`ChatModelApiFactory` is a singleton bean (one per wire +protocol). Per call, the registry resolves the factory by `ProviderConfiguration` discriminator +and produces a per-job `ChatModelApi` instance carrying the resolved configuration and (lazily +constructed) SDK client. `ChatClient` performs this resolution once at the start of each +request and reuses the instance across capability lookups, strategy application, and the call +itself. + +This mirrors the current `ChatModelProvider` pattern: factory beans are stateless, runtime +clients are per-call. + +### Streaming-first internally, blocking surface externally + +Each `ChatModelApi` implementation drives the SDK's streaming API internally. Streaming gives +us accurate tool-argument JSON accumulation across partial chunks, partial-content preservation +when a call errors mid-stream, mid-call abort on job cancel, and incremental token-usage +updates. None of these benefits require exposing a streaming primitive to callers. + +The public surface is `CompletableFuture` plus an optional +`ChatStreamListener` for in-process observability (logging, metrics, future event emission). +Listener defaults to NOOP. No reactive types in the public API. + +### Stream event hierarchy + +```java +public sealed interface ChatModelEvent permits + StartEvent, + TextStartEvent, TextDeltaEvent, TextEndEvent, + ReasoningStartEvent, ReasoningDeltaEvent, ReasoningEndEvent, + ToolCallStartEvent, ToolCallArgumentsDeltaEvent, ToolCallEndEvent, + UsageEvent, + DoneEvent, ErrorEvent { } +``` + +Each delta event carries a content-block index so the listener can group fragments by block. +`DoneEvent` carries the final assembled `ChatResponse`. `ErrorEvent` carries the error message, +partial content accumulated so far, and the partial usage record. + +### Error semantics + +Three classes of failure with distinct surface behavior: + +| Class | Examples | Surface | +|----------------------------|--------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------| +| Model-side terminal | `stop_reason: refusal`, content filter, max-tokens hit, malformed tool-use | `CompletableFuture` completes normally with `ChatResponse{stopReason=ERROR, errorMessage, content, usage}` | +| Transport / SDK / I/O | Connection refused, read timeout, TLS failure, malformed wire response | `CompletableFuture` completes exceptionally with `ConnectorException(ERROR_CODE_FAILED_MODEL_CALL, ...)` | +| Auth / config | Bad API key, region not enabled, model not found | `CompletableFuture` completes exceptionally with a distinct error code | + +Token usage and partial content accumulated before the failure are preserved on the response +record where applicable, so call accounting remains accurate even on terminal errors. + +## Domain Model Extensions + +All changes are additive and nullable. Existing serialized `agentContext` records and stored +conversations deserialize unchanged with new fields defaulting to `null` / empty. + +### `Content` sealed hierarchy + +One new variant: + +```java +public sealed interface Content permits + TextContent, DocumentContent, ObjectContent, + ReasoningContent { ... } // NEW + +public record ReasoningContent( + String text, + @Nullable String signature, // opaque round-trip blob + boolean redacted, + Map metadata +) implements Content { } +``` + +`signature` carries the provider-specific encrypted reasoning blob (Anthropic encrypted +thinking, Gemini `thoughtSignature`, OpenAI Responses encrypted reasoning item). It is +required for multi-turn reasoning continuation; the model rejects requests where prior +reasoning is not replayed verbatim. + +`DocumentContent` (Camunda `Document` reference) remains the single representation for any +multimedia in the system. Inline binary content does not appear in the domain model — bytes +exist only transiently inside a `ChatModelApi` implementation while serializing requests or +materializing model output to a Camunda Document. + +### `AssistantMessage` provenance and metrics + +Four new optional fields: + +```java +public record AssistantMessage( + List content, + List toolCalls, + @Nullable String modelId, // NEW + @Nullable String apiId, // NEW + @Nullable StopReason stopReason, // NEW + @Nullable TokenUsage usage, // NEW (per-message) + Map metadata // existing — escape hatch +) implements Message, ContentMessage { } +``` + +`StopReason` is a normalized enum: `STOP | LENGTH | TOOL_USE | ERROR | ABORTED | +CONTENT_FILTERED | GUARDRAIL`. Each implementation maps provider-specific finish reasons into +this set. + +### `ToolCallResult` multimodal support + +```java +public record ToolCallResult( + @Nullable String id, + @Nullable String name, + @Nullable Object content, // existing — string/JSON path + @Nullable List contentBlocks, // NEW — when present, used preferentially + Map properties +) { } +``` + +When `contentBlocks` is present, implementations whose target API supports multimodal tool +results pass it through as native content (`tool_result` content list on Anthropic, +`ToolResultContentBlock` on Bedrock). Implementations without native support delegate to +`ToolCallResultStrategy` — see below. + +### `TokenUsage` extension + +```java +public record TokenUsage( + int inputTokens, + int outputTokens, + int cacheReadInputTokens, // NEW + int cacheCreationInputTokens, // NEW + int reasoningTokens, // NEW + int totalTokens // computed when not reported +) { } +``` + +`AgentMetrics.tokenUsage` becomes the cumulative roll-up across calls; +`AssistantMessage.usage` is per-call. + +### Removed surface + +`AiFrameworkChatResponse#rawChatResponse()` is removed. The current interface exposes it +but no caller consumes it; the `AssistantMessage.metadata` map serves as the escape hatch +for provider-specific data when needed. + +## Capability Matrix + +A YAML resource on the classpath records capabilities per supported model. The schema is +oriented around what the runtime needs to decide: + +```yaml +anthropic-messages: + models: + - id: claude-opus-4-7 + aliases: [claude-opus-latest] + capabilities: + input_modalities: + user_message: [text, image, pdf] + tool_result: [text, image] + output_modalities: + assistant_message: [text] + supports_reasoning: true + supports_reasoning_signature_roundtrip: true + supports_prompt_caching: true + supports_parallel_tool_calls: true + context_window: 200000 + max_output_tokens: 64000 + - pattern: claude-opus-* + capabilities: { ... } # best-effort family default + - pattern: claude-haiku-* + capabilities: + supports_reasoning: false # haiku family does not yet have extended thinking + ... +``` + +Modality vocabulary: `text | image | pdf | audio | video`. Modality lists per location +(`user_message`, `tool_result`, `assistant_message`) are symmetric — every modality at every +location has an explicit answer for each model. + +### Resolution order + +Most-specific-first, scoped to the api family of the connector configuration: + +1. **Connector config override** — user-declared `modelCapabilities` block on the provider + configuration always wins. +2. **Exact id or alias match** — the `id` field of an entry, or any string in its `aliases` + list, equals the requested model id. +3. **Pattern match** — `pattern` (glob with `*` only) matches the requested model id; + longest-matching pattern wins. +4. **Conservative defaults** — text-only across the board, all `supports_*` flags `false`, + numeric limits null. + +Aliases resolve at step 2 directly (no pre-rewriting); patterns at step 3 match against +the original requested id. Resolution at steps 3 or 4 logs an INFO message on first use so +operators notice they are running on best-effort or default capabilities. Resolution at +step 2 is silent — alias mappings are verified declarations. + +### Conservative defaults for unknown models + +```yaml +input_modalities: + user_message: [text] + tool_result: [text] +output_modalities: + assistant_message: [text] +supports_reasoning: false +supports_reasoning_signature_roundtrip: false +supports_prompt_caching: false +supports_parallel_tool_calls: false +context_window: null +max_output_tokens: null +``` + +Unknown api families fail at validation, not at runtime — they have no factory bean to +resolve, so requests can never start. + +## Tool Call Result Routing + +`ToolCallResultStrategy` decides per tool result whether to pass `contentBlocks` through to +the provider as native multimodal content or to fall back to the existing user-message +extraction approach (PR #6999). The decision is per content block, driven by the capability +matrix: + +``` +for each Content block in tool result: + modality = modalityOf(block) // text | image | pdf | audio | video + if modality in capabilities.toolResultModalities(): + keep block inline as part of ToolCallResult.contentBlocks + else: + delegate to user-message fallback (synthetic UserMessage with DocumentContent) +``` + +Models with `supports_*_in_tool_result` modalities for a given media type get the native +path. Models without — including all models served via the LangChain4j bridge — get the +fallback. This makes PR #6999 the safe default and the multimodal-native path the +opt-in-by-capability optimization. + +## Reasoning Support + +Two-tier API: + +```java +public sealed interface ReasoningConfig + permits ReasoningEffort, ReasoningBudget, ReasoningDisabled { } + +public record ReasoningEffort(Effort level) implements ReasoningConfig { } + // MINIMAL | LOW | MEDIUM | HIGH | X_HIGH +public record ReasoningBudget(int tokens) implements ReasoningConfig { } +public record ReasoningDisabled() implements ReasoningConfig { } +``` + +`ChatOptions.reasoning` carries a high-level config; per-implementation translation maps it +to provider-native fields (Anthropic adaptive `effort` for newer Opus / Sonnet, budget-based +`thinking_budget_tokens` for older Claude 4; OpenAI Responses `reasoning.effort`; Gemini +`ThinkingConfig.thinkingBudget`; Bedrock per-model). Provider-specific overrides remain +available via `ChatOptions.providerOptions`. + +`ReasoningContent.signature` round-trip is mandatory for multi-turn reasoning. Each +implementation: + +* Accumulates signature deltas during streaming and stores them on the `ReasoningContent` + block in the assistant message. +* Replays signatures on subsequent requests when the prior assistant turn is included in the + conversation history. + +`TokenUsage.reasoningTokens` populates from provider-native fields where reported (OpenAI +`outputTokensDetails.reasoningTokens`, Gemini `thoughtsTokenCount`); zero where not separately +reported. + +## Prompt Caching + +```java +public enum CacheRetention { NONE, SHORT, LONG } +``` + +`SHORT` corresponds to the provider's default ephemeral retention (Anthropic 5 minutes, +OpenAI default prompt cache, Bedrock default). `LONG` corresponds to extended retention +(Anthropic 1 hour, OpenAI 24 hours, Bedrock 1 hour). `NONE` strips all cache markers. + +Cache breakpoint placement is implementation-specific. Each implementation places markers +at the boundaries the provider supports: + +* `anthropic-messages`: `cache_control` ephemeral markers on system prompt, last tool + definition, and last user / tool-result content block (up to 4 breakpoints). +* `bedrock-converse`: `cachePoint` blocks at the same boundaries, on supported models. +* `openai-responses` / `openai-completions`: caching is automatic; `prompt_cache_key` is + set to the conversation identifier. +* `google-genai`: implicit caching is always on; explicit context cache lifecycle is a + future extension. + +`TokenUsage.cacheReadInputTokens` and `cacheCreationInputTokens` populate from provider-native +fields. + +## Provider Configuration Restructure + +Configurations are restructured by wire format. The `ProviderConfiguration` sealed type +retains six members (matching today's count) but each gains a discriminator for backend or +api-family choice. Element template UI groups conditional fields under each discriminator +value. + +``` +ProviderConfiguration +├── AnthropicProviderConfiguration api: anthropic-messages +│ backend: { direct | bedrock | vertex | foundry } ← NEW +│ auth fields conditional on backend +│ model: AnthropicModel +├── BedrockProviderConfiguration api: bedrock-converse +│ (non-Anthropic Bedrock only: Nova, Mistral, Llama, Cohere) +│ auth (existing AWS chain) +│ model: BedrockModel +├── OpenAiProviderConfiguration api: openai-{responses|completions} +│ apiFamily: { responses | completions } ← NEW (default: responses) +│ auth (API key) +│ model: OpenAiModel +├── AzureOpenAiProviderConfiguration api: openai-{responses|completions} +│ apiFamily: { responses | completions } ← NEW +│ auth (existing — endpoint, key) +│ model: AzureModel +├── OpenAiCompatibleProviderConfiguration api: openai-{responses|completions} +│ apiFamily: { responses | completions } ← NEW (default: completions) +│ auth (existing — endpoint, optional key) +│ model: OpenAiCompatibleModel +└── GoogleGenAiProviderConfiguration api: google-genai + backend: { developer-api | vertex } ← NEW + auth fields conditional on backend + model: GeminiModel +``` + +`GoogleVertexAiProviderConfiguration` is renamed to `GoogleGenAiProviderConfiguration` since +the same configuration now covers both backends. + +`BedrockProviderConfiguration` becomes non-Anthropic-only at validation time. Saved +configurations with `anthropic.*` model identifiers are rewritten by the Jackson migration +deserializer to the Anthropic configuration with backend = bedrock — see below. + +### Backward compatibility — Jackson migration + +A custom `StdDeserializer` rewrites legacy configuration shapes to +the new structure at deserialization time. Existing process instances continue to work +without manual intervention. + +| Saved shape | Rewritten to | Detection | +|----------------------------------------------------------------------------|-------------------------------------------------------------------------------------------|-----------------------| +| `{type: bedrock, bedrock.model.model: "anthropic.claude-..."}` | `{type: anthropic, anthropic.backend: bedrock, anthropic.model.model: "anthropic..."}` | model-id prefix | +| `{type: bedrock, bedrock.model.model: "amazon.nova-..." \| "meta..." }` | passthrough | model-id prefix | +| `{type: googleVertexAi, ...}` | `{type: googleGenAi, googleGenAi.backend: vertex, ...}` | discriminator rename | +| `{type: anthropic, ...}` (no backend) | `{type: anthropic, anthropic.backend: direct, ...}` | missing field default | +| `{type: openai, ...}` (no apiFamily) | `{type: openai, openai.apiFamily: completions, ...}` | missing field default | +| `{type: azureOpenAi, ...}` (no apiFamily) | same with `apiFamily: completions` | missing field default | +| `{type: openAiCompatible, ...}` (no apiFamily) | same with `apiFamily: completions` | missing field default | + +Defaults during migration preserve current behavior (Chat Completions for OpenAI variants, +direct for Anthropic). Newly created configurations pick up the new defaults (Responses API +for OpenAI direct). + +The migration deserializer is permanent infrastructure — kept indefinitely so that stale +process variables remain readable. + +### Element template version bump + +Both element templates (`agenticai-aiagent-outbound-connector.json` and +`agenticai-aiagent-job-worker.json`) version-bump together per the AGENTS.md rule. Old +templates move to `element-templates/versioned/` per the existing pattern. The +`element-templates/README.md` index is updated accordingly. + +## Migration Plan + +Two phases: + +### Phase 0 — Domain model extensions (additive, behavior-preserving) + +* Add `ReasoningContent` to the `Content` sealed hierarchy. +* Add optional `modelId`, `apiId`, `stopReason`, `usage` fields to `AssistantMessage`. +* Add optional `contentBlocks` field to `ToolCallResult`. +* Add `cacheReadInputTokens`, `cacheCreationInputTokens`, `reasoningTokens` to `TokenUsage`. +* Drop `AiFrameworkChatResponse#rawChatResponse()`. +* LangChain4j converter populates the new fields where the framework provides them; uses + null defaults elsewhere. + +No call-site changes. Existing tests pass with no behavior change. + +### Phase 1 — Complete replacement (one shipping unit) + +* New SPI: `ChatModelApiFactory`, `ChatModelApiRegistry`, `ChatModelApi`, `ChatClient` + facade, `ChatModelEvent` sealed hierarchy, `ChatStreamListener`. +* Capability matrix YAML + resolution chain (config override → exact id / alias → pattern + → conservative defaults). +* `ToolCallResultStrategy` (multimodal-native and user-message-fallback policies). +* Five native `ChatModelApiFactory` implementations: `anthropic-messages`, + `bedrock-converse`, `openai-responses`, `openai-completions`, `google-genai`. +* `ProviderConfiguration` restructure with Jackson migration deserializer. +* Element template version bump (Task and Sub-process flavors). +* `BaseAgentRequestHandler` cuts over to `ChatClient`. The `AiFrameworkAdapter` interface + is removed. +* The LangChain4j integration (`framework/langchain4j/`) is preserved as an + opt-in bridge for users assembling custom bundles with provider models we have not yet + covered natively. It implements `ChatModelApiFactory` but is not registered by default. + Same plan applies to a future Spring AI bridge — the architecture supports it as a + drop-in `ChatModelApiFactory` implementation, but adoption is out of scope for this + iteration. + +The rollout is all-or-nothing. No feature flag selects between the legacy framework and +the native layer; users on supported models migrate transparently when they consume the +release. + +## Future Extensions + +Items deliberately scoped out of this iteration: + +* **Cross-model session normalization**. A `MessageTransformer` layer that handles + conversations spanning multiple models in a single session (degrading reasoning blocks + when model changes, normalizing tool-call IDs across provider ID-format constraints, + skipping errored assistant turns on replay) is unnecessary while a job is bound to a + single model. The architecture leaves an interception point at the boundary between + `ChatClient` and `ChatModelApi` where this slots in cleanly when needed. +* **Spring AI bridge**. The `ChatModelApiFactory` SPI is shaped so a future bridge + implementing the same interface lights up Spring AI's broader provider catalog without + blocking the current scope. +* **Cost tracking**. Token accounting is in scope; cost computation from token counts is + deferred. The capability matrix schema does not carry cost fields at this stage. +* **Reactive streaming surface**. The boundary is `CompletableFuture`; + exposing the underlying event stream as a reactive primitive is deferred. The internal + `ChatStreamListener` extension point covers in-process observability needs without + committing to a public reactive API. +* **Connector SDK retry classification**. The error semantics distinguish model-side errors + (response with `stopReason: ERROR`) from transport / auth errors (exceptional future + completion). Whether the Connector SDK can suppress automatic retry per error code so + authentication / configuration errors do not retry pointlessly is a separate work item; + the current design does not depend on it. +* **Output media materialization**. `output_modalities` in the capability matrix + accommodates models that emit images or other media; the `ChatModelApi` implementation + responsible for the first such model adds the byte → Camunda Document materialization + path at that time. The schema is forward-compatible. +* **Tool argument JSON repair**. A small utility that runs a best-effort repair pass on + malformed tool-argument JSON streamed from providers will be added under + `framework/util/` and called from each native implementation. Behavior is identical to + the SDK accumulators for well-formed JSON; recovery only kicks in on malformed cases + observed in production. + +## References + +* Current framework abstraction: + [`AiFrameworkAdapter.java`](../../src/main/java/io/camunda/connector/agenticai/aiagent/framework/AiFrameworkAdapter.java), + [`Langchain4JAiFrameworkAdapter.java`](../../src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkAdapter.java) +* Single migration call site: + [`BaseAgentRequestHandler.java`](../../src/main/java/io/camunda/connector/agenticai/aiagent/agent/BaseAgentRequestHandler.java) +* Domain model: + [`Message.java`](../../src/main/java/io/camunda/connector/agenticai/model/message/Message.java), + [`Content.java`](../../src/main/java/io/camunda/connector/agenticai/model/message/content/Content.java), + [`ToolCallResult.java`](../../src/main/java/io/camunda/connector/agenticai/model/tool/ToolCallResult.java), + [`AgentMetrics.java`](../../src/main/java/io/camunda/connector/agenticai/aiagent/model/AgentMetrics.java) +* Provider configurations: + [`ProviderConfiguration.java`](../../src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/ProviderConfiguration.java) +* Tool-result document handling (PR #6999): + [`AgentMessagesHandlerImpl.java`](../../src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java) +* Element templates: + [`agenticai-aiagent-outbound-connector.json`](../../element-templates/agenticai-aiagent-outbound-connector.json), + [`agenticai-aiagent-job-worker.json`](../../element-templates/agenticai-aiagent-job-worker.json), + [`element-templates/README.md`](../../element-templates/README.md) \ No newline at end of file From 226615ed6d978c656a0d0f505c2bb9ae9e6e9617 Mon Sep 17 00:00:00 2001 From: Dmitri Nikonov Date: Wed, 6 May 2026 13:33:25 +0200 Subject: [PATCH 40/81] test(agentic-ai): Add e2e tests against a wiremock as regression tests for later --- ...ropicMessagesApiAiAgentJobWorkerTests.java | 130 ++++++++++++++++ .../BaseWireFormatAiAgentJobWorkerTest.java | 108 +++++++++++++ ...enAiResponsesApiAiAgentJobWorkerTests.java | 146 ++++++++++++++++++ 3 files changed, 384 insertions(+) create mode 100644 connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/AnthropicMessagesApiAiAgentJobWorkerTests.java create mode 100644 connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/BaseWireFormatAiAgentJobWorkerTest.java create mode 100644 connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/OpenAiResponsesApiAiAgentJobWorkerTests.java diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/AnthropicMessagesApiAiAgentJobWorkerTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/AnthropicMessagesApiAiAgentJobWorkerTests.java new file mode 100644 index 00000000000..c78e72228e8 --- /dev/null +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/AnthropicMessagesApiAiAgentJobWorkerTests.java @@ -0,0 +1,130 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. Camunda licenses this file to you under the Apache License, + * Version 2.0; you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.camunda.connector.e2e.agenticai.aiagent.wireformat; + +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; +import static org.assertj.core.api.Assertions.assertThat; + +import io.camunda.connector.e2e.agenticai.assertj.JobWorkerAgentResponseAssert; +import io.camunda.connector.test.utils.annotation.SlowTest; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * Wire-format regression test for the Anthropic Messages API. Verifies the full HTTP contract + * between the AI Agent connector and the Anthropic {@code POST /v1/messages} endpoint using + * WireMock. Runs against LangChain4j initially; once ADR 004 native provider layer ships the same + * stubs cover the new implementation. + */ +@SlowTest +public class AnthropicMessagesApiAiAgentJobWorkerTests extends BaseWireFormatAiAgentJobWorkerTest { + + @Override + protected String llmApiPath() { + return "/v1/messages"; + } + + @Override + protected Map elementTemplateProperties() { + return Map.ofEntries( + Map.entry("agentContext", "=agent.context"), + Map.entry("provider.type", "anthropic"), + Map.entry("provider.anthropic.endpoint", "http://localhost:" + wireMockPort + "/v1"), + Map.entry("provider.anthropic.authentication.apiKey", "test-api-key"), + Map.entry("provider.anthropic.model.model", "claude-sonnet-4-6"), + Map.entry( + "data.systemPrompt.prompt", + "=\"You are a helpful AI assistant. Answer all the questions, but always be nice.\""), + Map.entry( + "data.userPrompt.prompt", + "=if (is defined(followUpUserPrompt)) then followUpUserPrompt else userPrompt"), + Map.entry("data.userPrompt.documents", "=[]"), + Map.entry("data.memory.storage.type", "in-process"), + Map.entry("data.response.includeAssistantMessage", "=true"), + Map.entry("data.response.includeAgentContext", "=true")); + } + + @Override + protected String toolCallResponseBody() { + return """ + { + "id": "msg_01XBvkPq5k7uHxRo45RsL7Ae", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01JnvVF6Cya67WpVoLMDkEaq", + "name": "SuperfluxProduct", + "input": {"a": 5, "b": 3} + } + ], + "model": "claude-sonnet-4-6", + "stop_reason": "tool_use", + "stop_sequence": null, + "usage": { + "input_tokens": 100, + "output_tokens": 50 + } + } + """; + } + + @Override + protected String finalResponseBody() { + return """ + { + "id": "msg_02YCwlPr6m8vJySp56SsM8Bf", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "%s" + } + ], + "model": "claude-sonnet-4-6", + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 200, + "output_tokens": 30 + } + } + """ + .formatted(RESPONSE_TEXT); + } + + @Test + void executesAgentWithToolCallAgainstAnthropicMessagesApi() throws Exception { + final var zeebeTest = runToolCallScenario(); + + assertAgentResponse( + zeebeTest, + agentResponse -> + JobWorkerAgentResponseAssert.assertThat(agentResponse) + .isReady() + .hasMetrics(EXPECTED_METRICS) + .hasResponseMessageText(RESPONSE_TEXT) + .hasResponseText(RESPONSE_TEXT)); + + assertThat(userFeedbackJobWorkerCounter.get()).isEqualTo(1); + verify(2, postRequestedFor(urlEqualTo(llmApiPath()))); + } +} diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/BaseWireFormatAiAgentJobWorkerTest.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/BaseWireFormatAiAgentJobWorkerTest.java new file mode 100644 index 00000000000..fd4d5cd229f --- /dev/null +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/BaseWireFormatAiAgentJobWorkerTest.java @@ -0,0 +1,108 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. Camunda licenses this file to you under the Apache License, + * Version 2.0; you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.camunda.connector.e2e.agenticai.aiagent.wireformat; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; + +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; +import io.camunda.connector.e2e.ZeebeTest; +import io.camunda.connector.e2e.agenticai.aiagent.BaseAiAgentJobWorkerTest; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; + +/** + * Base class for wire-format E2E tests that verify the AI Agent connector against a real HTTP API + * contract, mocked via WireMock. Unlike the Mockito-based tests that intercept at the {@code + * ChatModelFactory} level, these tests drive the full LLM HTTP stack so they remain valid as a + * regression suite when the provider implementation changes (e.g. switching from LangChain4j to the + * native provider layer described in ADR 004). + * + *

    Test scenario: single tool call (SuperfluxProduct) followed by a final text response, user + * satisfied on first feedback. + */ +abstract class BaseWireFormatAiAgentJobWorkerTest extends BaseAiAgentJobWorkerTest { + + protected static final String RESPONSE_TEXT = "The SuperfluxProduct of 5 and 3 is 24."; + protected static final String USER_PROMPT = "Calculate the superflux product of 5 and 3"; + + private static final String SCENARIO_NAME = "LLM Tool Call Flow"; + private static final String AFTER_TOOL_CALL_STATE = "AfterToolCall"; + + // Token counts in stub responses: turn1 input=100 output=50, turn2 input=200 output=30 + protected static final AgentMetrics EXPECTED_METRICS = + new AgentMetrics(2, new AgentMetrics.TokenUsage(300, 80)); + + protected int wireMockPort; + + @BeforeEach + void captureWireMockPort(WireMockRuntimeInfo wireMockRuntimeInfo) { + wireMockPort = wireMockRuntimeInfo.getHttpPort(); + } + + /** + * Sets user feedback to "satisfied" before each test so the process completes without a follow-up + * loop. Runs after {@link + * io.camunda.connector.e2e.agenticai.aiagent.BaseAiAgentTest#openUserFeedbackJobWorker()} resets + * the reference to an empty map. + */ + @BeforeEach + void setUserFeedbackToSatisfied() { + userFeedbackVariables.set(userSatisfiedFeedback()); + } + + /** The API path WireMock should intercept, e.g. {@code /v1/messages}. */ + protected abstract String llmApiPath(); + + /** Response body for the first LLM call — must instruct the agent to call SuperfluxProduct. */ + protected abstract String toolCallResponseBody(); + + /** Response body for the second LLM call — final text answer, no tool calls. */ + protected abstract String finalResponseBody(); + + protected void stubLlmApiForToolCallThenFinalResponse() { + stubFor( + post(urlEqualTo(llmApiPath())) + .inScenario(SCENARIO_NAME) + .whenScenarioStateIs(Scenario.STARTED) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(toolCallResponseBody())) + .willSetStateTo(AFTER_TOOL_CALL_STATE)); + + stubFor( + post(urlEqualTo(llmApiPath())) + .inScenario(SCENARIO_NAME) + .whenScenarioStateIs(AFTER_TOOL_CALL_STATE) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(finalResponseBody()))); + } + + protected ZeebeTest runToolCallScenario() throws Exception { + stubLlmApiForToolCallThenFinalResponse(); + return createProcessInstance(Map.of("userPrompt", USER_PROMPT)).waitForProcessCompletion(); + } +} diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/OpenAiResponsesApiAiAgentJobWorkerTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/OpenAiResponsesApiAiAgentJobWorkerTests.java new file mode 100644 index 00000000000..3527d4ea748 --- /dev/null +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/OpenAiResponsesApiAiAgentJobWorkerTests.java @@ -0,0 +1,146 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. Camunda licenses this file to you under the Apache License, + * Version 2.0; you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.camunda.connector.e2e.agenticai.aiagent.wireformat; + +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; +import static org.assertj.core.api.Assertions.assertThat; + +import io.camunda.connector.e2e.agenticai.assertj.JobWorkerAgentResponseAssert; +import io.camunda.connector.test.utils.annotation.SlowTest; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * Wire-format regression test for the OpenAI API. Uses the {@code openAiCompatible} provider + * (OpenAI Chat Completions, {@code POST /v1/chat/completions}) so the test runs against the + * LangChain4j implementation today. Once ADR 004's native {@code openai-responses} API family + * ships, this test will be updated to target the Responses API endpoint ({@code POST + * /v1/responses}) with the corresponding request/response schema. + */ +@SlowTest +public class OpenAiResponsesApiAiAgentJobWorkerTests extends BaseWireFormatAiAgentJobWorkerTest { + + @Override + protected String llmApiPath() { + return "/v1/chat/completions"; + } + + @Override + protected Map elementTemplateProperties() { + return Map.ofEntries( + Map.entry("agentContext", "=agent.context"), + Map.entry("provider.type", "openaiCompatible"), + Map.entry( + "provider.openaiCompatible.endpoint", "http://localhost:" + wireMockPort + "/v1/"), + Map.entry("provider.openaiCompatible.authentication.apiKey", "test-api-key"), + Map.entry("provider.openaiCompatible.model.model", "gpt-4o"), + Map.entry( + "data.systemPrompt.prompt", + "=\"You are a helpful AI assistant. Answer all the questions, but always be nice.\""), + Map.entry( + "data.userPrompt.prompt", + "=if (is defined(followUpUserPrompt)) then followUpUserPrompt else userPrompt"), + Map.entry("data.userPrompt.documents", "=[]"), + Map.entry("data.memory.storage.type", "in-process"), + Map.entry("data.response.includeAssistantMessage", "=true"), + Map.entry("data.response.includeAgentContext", "=true")); + } + + @Override + protected String toolCallResponseBody() { + return """ + { + "id": "chatcmpl-abc123", + "object": "chat.completion", + "created": 1728933352, + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_xyz789", + "type": "function", + "function": { + "name": "SuperfluxProduct", + "arguments": "{\\"a\\": 5, \\"b\\": 3}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150 + } + } + """; + } + + @Override + protected String finalResponseBody() { + return """ + { + "id": "chatcmpl-def456", + "object": "chat.completion", + "created": 1728933353, + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "%s" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 200, + "completion_tokens": 30, + "total_tokens": 230 + } + } + """ + .formatted(RESPONSE_TEXT); + } + + @Test + void executesAgentWithToolCallAgainstOpenAiChatCompletionsApi() throws Exception { + final var zeebeTest = runToolCallScenario(); + + assertAgentResponse( + zeebeTest, + agentResponse -> + JobWorkerAgentResponseAssert.assertThat(agentResponse) + .isReady() + .hasMetrics(EXPECTED_METRICS) + .hasResponseMessageText(RESPONSE_TEXT) + .hasResponseText(RESPONSE_TEXT)); + + assertThat(userFeedbackJobWorkerCounter.get()).isEqualTo(1); + verify(2, postRequestedFor(urlEqualTo(llmApiPath()))); + } +} From c15b64ae3bfcf76a5ae092b9899fa5c607519901 Mon Sep 17 00:00:00 2001 From: Dmitri Nikonov Date: Wed, 6 May 2026 13:51:09 +0200 Subject: [PATCH 41/81] test: verify api key authorization --- ...ropicMessagesApiAiAgentJobWorkerTests.java | 7 +++ .../BaseWireFormatAiAgentJobWorkerTest.java | 43 +++++++++++-------- ...enAiResponsesApiAiAgentJobWorkerTests.java | 10 ++++- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/AnthropicMessagesApiAiAgentJobWorkerTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/AnthropicMessagesApiAiAgentJobWorkerTests.java index c78e72228e8..ff287ce24e1 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/AnthropicMessagesApiAiAgentJobWorkerTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/AnthropicMessagesApiAiAgentJobWorkerTests.java @@ -16,11 +16,13 @@ */ package io.camunda.connector.e2e.agenticai.aiagent.wireformat; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; import static com.github.tomakehurst.wiremock.client.WireMock.verify; import static org.assertj.core.api.Assertions.assertThat; +import com.github.tomakehurst.wiremock.client.MappingBuilder; import io.camunda.connector.e2e.agenticai.assertj.JobWorkerAgentResponseAssert; import io.camunda.connector.test.utils.annotation.SlowTest; import java.util.Map; @@ -60,6 +62,11 @@ protected Map elementTemplateProperties() { Map.entry("data.response.includeAgentContext", "=true")); } + @Override + protected MappingBuilder withApiKeyHeaderMatcher(MappingBuilder stub) { + return stub.withHeader("x-api-key", equalTo("test-api-key")); + } + @Override protected String toolCallResponseBody() { return """ diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/BaseWireFormatAiAgentJobWorkerTest.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/BaseWireFormatAiAgentJobWorkerTest.java index fd4d5cd229f..a696be5cae7 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/BaseWireFormatAiAgentJobWorkerTest.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/BaseWireFormatAiAgentJobWorkerTest.java @@ -21,6 +21,7 @@ import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import com.github.tomakehurst.wiremock.client.MappingBuilder; import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import com.github.tomakehurst.wiremock.stubbing.Scenario; import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; @@ -78,27 +79,35 @@ void setUserFeedbackToSatisfied() { /** Response body for the second LLM call — final text answer, no tool calls. */ protected abstract String finalResponseBody(); + /** + * Adds provider-specific API key header matching to the stub, e.g. {@code x-api-key} for + * Anthropic or {@code Authorization: Bearer} for OpenAI. + */ + protected abstract MappingBuilder withApiKeyHeaderMatcher(MappingBuilder stub); + protected void stubLlmApiForToolCallThenFinalResponse() { stubFor( - post(urlEqualTo(llmApiPath())) - .inScenario(SCENARIO_NAME) - .whenScenarioStateIs(Scenario.STARTED) - .willReturn( - aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody(toolCallResponseBody())) - .willSetStateTo(AFTER_TOOL_CALL_STATE)); + withApiKeyHeaderMatcher( + post(urlEqualTo(llmApiPath())) + .inScenario(SCENARIO_NAME) + .whenScenarioStateIs(Scenario.STARTED) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(toolCallResponseBody())) + .willSetStateTo(AFTER_TOOL_CALL_STATE))); stubFor( - post(urlEqualTo(llmApiPath())) - .inScenario(SCENARIO_NAME) - .whenScenarioStateIs(AFTER_TOOL_CALL_STATE) - .willReturn( - aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody(finalResponseBody()))); + withApiKeyHeaderMatcher( + post(urlEqualTo(llmApiPath())) + .inScenario(SCENARIO_NAME) + .whenScenarioStateIs(AFTER_TOOL_CALL_STATE) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(finalResponseBody())))); } protected ZeebeTest runToolCallScenario() throws Exception { diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/OpenAiResponsesApiAiAgentJobWorkerTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/OpenAiResponsesApiAiAgentJobWorkerTests.java index 3527d4ea748..2e125f8899a 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/OpenAiResponsesApiAiAgentJobWorkerTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/OpenAiResponsesApiAiAgentJobWorkerTests.java @@ -16,11 +16,13 @@ */ package io.camunda.connector.e2e.agenticai.aiagent.wireformat; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; import static com.github.tomakehurst.wiremock.client.WireMock.verify; import static org.assertj.core.api.Assertions.assertThat; +import com.github.tomakehurst.wiremock.client.MappingBuilder; import io.camunda.connector.e2e.agenticai.assertj.JobWorkerAgentResponseAssert; import io.camunda.connector.test.utils.annotation.SlowTest; import java.util.Map; @@ -46,8 +48,7 @@ protected Map elementTemplateProperties() { return Map.ofEntries( Map.entry("agentContext", "=agent.context"), Map.entry("provider.type", "openaiCompatible"), - Map.entry( - "provider.openaiCompatible.endpoint", "http://localhost:" + wireMockPort + "/v1/"), + Map.entry("provider.openaiCompatible.endpoint", "http://localhost:" + wireMockPort + "/v1"), Map.entry("provider.openaiCompatible.authentication.apiKey", "test-api-key"), Map.entry("provider.openaiCompatible.model.model", "gpt-4o"), Map.entry( @@ -62,6 +63,11 @@ protected Map elementTemplateProperties() { Map.entry("data.response.includeAgentContext", "=true")); } + @Override + protected MappingBuilder withApiKeyHeaderMatcher(MappingBuilder stub) { + return stub.withHeader("Authorization", equalTo("Bearer test-api-key")); + } + @Override protected String toolCallResponseBody() { return """ From e4a04c47a196697c01e126e2c2a451b2282c66dc Mon Sep 17 00:00:00 2001 From: Dmitri Nikonov Date: Wed, 6 May 2026 15:17:14 +0200 Subject: [PATCH 42/81] feat(agentic-ai): extend data model to support common llm provider properties --- .../aiagent/framework/AiFrameworkAdapter.java | 2 +- .../framework/AiFrameworkChatResponse.java | 4 +- .../langchain4j/ChatMessageConverterImpl.java | 65 ++++++++- .../langchain4j/ContentConverterImpl.java | 4 + .../Langchain4JAiFrameworkAdapter.java | 20 +-- .../Langchain4JAiFrameworkChatResponse.java | 5 +- .../agenticai/aiagent/model/AgentMetrics.java | 17 ++- .../model/message/AssistantMessage.java | 6 + .../agenticai/model/message/StopReason.java | 17 +++ .../model/message/content/Content.java | 6 +- .../message/content/ReasoningContent.java | 37 +++++ .../agenticai/model/tool/ToolCallResult.java | 3 + .../agent/AgentResponseHandlerTest.java | 2 +- .../JobWorkerAgentRequestHandlerTest.java | 27 ++-- ...boundConnectorAgentRequestHandlerTest.java | 23 +-- .../langchain4j/ChatMessageConverterTest.java | 138 ++++++++++++++++++ .../langchain4j/ContentConverterTest.java | 12 ++ .../Langchain4JAiFrameworkAdapterTest.java | 9 +- .../aiagent/model/AgentContextTest.java | 12 +- .../aiagent/model/AgentMetricsTest.java | 45 +++++- 20 files changed, 394 insertions(+), 60 deletions(-) create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/StopReason.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/content/ReasoningContent.java diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/AiFrameworkAdapter.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/AiFrameworkAdapter.java index 732e1884529..8c36fdf03d0 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/AiFrameworkAdapter.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/AiFrameworkAdapter.java @@ -10,7 +10,7 @@ import io.camunda.connector.agenticai.aiagent.model.AgentContext; import io.camunda.connector.agenticai.aiagent.model.AgentExecutionContext; -public interface AiFrameworkAdapter> { +public interface AiFrameworkAdapter { R executeChatRequest( AgentExecutionContext executionContext, AgentContext agentContext, diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/AiFrameworkChatResponse.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/AiFrameworkChatResponse.java index 50b04538f45..66a1bb4b8f9 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/AiFrameworkChatResponse.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/AiFrameworkChatResponse.java @@ -9,10 +9,8 @@ import io.camunda.connector.agenticai.aiagent.model.AgentContext; import io.camunda.connector.agenticai.model.message.AssistantMessage; -public interface AiFrameworkChatResponse { +public interface AiFrameworkChatResponse { AgentContext agentContext(); AssistantMessage assistantMessage(); - - T rawChatResponse(); } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterImpl.java index 246b9e67ebd..5e149d9c6af 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterImpl.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterImpl.java @@ -13,12 +13,18 @@ import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.data.message.ToolExecutionResultMessage; import dev.langchain4j.internal.Json; +import dev.langchain4j.model.anthropic.AnthropicTokenUsage; +import dev.langchain4j.model.bedrock.BedrockTokenUsage; import dev.langchain4j.model.chat.response.ChatResponse; import dev.langchain4j.model.chat.response.ChatResponseMetadata; +import dev.langchain4j.model.openai.OpenAiTokenUsage; +import dev.langchain4j.model.output.FinishReason; import dev.langchain4j.model.output.TokenUsage; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolCallConverter; +import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.agenticai.model.message.AssistantMessage; import io.camunda.connector.agenticai.model.message.AssistantMessageBuilder; +import io.camunda.connector.agenticai.model.message.StopReason; import io.camunda.connector.agenticai.model.message.SystemMessage; import io.camunda.connector.agenticai.model.message.ToolCallResultMessage; import io.camunda.connector.agenticai.model.message.UserMessage; @@ -132,11 +138,23 @@ public AssistantMessage toAssistantMessage(ChatResponse chatResponse) { protected AssistantMessageBuilder toAssistantMessageBuilder(ChatResponse chatResponse) { final var builder = AssistantMessage.builder(); - if (chatResponse.metadata() != null) { + final ChatResponseMetadata metadata = chatResponse.metadata(); + if (metadata != null) { builder.metadata( Map.of( "timestamp", ZonedDateTime.now(), - "framework", serializedChatResponseMetadata(chatResponse.metadata()))); + "framework", serializedChatResponseMetadata(metadata))); + + Optional.ofNullable(metadata.modelName()) + .filter(StringUtils::isNotBlank) + .ifPresent(builder::modelId); + Optional.ofNullable(metadata.id()).filter(StringUtils::isNotBlank).ifPresent(builder::apiId); + Optional.ofNullable(metadata.finishReason()) + .map(this::toStopReason) + .ifPresent(builder::stopReason); + Optional.ofNullable(metadata.tokenUsage()) + .map(this::toDomainTokenUsage) + .ifPresent(builder::usage); } final var aiMessage = chatResponse.aiMessage(); @@ -152,6 +170,49 @@ protected AssistantMessageBuilder toAssistantMessageBuilder(ChatResponse chatRes return builder; } + private StopReason toStopReason(FinishReason finishReason) { + return switch (finishReason) { + case STOP -> StopReason.STOP; + case LENGTH -> StopReason.LENGTH; + case TOOL_EXECUTION -> StopReason.TOOL_USE; + case CONTENT_FILTER -> StopReason.CONTENT_FILTERED; + case OTHER -> null; + }; + } + + AgentMetrics.TokenUsage toDomainTokenUsage(TokenUsage tokenUsage) { + if (tokenUsage == null) { + return AgentMetrics.TokenUsage.empty(); + } + + final var builder = + AgentMetrics.TokenUsage.builder() + .inputTokenCount(Optional.ofNullable(tokenUsage.inputTokenCount()).orElse(0)) + .outputTokenCount(Optional.ofNullable(tokenUsage.outputTokenCount()).orElse(0)); + + if (tokenUsage instanceof AnthropicTokenUsage anthropicTokenUsage) { + Optional.ofNullable(anthropicTokenUsage.cacheReadInputTokens()) + .ifPresent(builder::cacheReadInputTokenCount); + Optional.ofNullable(anthropicTokenUsage.cacheCreationInputTokens()) + .ifPresent(builder::cacheCreationInputTokenCount); + } else if (tokenUsage instanceof BedrockTokenUsage bedrockTokenUsage) { + Optional.ofNullable(bedrockTokenUsage.cacheReadInputTokens()) + .ifPresent(builder::cacheReadInputTokenCount); + // Bedrock uses "write" terminology; we expose it as "creation" to match Anthropic's semantics + Optional.ofNullable(bedrockTokenUsage.cacheWriteInputTokens()) + .ifPresent(builder::cacheCreationInputTokenCount); + } else if (tokenUsage instanceof OpenAiTokenUsage openAiTokenUsage) { + Optional.ofNullable(openAiTokenUsage.inputTokensDetails()) + .map(OpenAiTokenUsage.InputTokensDetails::cachedTokens) + .ifPresent(builder::cacheReadInputTokenCount); + Optional.ofNullable(openAiTokenUsage.outputTokensDetails()) + .map(OpenAiTokenUsage.OutputTokensDetails::reasoningTokens) + .ifPresent(builder::reasoningTokenCount); + } + + return builder.build(); + } + protected Map serializedChatResponseMetadata( ChatResponseMetadata chatResponseMetadata) { if (chatResponseMetadata == null) { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ContentConverterImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ContentConverterImpl.java index 38a0cc3dbe1..843a3c3ca6c 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ContentConverterImpl.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ContentConverterImpl.java @@ -12,6 +12,7 @@ import io.camunda.connector.agenticai.model.message.content.Content; import io.camunda.connector.agenticai.model.message.content.DocumentContent; import io.camunda.connector.agenticai.model.message.content.ObjectContent; +import io.camunda.connector.agenticai.model.message.content.ReasoningContent; import io.camunda.connector.agenticai.model.message.content.TextContent; public class ContentConverterImpl implements ContentConverter { @@ -34,6 +35,9 @@ public dev.langchain4j.data.message.Content convertToContent(Content content) documentToContentConverter.convert(documentContent.document()); case ObjectContent objectContent -> new dev.langchain4j.data.message.TextContent(convertToString(objectContent.content())); + case ReasoningContent ignored -> + throw new UnsupportedOperationException( + "ReasoningContent is not supported by the LangChain4j framework integration"); }; } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkAdapter.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkAdapter.java index 5d8b1de6a49..8769794d358 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkAdapter.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkAdapter.java @@ -14,7 +14,6 @@ import dev.langchain4j.model.chat.request.ResponseFormatType; import dev.langchain4j.model.chat.request.json.JsonSchema; import dev.langchain4j.model.chat.response.ChatResponse; -import dev.langchain4j.model.output.TokenUsage; import io.camunda.connector.agenticai.aiagent.framework.AiFrameworkAdapter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.jsonschema.JsonSchemaConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverter; @@ -70,10 +69,12 @@ public Langchain4JAiFrameworkChatResponse executeChatRequest( agentContext .metrics() .incrementModelCalls(1) - .incrementTokenUsage(tokenUsage(chatResponse.tokenUsage()))); + .incrementTokenUsage( + assistantMessage.usage() != null + ? assistantMessage.usage() + : AgentMetrics.TokenUsage.empty())); - return new Langchain4JAiFrameworkChatResponse( - updatedAgentContext, assistantMessage, chatResponse); + return new Langchain4JAiFrameworkChatResponse(updatedAgentContext, assistantMessage); } private void configureResponseFormat( @@ -122,15 +123,4 @@ private ChatResponse doChat(ChatModel chatModel, ChatRequest.Builder chatRequest ERROR_CODE_FAILED_MODEL_CALL, "Model call failed: %s".formatted(message), e); } } - - private AgentMetrics.TokenUsage tokenUsage(TokenUsage tokenUsage) { - if (tokenUsage == null) { - return AgentMetrics.TokenUsage.empty(); - } - - return AgentMetrics.TokenUsage.builder() - .inputTokenCount(Optional.ofNullable(tokenUsage.inputTokenCount()).orElse(0)) - .outputTokenCount(Optional.ofNullable(tokenUsage.outputTokenCount()).orElse(0)) - .build(); - } } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkChatResponse.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkChatResponse.java index b666672a843..09e00a87cd0 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkChatResponse.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkChatResponse.java @@ -6,11 +6,10 @@ */ package io.camunda.connector.agenticai.aiagent.framework.langchain4j; -import dev.langchain4j.model.chat.response.ChatResponse; import io.camunda.connector.agenticai.aiagent.framework.AiFrameworkChatResponse; import io.camunda.connector.agenticai.aiagent.model.AgentContext; import io.camunda.connector.agenticai.model.message.AssistantMessage; public record Langchain4JAiFrameworkChatResponse( - AgentContext agentContext, AssistantMessage assistantMessage, ChatResponse rawChatResponse) - implements AiFrameworkChatResponse {} + AgentContext agentContext, AssistantMessage assistantMessage) + implements AiFrameworkChatResponse {} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/AgentMetrics.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/AgentMetrics.java index 59ff1fcd5ec..88db5688d6e 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/AgentMetrics.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/AgentMetrics.java @@ -6,6 +6,7 @@ */ package io.camunda.connector.agenticai.aiagent.model; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import io.camunda.connector.agenticai.model.AgenticAiRecord; @@ -47,7 +48,12 @@ public static class AgentMetricsJacksonProxyBuilder extends AgentMetricsBuilder @AgenticAiRecord @JsonDeserialize(builder = TokenUsage.AgentMetricsTokenUsageJacksonProxyBuilder.class) - public record TokenUsage(int inputTokenCount, int outputTokenCount) + public record TokenUsage( + int inputTokenCount, + int outputTokenCount, + @JsonInclude(JsonInclude.Include.NON_DEFAULT) int cacheReadInputTokenCount, + @JsonInclude(JsonInclude.Include.NON_DEFAULT) int cacheCreationInputTokenCount, + @JsonInclude(JsonInclude.Include.NON_DEFAULT) int reasoningTokenCount) implements AgentMetricsTokenUsageBuilder.With { public int totalTokenCount() { @@ -59,7 +65,14 @@ public TokenUsage add(TokenUsage tokenUsage) { builder -> builder .inputTokenCount(builder.inputTokenCount() + tokenUsage.inputTokenCount()) - .outputTokenCount(builder.outputTokenCount() + tokenUsage.outputTokenCount())); + .outputTokenCount(builder.outputTokenCount() + tokenUsage.outputTokenCount()) + .cacheReadInputTokenCount( + builder.cacheReadInputTokenCount() + tokenUsage.cacheReadInputTokenCount()) + .cacheCreationInputTokenCount( + builder.cacheCreationInputTokenCount() + + tokenUsage.cacheCreationInputTokenCount()) + .reasoningTokenCount( + builder.reasoningTokenCount() + tokenUsage.reasoningTokenCount())); } public static TokenUsage empty() { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/AssistantMessage.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/AssistantMessage.java index 202dad6b53b..0f578259e71 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/AssistantMessage.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/AssistantMessage.java @@ -9,17 +9,23 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.agenticai.model.AgenticAiRecord; import io.camunda.connector.agenticai.model.message.content.Content; import io.camunda.connector.agenticai.model.tool.ToolCall; import java.util.List; import java.util.Map; +import org.springframework.lang.Nullable; @AgenticAiRecord @JsonDeserialize(builder = AssistantMessage.AssistantMessageJacksonProxyBuilder.class) public record AssistantMessage( @JsonInclude(JsonInclude.Include.NON_EMPTY) List content, @JsonInclude(JsonInclude.Include.NON_EMPTY) List toolCalls, + @Nullable String modelId, + @Nullable String apiId, + @Nullable StopReason stopReason, + @Nullable AgentMetrics.TokenUsage usage, @JsonInclude(JsonInclude.Include.NON_EMPTY) Map metadata) implements AssistantMessageBuilder.With, Message, ContentMessage { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/StopReason.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/StopReason.java new file mode 100644 index 00000000000..b540e3b976a --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/StopReason.java @@ -0,0 +1,17 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.model.message; + +public enum StopReason { + STOP, + LENGTH, + TOOL_USE, + ERROR, + ABORTED, + CONTENT_FILTERED, + GUARDRAIL +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/content/Content.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/content/Content.java index aabdba82e0b..8d6e7c5e261 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/content/Content.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/content/Content.java @@ -14,8 +14,10 @@ @JsonSubTypes({ @JsonSubTypes.Type(value = TextContent.class, name = "text"), @JsonSubTypes.Type(value = DocumentContent.class, name = "document"), - @JsonSubTypes.Type(value = ObjectContent.class, name = "object") + @JsonSubTypes.Type(value = ObjectContent.class, name = "object"), + @JsonSubTypes.Type(value = ReasoningContent.class, name = "reasoning") }) -public sealed interface Content permits TextContent, DocumentContent, ObjectContent { +public sealed interface Content + permits TextContent, DocumentContent, ObjectContent, ReasoningContent { Map metadata(); } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/content/ReasoningContent.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/content/ReasoningContent.java new file mode 100644 index 00000000000..6a2c19a84e7 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/content/ReasoningContent.java @@ -0,0 +1,37 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.model.message.content; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import java.util.Map; +import org.springframework.lang.Nullable; + +@JsonIgnoreProperties(ignoreUnknown = true) +public record ReasoningContent( + String text, + @Nullable String signature, + boolean redacted, + @JsonInclude(JsonInclude.Include.NON_EMPTY) Map metadata) + implements Content { + + public ReasoningContent { + if (text == null) { + throw new IllegalArgumentException("Text cannot be null"); + } + // empty/blank text is allowed — providers emit empty thinking blocks + // with non-null signatures for redaction or roundtrip-only purposes + } + + public static ReasoningContent reasoningContent(String text) { + return new ReasoningContent(text, null, false, null); + } + + public static ReasoningContent reasoningContent(String text, String signature) { + return new ReasoningContent(text, signature, false, null); + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/tool/ToolCallResult.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/tool/ToolCallResult.java index 3e08d83e161..0a0194ea717 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/tool/ToolCallResult.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/tool/ToolCallResult.java @@ -12,7 +12,9 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import io.camunda.connector.agenticai.model.AgenticAiRecord; +import io.camunda.connector.agenticai.model.message.content.Content; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.springframework.lang.Nullable; @@ -22,6 +24,7 @@ public record ToolCallResult( @Nullable String id, @Nullable String name, @Nullable Object content, + @Nullable @JsonInclude(JsonInclude.Include.NON_EMPTY) List contentBlocks, @JsonInclude(JsonInclude.Include.NON_EMPTY) @JsonAnySetter @JsonAnyGetter Map properties) implements ToolCallResultBuilder.With { diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentResponseHandlerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentResponseHandlerTest.java index 15d40991d02..714255b8aed 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentResponseHandlerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentResponseHandlerTest.java @@ -206,7 +206,7 @@ void returnsAssistantMessageIfConfigured() { static Stream emptyAssistantMessages() { return Stream.of( - new AssistantMessage(List.of(), List.of(), Map.of()), + AssistantMessage.builder().content(List.of()).toolCalls(List.of()).build(), assistantMessage(List.of(DocumentContent.documentContent(mock(Document.class))))); } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/JobWorkerAgentRequestHandlerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/JobWorkerAgentRequestHandlerTest.java index b7e5a82e897..ba8b3810606 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/JobWorkerAgentRequestHandlerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/JobWorkerAgentRequestHandlerTest.java @@ -182,7 +182,9 @@ void orchestratesRequestExecutionWithoutToolCalls() { assertThat(agentResponse.context()).isEqualTo(response.variables().get("agentContext")); assertThat(agentResponse.context().state()).isEqualTo(AgentState.READY); assertThat(agentResponse.context().metrics()) - .isEqualTo(new AgentMetrics(1, new TokenUsage(10, 20))); + .isEqualTo( + new AgentMetrics( + 1, TokenUsage.builder().inputTokenCount(10).outputTokenCount(20).build())); assertThat(agentResponse.context().conversation()) .isNotNull() .isInstanceOfSatisfying( @@ -248,7 +250,9 @@ void orchestratesRequestExecutionWithToolCalls() { assertThat(agentResponse).isNotNull(); assertThat(agentResponse.context().state()).isEqualTo(AgentState.READY); assertThat(agentResponse.context().metrics()) - .isEqualTo(new AgentMetrics(1, new TokenUsage(10, 20))); + .isEqualTo( + new AgentMetrics( + 1, TokenUsage.builder().inputTokenCount(10).outputTokenCount(20).build())); assertThat(agentResponse.context().conversation()) .isNotNull() .isInstanceOfSatisfying( @@ -345,7 +349,9 @@ void orchestratesRequestExecutionWithInterruptedToolCall() { assertThat(agentResponse.context()).isEqualTo(response.variables().get("agentContext")); assertThat(agentResponse.context().state()).isEqualTo(AgentState.READY); assertThat(agentResponse.context().metrics()) - .isEqualTo(new AgentMetrics(1, new TokenUsage(10, 20))); + .isEqualTo( + new AgentMetrics( + 1, TokenUsage.builder().inputTokenCount(10).outputTokenCount(20).build())); assertThat(agentResponse.context().conversation()) .isNotNull() .isInstanceOfSatisfying( @@ -528,15 +534,16 @@ private void mockFrameworkExecution(AssistantMessage assistantMessage) { agentContext .metrics() .incrementModelCalls(1) - .incrementTokenUsage(new TokenUsage(10, 20))), - assistantMessage, - Map.of("message", assistantMessage.content())); + .incrementTokenUsage( + TokenUsage.builder() + .inputTokenCount(10) + .outputTokenCount(20) + .build())), + assistantMessage); }); } private record TestFrameworkChatResponse( - AgentContext agentContext, - AssistantMessage assistantMessage, - Map rawChatResponse) - implements AiFrameworkChatResponse> {} + AgentContext agentContext, AssistantMessage assistantMessage) + implements AiFrameworkChatResponse {} } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/OutboundConnectorAgentRequestHandlerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/OutboundConnectorAgentRequestHandlerTest.java index def3e654177..bd93d6f7963 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/OutboundConnectorAgentRequestHandlerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/OutboundConnectorAgentRequestHandlerTest.java @@ -163,7 +163,9 @@ void orchestratesRequestExecutionWithoutToolCalls() { assertThat(agentResponse).isNotNull(); assertThat(agentResponse.context().state()).isEqualTo(AgentState.READY); assertThat(agentResponse.context().metrics()) - .isEqualTo(new AgentMetrics(1, new TokenUsage(10, 20))); + .isEqualTo( + new AgentMetrics( + 1, TokenUsage.builder().inputTokenCount(10).outputTokenCount(20).build())); assertThat(agentResponse.context().conversation()) .isNotNull() .isInstanceOfSatisfying( @@ -224,7 +226,9 @@ void orchestratesRequestExecutionWithToolCalls() { assertThat(agentResponse).isNotNull(); assertThat(agentResponse.context().state()).isEqualTo(AgentState.READY); assertThat(agentResponse.context().metrics()) - .isEqualTo(new AgentMetrics(1, new TokenUsage(10, 20))); + .isEqualTo( + new AgentMetrics( + 1, TokenUsage.builder().inputTokenCount(10).outputTokenCount(20).build())); assertThat(agentResponse.context().conversation()) .isNotNull() .isInstanceOfSatisfying( @@ -390,15 +394,16 @@ private void mockFrameworkExecution(AssistantMessage assistantMessage) { agentContext .metrics() .incrementModelCalls(1) - .incrementTokenUsage(new TokenUsage(10, 20))), - assistantMessage, - Map.of("message", assistantMessage.content())); + .incrementTokenUsage( + TokenUsage.builder() + .inputTokenCount(10) + .outputTokenCount(20) + .build())), + assistantMessage); }); } private record TestFrameworkChatResponse( - AgentContext agentContext, - AssistantMessage assistantMessage, - Map rawChatResponse) - implements AiFrameworkChatResponse> {} + AgentContext agentContext, AssistantMessage assistantMessage) + implements AiFrameworkChatResponse {} } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterTest.java index b6008e673ad..9332db9f6d7 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterTest.java @@ -11,6 +11,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.entry; import static org.assertj.core.api.Assertions.within; +import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -23,6 +24,8 @@ import dev.langchain4j.data.message.Content; import dev.langchain4j.data.message.ToolExecutionResultMessage; import dev.langchain4j.http.client.SuccessfulHttpResponse; +import dev.langchain4j.model.anthropic.AnthropicTokenUsage; +import dev.langchain4j.model.bedrock.BedrockTokenUsage; import dev.langchain4j.model.chat.response.ChatResponse; import dev.langchain4j.model.chat.response.ChatResponseMetadata; import dev.langchain4j.model.openai.OpenAiChatResponseMetadata; @@ -30,8 +33,10 @@ import dev.langchain4j.model.output.FinishReason; import dev.langchain4j.model.output.TokenUsage; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolCallConverter; +import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.agenticai.model.message.AssistantMessage; import io.camunda.connector.agenticai.model.message.Message; +import io.camunda.connector.agenticai.model.message.StopReason; import io.camunda.connector.agenticai.model.message.SystemMessage; import io.camunda.connector.agenticai.model.message.ToolCallResultMessage; import io.camunda.connector.agenticai.model.message.UserMessage; @@ -46,9 +51,13 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.stream.Stream; import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Spy; @@ -426,4 +435,133 @@ public Map metadata() { .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Unknown message type"); } + + @Test + void toAssistantMessage_populatesTypedProvenanceFields() { + final var aiMessage = AiMessage.builder().text("Hello").build(); + final var chatResponseMetadata = + ChatResponseMetadata.builder() + .id("msg-abc") + .modelName("claude-3-7-sonnet") + .finishReason(FinishReason.STOP) + .tokenUsage(new TokenUsage(5, 10)) + .build(); + final var chatResponse = + new ChatResponse.Builder().aiMessage(aiMessage).metadata(chatResponseMetadata).build(); + + final var result = chatMessageConverter.toAssistantMessage(chatResponse); + + assertThat(result.apiId()).isEqualTo("msg-abc"); + assertThat(result.modelId()).isEqualTo("claude-3-7-sonnet"); + assertThat(result.stopReason()).isEqualTo(StopReason.STOP); + assertThat(result.usage()) + .isEqualTo( + AgentMetrics.TokenUsage.builder().inputTokenCount(5).outputTokenCount(10).build()); + } + + @Test + void toAssistantMessage_extractsAnthropicCacheTokens() { + final var aiMessage = AiMessage.builder().build(); + final var anthropicTokenUsage = + AnthropicTokenUsage.builder() + .inputTokenCount(10) + .outputTokenCount(5) + .cacheReadInputTokens(3) + .cacheCreationInputTokens(7) + .build(); + final var chatResponseMetadata = + ChatResponseMetadata.builder().tokenUsage(anthropicTokenUsage).build(); + final var chatResponse = + new ChatResponse.Builder().aiMessage(aiMessage).metadata(chatResponseMetadata).build(); + + final var result = chatMessageConverter.toAssistantMessage(chatResponse); + + assertThat(result.usage()) + .isEqualTo( + AgentMetrics.TokenUsage.builder() + .inputTokenCount(10) + .outputTokenCount(5) + .cacheReadInputTokenCount(3) + .cacheCreationInputTokenCount(7) + .build()); + } + + @Test + void toAssistantMessage_extractsBedrockCacheTokens() { + final var aiMessage = AiMessage.builder().build(); + final var bedrockTokenUsage = + BedrockTokenUsage.builder() + .inputTokenCount(8) + .outputTokenCount(4) + .cacheReadInputTokens(2) + .cacheWriteInputTokens(6) + .build(); + final var chatResponseMetadata = + ChatResponseMetadata.builder().tokenUsage(bedrockTokenUsage).build(); + final var chatResponse = + new ChatResponse.Builder().aiMessage(aiMessage).metadata(chatResponseMetadata).build(); + + final var result = chatMessageConverter.toAssistantMessage(chatResponse); + + assertThat(result.usage()) + .isEqualTo( + AgentMetrics.TokenUsage.builder() + .inputTokenCount(8) + .outputTokenCount(4) + .cacheReadInputTokenCount(2) + .cacheCreationInputTokenCount(6) + .build()); + } + + @Test + void toAssistantMessage_extractsOpenAiReasoningTokens() { + final var aiMessage = AiMessage.builder().build(); + final var openAiTokenUsage = + OpenAiTokenUsage.builder() + .inputTokenCount(12) + .inputTokensDetails( + OpenAiTokenUsage.InputTokensDetails.builder().cachedTokens(4).build()) + .outputTokenCount(8) + .outputTokensDetails( + OpenAiTokenUsage.OutputTokensDetails.builder().reasoningTokens(3).build()) + .build(); + final var chatResponseMetadata = + ChatResponseMetadata.builder().tokenUsage(openAiTokenUsage).build(); + final var chatResponse = + new ChatResponse.Builder().aiMessage(aiMessage).metadata(chatResponseMetadata).build(); + + final var result = chatMessageConverter.toAssistantMessage(chatResponse); + + assertThat(result.usage()) + .isEqualTo( + AgentMetrics.TokenUsage.builder() + .inputTokenCount(12) + .outputTokenCount(8) + .cacheReadInputTokenCount(4) + .reasoningTokenCount(3) + .build()); + } + + @ParameterizedTest + @MethodSource("finishReasonMappings") + void toAssistantMessage_finishReasonMapping(FinishReason finishReason, StopReason expected) { + final var aiMessage = AiMessage.builder().build(); + final var chatResponseMetadata = + ChatResponseMetadata.builder().finishReason(finishReason).build(); + final var chatResponse = + new ChatResponse.Builder().aiMessage(aiMessage).metadata(chatResponseMetadata).build(); + + final var result = chatMessageConverter.toAssistantMessage(chatResponse); + + assertThat(result.stopReason()).isEqualTo(expected); + } + + static Stream finishReasonMappings() { + return Stream.of( + arguments(FinishReason.STOP, StopReason.STOP), + arguments(FinishReason.LENGTH, StopReason.LENGTH), + arguments(FinishReason.TOOL_EXECUTION, StopReason.TOOL_USE), + arguments(FinishReason.CONTENT_FILTER, StopReason.CONTENT_FILTERED), + arguments(FinishReason.OTHER, null)); + } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ContentConverterTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ContentConverterTest.java index 722af400699..6b949b5ece5 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ContentConverterTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ContentConverterTest.java @@ -7,13 +7,16 @@ package io.camunda.connector.agenticai.aiagent.framework.langchain4j; import static io.camunda.connector.agenticai.model.message.content.ObjectContent.objectContent; +import static io.camunda.connector.agenticai.model.message.content.ReasoningContent.reasoningContent; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.document.DocumentToContentConverterImpl; import io.camunda.connector.agenticai.model.message.content.DocumentContent; import io.camunda.connector.agenticai.model.message.content.ObjectContent; +import io.camunda.connector.agenticai.model.message.content.ReasoningContent; import io.camunda.connector.agenticai.model.message.content.TextContent; import io.camunda.connector.api.document.Document; import io.camunda.connector.api.document.DocumentCreationRequest; @@ -84,6 +87,15 @@ void supportsObjectContent() throws JsonProcessingException { assertThat(((dev.langchain4j.data.message.TextContent) content).text()) .isEqualTo("{\"key\":\"value\"}"); } + + @Test + void convertReasoningContent_throwsUnsupported() { + final ReasoningContent reasoningContent = reasoningContent("thinking..."); + + assertThatThrownBy(() -> contentConverter.convertToContent(reasoningContent)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("ReasoningContent is not supported by the LangChain4j"); + } } @Nested diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkAdapterTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkAdapterTest.java index 3fbeff8565d..0cd1c150caa 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkAdapterTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkAdapterTest.java @@ -26,7 +26,6 @@ import dev.langchain4j.model.chat.request.ResponseFormatType; import dev.langchain4j.model.chat.request.json.JsonObjectSchema; import dev.langchain4j.model.chat.response.ChatResponse; -import dev.langchain4j.model.output.TokenUsage; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.jsonschema.JsonSchemaConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverter; import io.camunda.connector.agenticai.aiagent.memory.runtime.DefaultRuntimeMemory; @@ -121,8 +120,10 @@ void setUp() { when(chatModelFactory.createChatModel(any())).thenReturn(chatModel); when(chatModel.chat(chatRequestCaptor.capture())).thenReturn(chatResponse); - when(chatResponse.tokenUsage()).thenReturn(new TokenUsage(5, 6)); - when(chatMessageConverter.toAssistantMessage(chatResponse)).thenReturn(ASSISTANT_MESSAGE); + when(chatMessageConverter.toAssistantMessage(chatResponse)) + .thenReturn( + ASSISTANT_MESSAGE.withUsage( + AgentMetrics.TokenUsage.builder().inputTokenCount(5).outputTokenCount(6).build())); adapter = new Langchain4JAiFrameworkAdapter( @@ -289,7 +290,7 @@ void incrementsMetricsFromResponse() { @Test void tokenUsageIsUnchangedIfMissingInResponse() { - when(chatResponse.tokenUsage()).thenReturn(null); + when(chatMessageConverter.toAssistantMessage(chatResponse)).thenReturn(ASSISTANT_MESSAGE); final var adapterResponse = adapter.executeChatRequest(createExecutionContext(), AGENT_CONTEXT, runtimeMemory); diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/AgentContextTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/AgentContextTest.java index 02ae5e7683b..bd279577fd5 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/AgentContextTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/AgentContextTest.java @@ -51,7 +51,9 @@ void withState() { @Test void withMetrics() { - final var updatedMetrics = new AgentMetrics(1, new AgentMetrics.TokenUsage(10, 20)); + final var updatedMetrics = + new AgentMetrics( + 1, AgentMetrics.TokenUsage.builder().inputTokenCount(10).outputTokenCount(20).build()); final var initialContext = AgentContext.empty(); final var updatedContext = initialContext.withMetrics(updatedMetrics); @@ -137,7 +139,13 @@ void throwsExceptionOnInvalidConstructorParameters( void canBeSerializedAndDeserialized() throws JsonProcessingException { final var agentContext = AgentContext.builder() - .metrics(new AgentMetrics(1, new AgentMetrics.TokenUsage(10, 20))) + .metrics( + new AgentMetrics( + 1, + AgentMetrics.TokenUsage.builder() + .inputTokenCount(10) + .outputTokenCount(20) + .build())) .toolDefinitions( List.of( ToolDefinition.builder() diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/AgentMetricsTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/AgentMetricsTest.java index 993adcd5731..7e0e57535be 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/AgentMetricsTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/AgentMetricsTest.java @@ -20,6 +20,10 @@ class AgentMetricsTest { private static final AgentMetrics EMPTY_METRICS = AgentMetrics.empty(); + private static TokenUsage tokenUsage(int input, int output) { + return TokenUsage.builder().inputTokenCount(input).outputTokenCount(output).build(); + } + @Test void emptyMetrics() { final var metrics = AgentMetrics.empty(); @@ -53,27 +57,56 @@ void incrementModelCalls() { @Test void withTokenUsage() { final var initialMetrics = AgentMetrics.empty(); - final var updatedMetrics = initialMetrics.withTokenUsage(new TokenUsage(10, 20)); + final var updatedMetrics = initialMetrics.withTokenUsage(tokenUsage(10, 20)); assertThat(updatedMetrics).isNotEqualTo(initialMetrics); assertThat(initialMetrics.tokenUsage()).isEqualTo(EMPTY_METRICS.tokenUsage()); - assertThat(updatedMetrics.tokenUsage()).isEqualTo(new TokenUsage(10, 20)); + assertThat(updatedMetrics.tokenUsage()).isEqualTo(tokenUsage(10, 20)); assertThat(updatedMetrics.tokenUsage().totalTokenCount()).isEqualTo(30); } @Test void incrementTokenUsage() { - final var initialMetrics = AgentMetrics.empty().withTokenUsage(new TokenUsage(10, 20)); - final var updatedMetrics = initialMetrics.incrementTokenUsage(new TokenUsage(1, 2)); + final var initialMetrics = AgentMetrics.empty().withTokenUsage(tokenUsage(10, 20)); + final var updatedMetrics = initialMetrics.incrementTokenUsage(tokenUsage(1, 2)); assertThat(updatedMetrics).isNotEqualTo(initialMetrics); - assertThat(initialMetrics.tokenUsage()).isEqualTo(new TokenUsage(10, 20)); + assertThat(initialMetrics.tokenUsage()).isEqualTo(tokenUsage(10, 20)); - assertThat(updatedMetrics.tokenUsage()).isEqualTo(new TokenUsage(11, 22)); + assertThat(updatedMetrics.tokenUsage()).isEqualTo(tokenUsage(11, 22)); assertThat(updatedMetrics.tokenUsage().totalTokenCount()).isEqualTo(33); } + @Test + void tokenUsage_addRollsUpAllFields() { + final var first = + TokenUsage.builder() + .inputTokenCount(10) + .outputTokenCount(20) + .cacheReadInputTokenCount(3) + .cacheCreationInputTokenCount(5) + .reasoningTokenCount(2) + .build(); + final var second = + TokenUsage.builder() + .inputTokenCount(1) + .outputTokenCount(2) + .cacheReadInputTokenCount(4) + .cacheCreationInputTokenCount(6) + .reasoningTokenCount(7) + .build(); + + final var result = first.add(second); + + assertThat(result.inputTokenCount()).isEqualTo(11); + assertThat(result.outputTokenCount()).isEqualTo(22); + assertThat(result.cacheReadInputTokenCount()).isEqualTo(7); + assertThat(result.cacheCreationInputTokenCount()).isEqualTo(11); + assertThat(result.reasoningTokenCount()).isEqualTo(9); + assertThat(result.totalTokenCount()).isEqualTo(33); + } + @ParameterizedTest @MethodSource("invalidConstructorParameters") void throwsExceptionOnInvalidConstructorParameters( From 35fea0585ddefcb0877ed3c8b7dd7464399c3b78 Mon Sep 17 00:00:00 2001 From: Dmitri Nikonov Date: Thu, 7 May 2026 07:11:18 +0200 Subject: [PATCH 43/81] feat: add blank spi without logic as preparation --- connectors/agentic-ai/AI_AGENT.md | 17 ++-- .../aiagent/framework/api/CacheRetention.java | 22 +++++ .../aiagent/framework/api/ChatClient.java | 32 +++++++ .../aiagent/framework/api/ChatModelApi.java | 28 ++++++ .../framework/api/ChatModelApiFactory.java | 31 +++++++ .../framework/api/ChatModelApiRegistry.java | 24 +++++ .../aiagent/framework/api/ChatOptions.java | 24 +++++ .../aiagent/framework/api/ChatRequest.java | 24 +++++ .../aiagent/framework/api/ChatResponse.java | 27 ++++++ .../framework/api/ChatStreamListener.java | 25 +++++ .../framework/api/ModelCapabilities.java | 38 ++++++++ .../framework/api/ReasoningConfig.java | 35 +++++++ .../framework/api/event/ChatModelEvent.java | 68 ++++++++++++++ .../langchain4j/ChatMessageConverterImpl.java | 56 +---------- ...icAiLangchain4JFrameworkConfiguration.java | 6 +- .../aiagent/model/AgentResponse.java | 19 ++-- .../langchain4j/ChatMessageConverterTest.java | 93 ++++--------------- 17 files changed, 413 insertions(+), 156 deletions(-) create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/CacheRetention.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClient.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApi.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatResponse.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatStreamListener.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ReasoningConfig.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/event/ChatModelEvent.java diff --git a/connectors/agentic-ai/AI_AGENT.md b/connectors/agentic-ai/AI_AGENT.md index 4e030e21a37..4a58b3ad994 100644 --- a/connectors/agentic-ai/AI_AGENT.md +++ b/connectors/agentic-ai/AI_AGENT.md @@ -140,20 +140,19 @@ leading to the following result }, "responseMessage" : { "role" : "assistant", + "apiId" : "chatcmpl-123", "content" : [ { "type" : "text", "text" : "This is a sample response text from the AI agent." } ], "metadata" : { - "framework" : { - "finishReason" : "STOP", - "id" : "chatcmpl-123", - "tokenUsage" : { - "inputTokenCount" : 5, - "outputTokenCount" : 6, - "totalTokenCount" : 11 - } - } + "timestamp" : "2025-01-15T10:30:00Z" + }, + "modelId" : "gpt-5", + "stopReason" : "STOP", + "usage" : { + "inputTokenCount" : 5, + "outputTokenCount" : 6 } }, "toolCalls" : [ ] diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/CacheRetention.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/CacheRetention.java new file mode 100644 index 00000000000..01b3285b9ea --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/CacheRetention.java @@ -0,0 +1,22 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.api; + +/** + * Prompt-cache retention preference passed via {@link ChatOptions}. + * + *

    {@code SHORT} maps to each provider's default ephemeral retention, {@code LONG} to extended + * retention; {@code NONE} strips cache markers entirely. Concrete breakpoint placement is + * implementation-specific. + * + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + */ +public enum CacheRetention { + NONE, + SHORT, + LONG +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClient.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClient.java new file mode 100644 index 00000000000..1766b4a3b88 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClient.java @@ -0,0 +1,32 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.api; + +import io.camunda.connector.agenticai.aiagent.memory.runtime.RuntimeMemory; +import io.camunda.connector.agenticai.aiagent.model.AgentContext; +import io.camunda.connector.agenticai.aiagent.model.AgentExecutionContext; +import java.util.concurrent.CompletableFuture; + +/** + * High-level facade called by {@code BaseAgentRequestHandler} once Phase 1 lands. Resolves the + * model and capabilities via {@link ChatModelApiRegistry}, applies the tool-result strategy, + * assembles the {@link ChatRequest} from the runtime memory, and dispatches to {@link + * ChatModelApi#complete}. + * + *

    Replaces today's {@code AiFrameworkAdapter} call site. Mirroring its signature keeps the + * cutover diff small. + * + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + */ +public interface ChatClient { + + CompletableFuture chat( + AgentExecutionContext executionContext, + AgentContext agentContext, + RuntimeMemory runtimeMemory, + ChatStreamListener listener); +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApi.java new file mode 100644 index 00000000000..79787938ef9 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApi.java @@ -0,0 +1,28 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.api; + +import java.util.concurrent.CompletableFuture; + +/** + * Per-job model client with the resolved provider configuration baked in. Created on demand by a + * {@link ChatModelApiFactory} and reused across capability lookups, tool-result strategy + * application, and the {@link #complete} call within a single request. + * + *

    Implementations drive the underlying SDK's streaming endpoint internally and expose a blocking + * {@link CompletableFuture} surface to callers. The optional {@link ChatStreamListener} receives + * discriminated stream events for in-process observability. + * + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + */ +public interface ChatModelApi { + + ModelCapabilities capabilities(); + + CompletableFuture complete( + ChatRequest request, ChatOptions options, ChatStreamListener listener); +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java new file mode 100644 index 00000000000..9bfd55c6173 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java @@ -0,0 +1,31 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.api; + +import io.camunda.connector.agenticai.aiagent.model.request.provider.ProviderConfiguration; + +/** + * Stateless factory that produces per-job {@link ChatModelApi} instances for a single wire-protocol + * family ({@code anthropic-messages}, {@code bedrock-converse}, {@code openai-responses}, {@code + * openai-completions}, {@code google-genai}). + * + *

    One factory bean per family. The {@link ChatModelApiRegistry} routes a {@link + * ProviderConfiguration} to the correct factory at request time using {@link #apiFamily} and {@link + * #configurationType} as discriminators. + * + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + * + * @param the {@link ProviderConfiguration} subtype this factory handles + */ +public interface ChatModelApiFactory { + + String apiFamily(); + + Class configurationType(); + + ChatModelApi create(C configuration); +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java new file mode 100644 index 00000000000..c9f487b79bf --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java @@ -0,0 +1,24 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.api; + +import io.camunda.connector.agenticai.aiagent.model.request.provider.ProviderConfiguration; + +/** + * Resolves a {@link ChatModelApi} for a given {@link ProviderConfiguration}. Composed of all + * registered {@link ChatModelApiFactory} beans; factory selection is driven by the configuration's + * discriminator. + * + *

    Unknown api families fail fast at validation rather than runtime — there is no factory bean to + * resolve, so requests can never start. + * + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + */ +public interface ChatModelApiRegistry { + + ChatModelApi resolve(ProviderConfiguration configuration); +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java new file mode 100644 index 00000000000..23a4c1e6045 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java @@ -0,0 +1,24 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.api; + +import java.util.Map; +import org.springframework.lang.Nullable; + +/** + * Per-call tunables passed alongside a {@link ChatRequest}. {@code reasoning} and {@code + * cacheRetention} are normalised across providers; {@code providerOptions} is the typed escape + * hatch for vendor-specific knobs (Anthropic beta flags, Bedrock guardrail config, etc.). + * + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + */ +public record ChatOptions( + @Nullable Integer maxOutputTokens, + @Nullable Double temperature, + @Nullable ReasoningConfig reasoning, + @Nullable CacheRetention cacheRetention, + Map providerOptions) {} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java new file mode 100644 index 00000000000..ececa78836a --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java @@ -0,0 +1,24 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.api; + +import io.camunda.connector.agenticai.model.message.Message; +import io.camunda.connector.agenticai.model.tool.ToolDefinition; +import java.util.List; +import org.springframework.lang.Nullable; + +/** + * Inputs to a {@link ChatModelApi#complete} call assembled by {@code ChatClient}. Carries the + * conversation messages, an optional system prompt, and the resolved tool definitions; per-call + * tunables (max tokens, temperature, reasoning, cache retention, vendor escape hatches) live on + * {@link ChatOptions} so a request can be reused while options vary. + * + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime — concrete fields + * will be finalised when the first native {@code ChatModelApi} implementation lands. + */ +public record ChatRequest( + List messages, @Nullable String systemPrompt, List tools) {} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatResponse.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatResponse.java new file mode 100644 index 00000000000..af6388bd043 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatResponse.java @@ -0,0 +1,27 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.api; + +import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; +import io.camunda.connector.agenticai.model.message.AssistantMessage; +import io.camunda.connector.agenticai.model.message.StopReason; +import org.springframework.lang.Nullable; + +/** + * Output of {@link ChatModelApi#complete}. Carries the assembled assistant message plus a + * normalised {@link StopReason} and per-call {@link AgentMetrics.TokenUsage}. {@code errorMessage} + * is populated only for model-side terminal failures (refusal, content filter, malformed tool-use) + * where {@code stopReason == ERROR}; transport / SDK / auth failures complete the future + * exceptionally instead. + * + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + */ +public record ChatResponse( + AssistantMessage assistantMessage, + @Nullable StopReason stopReason, + @Nullable AgentMetrics.TokenUsage usage, + @Nullable String errorMessage) {} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatStreamListener.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatStreamListener.java new file mode 100644 index 00000000000..bab42b4c654 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatStreamListener.java @@ -0,0 +1,25 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.api; + +import io.camunda.connector.agenticai.aiagent.framework.api.event.ChatModelEvent; + +/** + * In-process observability hook invoked for every {@link ChatModelEvent} produced while a {@link + * ChatModelApi} drives a provider's streaming endpoint. Listeners default to {@link #NOOP}; the + * listener is intentionally not exposed as a reactive type — the public chat surface remains a + * blocking {@code CompletableFuture}. + * + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + */ +@FunctionalInterface +public interface ChatStreamListener { + + ChatStreamListener NOOP = event -> {}; + + void onEvent(ChatModelEvent event); +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java new file mode 100644 index 00000000000..2b091b2ded2 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java @@ -0,0 +1,38 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.api; + +import java.util.List; +import org.springframework.lang.Nullable; + +/** + * Per-model capability descriptor materialised from the capability matrix YAML (or a connector + * config override). Drives runtime decisions like tool-result strategy selection, reasoning + * negotiation, and cache-marker placement. The vocabulary for {@link Modality} is fixed; modality + * lists per location are symmetric so every location has an explicit answer. + * + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + */ +public record ModelCapabilities( + List userMessageModalities, + List toolResultModalities, + List assistantMessageModalities, + boolean supportsReasoning, + boolean supportsReasoningSignatureRoundtrip, + boolean supportsPromptCaching, + boolean supportsParallelToolCalls, + @Nullable Integer contextWindow, + @Nullable Integer maxOutputTokens) { + + public enum Modality { + TEXT, + IMAGE, + PDF, + AUDIO, + VIDEO + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ReasoningConfig.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ReasoningConfig.java new file mode 100644 index 00000000000..f7736e09fae --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ReasoningConfig.java @@ -0,0 +1,35 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.api; + +/** + * Two-tier reasoning configuration: high-level {@link Effort} or explicit token {@link + * ReasoningBudget}, with {@link ReasoningDisabled} to opt out. Per-implementation translation maps + * this onto provider-native fields (Anthropic adaptive effort / thinking budget, OpenAI Responses + * reasoning effort, Gemini thinking budget, etc.). + * + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + */ +public sealed interface ReasoningConfig + permits ReasoningConfig.ReasoningEffort, + ReasoningConfig.ReasoningBudget, + ReasoningConfig.ReasoningDisabled { + + enum Effort { + MINIMAL, + LOW, + MEDIUM, + HIGH, + X_HIGH + } + + record ReasoningEffort(Effort level) implements ReasoningConfig {} + + record ReasoningBudget(int tokens) implements ReasoningConfig {} + + record ReasoningDisabled() implements ReasoningConfig {} +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/event/ChatModelEvent.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/event/ChatModelEvent.java new file mode 100644 index 00000000000..66fbe781cab --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/event/ChatModelEvent.java @@ -0,0 +1,68 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.api.event; + +import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; +import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; +import org.springframework.lang.Nullable; + +/** + * Discriminated stream events emitted by a {@code ChatModelApi} implementation while driving a + * provider's streaming endpoint. Listeners receive every event in order; {@link DoneEvent} carries + * the assembled {@link ChatResponse}, while {@link ErrorEvent} carries the error message plus any + * partial content / usage accumulated before the failure. + * + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + */ +public sealed interface ChatModelEvent + permits ChatModelEvent.StartEvent, + ChatModelEvent.TextStartEvent, + ChatModelEvent.TextDeltaEvent, + ChatModelEvent.TextEndEvent, + ChatModelEvent.ReasoningStartEvent, + ChatModelEvent.ReasoningDeltaEvent, + ChatModelEvent.ReasoningEndEvent, + ChatModelEvent.ToolCallStartEvent, + ChatModelEvent.ToolCallArgumentsDeltaEvent, + ChatModelEvent.ToolCallEndEvent, + ChatModelEvent.UsageEvent, + ChatModelEvent.DoneEvent, + ChatModelEvent.ErrorEvent { + + record StartEvent(@Nullable String modelId, @Nullable String apiId) implements ChatModelEvent {} + + record TextStartEvent(int blockIndex) implements ChatModelEvent {} + + record TextDeltaEvent(int blockIndex, String delta) implements ChatModelEvent {} + + record TextEndEvent(int blockIndex) implements ChatModelEvent {} + + record ReasoningStartEvent(int blockIndex) implements ChatModelEvent {} + + record ReasoningDeltaEvent(int blockIndex, String delta, @Nullable String signatureDelta) + implements ChatModelEvent {} + + record ReasoningEndEvent(int blockIndex) implements ChatModelEvent {} + + record ToolCallStartEvent(int blockIndex, String toolCallId, String toolName) + implements ChatModelEvent {} + + record ToolCallArgumentsDeltaEvent(int blockIndex, String argumentsDelta) + implements ChatModelEvent {} + + record ToolCallEndEvent(int blockIndex) implements ChatModelEvent {} + + record UsageEvent(AgentMetrics.TokenUsage usage) implements ChatModelEvent {} + + record DoneEvent(ChatResponse response) implements ChatModelEvent {} + + record ErrorEvent( + String errorMessage, + @Nullable ChatResponse partialResponse, + @Nullable AgentMetrics.TokenUsage partialUsage) + implements ChatModelEvent {} +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterImpl.java index 5e149d9c6af..108fd8c4e9d 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterImpl.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterImpl.java @@ -9,10 +9,8 @@ import static io.camunda.connector.agenticai.util.JacksonExceptionMessageExtractor.humanReadableJsonProcessingExceptionMessage; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.data.message.ToolExecutionResultMessage; -import dev.langchain4j.internal.Json; import dev.langchain4j.model.anthropic.AnthropicTokenUsage; import dev.langchain4j.model.bedrock.BedrockTokenUsage; import dev.langchain4j.model.chat.response.ChatResponse; @@ -30,33 +28,23 @@ import io.camunda.connector.agenticai.model.message.UserMessage; import io.camunda.connector.agenticai.model.message.content.Content; import io.camunda.connector.agenticai.model.message.content.TextContent; -import io.camunda.connector.agenticai.util.ObjectMapperConstants; import io.camunda.connector.api.error.ConnectorException; import java.time.ZonedDateTime; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.util.CollectionUtils; public class ChatMessageConverterImpl implements ChatMessageConverter { - private static final Logger LOGGER = LoggerFactory.getLogger(ChatMessageConverterImpl.class); - private final ContentConverter contentConverter; private final ToolCallConverter toolCallConverter; - private final ObjectMapper objectMapper; public ChatMessageConverterImpl( - ContentConverter contentConverter, - ToolCallConverter toolCallConverter, - ObjectMapper objectMapper) { + ContentConverter contentConverter, ToolCallConverter toolCallConverter) { this.contentConverter = contentConverter; this.toolCallConverter = toolCallConverter; - this.objectMapper = objectMapper; } @Override @@ -140,10 +128,7 @@ protected AssistantMessageBuilder toAssistantMessageBuilder(ChatResponse chatRes final ChatResponseMetadata metadata = chatResponse.metadata(); if (metadata != null) { - builder.metadata( - Map.of( - "timestamp", ZonedDateTime.now(), - "framework", serializedChatResponseMetadata(metadata))); + builder.metadata(Map.of("timestamp", ZonedDateTime.now())); Optional.ofNullable(metadata.modelName()) .filter(StringUtils::isNotBlank) @@ -213,43 +198,6 @@ AgentMetrics.TokenUsage toDomainTokenUsage(TokenUsage tokenUsage) { return builder.build(); } - protected Map serializedChatResponseMetadata( - ChatResponseMetadata chatResponseMetadata) { - if (chatResponseMetadata == null) { - return Map.of(); - } - - final var metadata = new LinkedHashMap(); - Optional.ofNullable(chatResponseMetadata.id()) - .filter(StringUtils::isNotBlank) - .ifPresent(id -> metadata.put("id", id)); - Optional.ofNullable(chatResponseMetadata.finishReason()) - .ifPresent(finishReason -> metadata.put("finishReason", finishReason.name())); - - final var tokenUsage = serializedTokenUsage(chatResponseMetadata.tokenUsage()); - if (!tokenUsage.isEmpty()) { - metadata.put("tokenUsage", tokenUsage); - } - - return metadata; - } - - protected Map serializedTokenUsage(TokenUsage tokenUsage) { - if (tokenUsage == null) { - return Map.of(); - } - - try { - return objectMapper.readValue( - Json.toJson(tokenUsage), ObjectMapperConstants.STRING_OBJECT_MAP_TYPE_REFERENCE); - } catch (JsonProcessingException e) { - LOGGER.warn( - "Failed to deserialize token usage metadata: {}", - humanReadableJsonProcessingExceptionMessage(e)); - return Map.of(); - } - } - @Override public List fromToolCallResultMessage( ToolCallResultMessage toolCallResultMessage) { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java index 27924b197f4..ae93ad14f2a 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java @@ -73,10 +73,8 @@ public ToolSpecificationConverter langchain4JToolSpecificationConverter( @Bean @ConditionalOnMissingBean public ChatMessageConverter langchain4JChatMessageConverter( - ContentConverter contentConverter, - ToolCallConverter toolCallConverter, - @ConnectorsObjectMapper ObjectMapper objectMapper) { - return new ChatMessageConverterImpl(contentConverter, toolCallConverter, objectMapper); + ContentConverter contentConverter, ToolCallConverter toolCallConverter) { + return new ChatMessageConverterImpl(contentConverter, toolCallConverter); } @Bean diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/AgentResponse.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/AgentResponse.java index a85b6abe5b1..ae8fc94cf9b 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/AgentResponse.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/AgentResponse.java @@ -12,10 +12,12 @@ import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import io.camunda.connector.agenticai.model.AgenticAiRecord; import io.camunda.connector.agenticai.model.message.AssistantMessage; +import io.camunda.connector.agenticai.model.message.StopReason; import io.camunda.connector.agenticai.model.tool.ToolCall; import io.camunda.connector.agenticai.model.tool.ToolCallProcessVariable; import io.camunda.connector.agenticai.model.tool.ToolDefinition; import io.camunda.connector.generator.java.annotation.DataExample; +import java.time.ZonedDateTime; import java.util.List; import java.util.Map; import org.springframework.lang.Nullable; @@ -87,18 +89,11 @@ public static AgentResponse exampleResultWithAssistantMessageResponse() { final var assistantMessage = AssistantMessage.builder() .content(singleTextContent("This is a sample response text from the AI agent.")) - .metadata( - Map.of( - "framework", - Map.ofEntries( - Map.entry("id", "chatcmpl-123"), - Map.entry("finishReason", "STOP"), - Map.entry( - "tokenUsage", - Map.of( - "inputTokenCount", 5, - "outputTokenCount", 6, - "totalTokenCount", 11))))) + .modelId("gpt-5") + .apiId("chatcmpl-123") + .stopReason(StopReason.STOP) + .usage(AgentMetrics.TokenUsage.builder().inputTokenCount(5).outputTokenCount(6).build()) + .metadata(Map.of("timestamp", ZonedDateTime.parse("2025-01-15T10:30:00Z"))) .build(); return AgentResponse.builder() diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterTest.java index 9332db9f6d7..5c7a4998eb0 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterTest.java @@ -9,7 +9,6 @@ import static io.camunda.connector.agenticai.model.message.content.TextContent.textContent; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.assertj.core.api.Assertions.entry; import static org.assertj.core.api.Assertions.within; import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.Mockito.doReturn; @@ -17,18 +16,15 @@ import static org.mockito.Mockito.when; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import dev.langchain4j.agent.tool.ToolExecutionRequest; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.data.message.ChatMessage; import dev.langchain4j.data.message.Content; import dev.langchain4j.data.message.ToolExecutionResultMessage; -import dev.langchain4j.http.client.SuccessfulHttpResponse; import dev.langchain4j.model.anthropic.AnthropicTokenUsage; import dev.langchain4j.model.bedrock.BedrockTokenUsage; import dev.langchain4j.model.chat.response.ChatResponse; import dev.langchain4j.model.chat.response.ChatResponseMetadata; -import dev.langchain4j.model.openai.OpenAiChatResponseMetadata; import dev.langchain4j.model.openai.OpenAiTokenUsage; import dev.langchain4j.model.output.FinishReason; import dev.langchain4j.model.output.TokenUsage; @@ -48,11 +44,9 @@ import java.time.ZonedDateTime; import java.time.temporal.ChronoUnit; import java.util.Collections; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.stream.Stream; -import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; @@ -60,7 +54,6 @@ import org.junit.jupiter.params.provider.MethodSource; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) @@ -68,7 +61,6 @@ class ChatMessageConverterTest { @Mock private ToolCallConverter toolCallConverter; @Mock private ContentConverter contentConverter; - @Spy private ObjectMapper objectMapper = new ObjectMapper(); @InjectMocks private ChatMessageConverterImpl chatMessageConverter; @@ -235,64 +227,15 @@ void toAssistantMessage_convertsFromChatResponse() { .isEqualTo("AI response"); }); - assertThat(result.metadata()).containsKey("timestamp"); + assertThat(result.metadata()).containsOnlyKeys("timestamp"); assertThat((ZonedDateTime) result.metadata().get("timestamp")) .isCloseTo(ZonedDateTime.now(), within(1, ChronoUnit.SECONDS)); - assertThat(result.metadata()).containsKey("framework"); - assertThat(result.metadata().get("framework")) - .asInstanceOf(InstanceOfAssertFactories.MAP) - .containsExactly( - entry("id", "chatcmpl-123"), - entry("finishReason", "STOP"), - entry( - "tokenUsage", - Map.of("inputTokenCount", 10, "outputTokenCount", 20, "totalTokenCount", 30))); - } - - @Test - void toAssistantMessage_containsOnlyBasicMetadata() { - final var aiMessage = AiMessage.builder().text("AI response").build(); - - final var chatResponseMetadata = - OpenAiChatResponseMetadata.builder() - .id("chatcmpl-123") - .finishReason(FinishReason.TOOL_EXECUTION) - .tokenUsage( - OpenAiTokenUsage.builder() - .inputTokenCount(10) - .inputTokensDetails( - OpenAiTokenUsage.InputTokensDetails.builder().cachedTokens(1).build()) - .outputTokenCount(20) - .totalTokenCount(30) - .build()) - .serviceTier("super-premium") - .rawHttpResponse( - SuccessfulHttpResponse.builder() - .statusCode(200) - .headers(Map.of("x-my-header", List.of("dummy"))) - .body("AI response") - .build()) - .build(); - - final var chatResponse = - new ChatResponse.Builder().aiMessage(aiMessage).metadata(chatResponseMetadata).build(); - final var result = chatMessageConverter.toAssistantMessage(chatResponse); - - final var expectedTokenUsage = new LinkedHashMap(); - expectedTokenUsage.put("inputTokenCount", 10); - expectedTokenUsage.put("inputTokensDetails", Map.of("cachedTokens", 1)); - expectedTokenUsage.put("outputTokenCount", 20); - expectedTokenUsage.put("outputTokensDetails", null); - expectedTokenUsage.put("totalTokenCount", 30); - - assertThat(result.metadata().get("framework")) - .asInstanceOf(InstanceOfAssertFactories.MAP) - .containsExactly( - entry("id", "chatcmpl-123"), - entry("finishReason", "TOOL_EXECUTION"), - entry("tokenUsage", expectedTokenUsage)) - .doesNotContainKeys("serviceTier", "rawHttpResponse"); + assertThat(result.apiId()).isEqualTo("chatcmpl-123"); + assertThat(result.stopReason()).isEqualTo(StopReason.STOP); + assertThat(result.usage()) + .isEqualTo( + AgentMetrics.TokenUsage.builder().inputTokenCount(10).outputTokenCount(20).build()); } @Test @@ -313,15 +256,12 @@ void toAssistantMessage_convertsFromChatResponse_withoutContentText() { assertThat(result.content()).isEmpty(); - assertThat(result.metadata()).containsKey("framework"); - assertThat(result.metadata().get("framework")) - .asInstanceOf(InstanceOfAssertFactories.MAP) - .containsExactly( - entry("id", "chatcmpl-123"), - entry("finishReason", "CONTENT_FILTER"), - entry( - "tokenUsage", - Map.of("inputTokenCount", 10, "outputTokenCount", 0, "totalTokenCount", 10))); + assertThat(result.metadata()).containsOnlyKeys("timestamp"); + assertThat(result.apiId()).isEqualTo("chatcmpl-123"); + assertThat(result.stopReason()).isEqualTo(StopReason.CONTENT_FILTERED); + assertThat(result.usage()) + .isEqualTo( + AgentMetrics.TokenUsage.builder().inputTokenCount(10).outputTokenCount(0).build()); } @Test @@ -342,11 +282,10 @@ void toAssistantMessage_convertsFromChatResponse_withoutMetadata() { final var result = chatMessageConverter.toAssistantMessage(chatResponse); - assertThat(result.metadata()).containsKey("framework"); - assertThat(result.metadata().get("framework")) - .isNotNull() - .asInstanceOf(InstanceOfAssertFactories.MAP) - .isEmpty(); + assertThat(result.metadata()).containsOnlyKeys("timestamp"); + assertThat(result.apiId()).isNull(); + assertThat(result.stopReason()).isNull(); + assertThat(result.usage()).isNull(); } @Test From aa13533232fc4dcd07cdcbf00eac3c824ebbf7c4 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 10:14:43 +0200 Subject: [PATCH 44/81] fix: add canonical constructor to AgentMetrics.TokenUsage --- .../connector/agenticai/aiagent/model/AgentMetrics.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/AgentMetrics.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/AgentMetrics.java index 88db5688d6e..d73741b7d1a 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/AgentMetrics.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/AgentMetrics.java @@ -56,6 +56,10 @@ public record TokenUsage( @JsonInclude(JsonInclude.Include.NON_DEFAULT) int reasoningTokenCount) implements AgentMetricsTokenUsageBuilder.With { + public TokenUsage(int inputTokenCount, int outputTokenCount) { + this(inputTokenCount, outputTokenCount, 0, 0, 0); + } + public int totalTokenCount() { return inputTokenCount + outputTokenCount; } From 98e0011431d8678231075ea66a1bd97b11bb5c83 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 11:09:52 +0200 Subject: [PATCH 45/81] refactor(agentic-ai): tighten naming and ChatOptions surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Reorder AssistantMessage fields so identity (modelId, messageId, stopReason, usage) precedes payload (content, toolCalls, metadata). * Rename AssistantMessage.apiId to messageId — the value is the provider-issued response message id, not a wire-protocol family identifier. * Rename ChatModelEvent.StartEvent.apiId to apiFamily for consistency with ChatModelApiFactory.apiFamily() and the YAML matrix grouping; reorder to (apiFamily, modelId). * Drop temperature from ChatOptions — it is not universally supported (reasoning models reject it). Move it to the providerOptions escape hatch alongside topP, topK, seed, etc. * Document the maxOutputTokens vs reasoning-token interaction on the Javadoc. --- connectors/agentic-ai/AI_AGENT.md | 2 +- .../aiagent/framework/api/ChatOptions.java | 21 +++++++++++++++---- .../aiagent/framework/api/ChatRequest.java | 4 ++-- .../framework/api/event/ChatModelEvent.java | 3 ++- .../langchain4j/ChatMessageConverterImpl.java | 4 +++- .../aiagent/model/AgentResponse.java | 2 +- .../model/message/AssistantMessage.java | 6 +++--- .../langchain4j/ChatMessageConverterTest.java | 8 +++---- 8 files changed, 33 insertions(+), 17 deletions(-) diff --git a/connectors/agentic-ai/AI_AGENT.md b/connectors/agentic-ai/AI_AGENT.md index 4a58b3ad994..cae54ae7af1 100644 --- a/connectors/agentic-ai/AI_AGENT.md +++ b/connectors/agentic-ai/AI_AGENT.md @@ -140,11 +140,11 @@ leading to the following result }, "responseMessage" : { "role" : "assistant", - "apiId" : "chatcmpl-123", "content" : [ { "type" : "text", "text" : "This is a sample response text from the AI agent." } ], + "messageId" : "chatcmpl-123", "metadata" : { "timestamp" : "2025-01-15T10:30:00Z" }, diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java index 23a4c1e6045..5cdff0724d1 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java @@ -10,15 +10,28 @@ import org.springframework.lang.Nullable; /** - * Per-call tunables passed alongside a {@link ChatRequest}. {@code reasoning} and {@code - * cacheRetention} are normalised across providers; {@code providerOptions} is the typed escape - * hatch for vendor-specific knobs (Anthropic beta flags, Bedrock guardrail config, etc.). + * Per-call tunables passed alongside a {@link ChatRequest}. The shape is intentionally narrow: + * fields here are the ones with consistent semantics across every wire-protocol family we + * implement. Sampling parameters that are partially supported (e.g. {@code temperature}, {@code + * topP}, {@code topK}, {@code seed}, {@code frequencyPenalty}) live in {@link #providerOptions} + * under their well-known keys; each {@link ChatModelApi} implementation reads the keys it cares + * about and ignores the rest. + * + *

    Note on {@link #maxOutputTokens} and reasoning tokens: on most providers + * (OpenAI Chat Completions, OpenAI Responses, Google GenAI, AWS Bedrock) reasoning / thinking + * tokens count toward this cap — a small value combined with reasoning enabled can leave zero room + * for visible output. On Anthropic Claude 4+ thinking tokens do not affect the {@code max_tokens} + * cap but are still billed in full. Implementations document any provider-specific clamping or + * adjustment they apply. + * + *

    Resolution of the actual value sent on the wire happens in {@code ChatClient}: explicit + * caller-supplied value wins, otherwise the resolved {@link ModelCapabilities#maxOutputTokens()} is + * used as a fallback, otherwise the implementation supplies its own per-API default. * *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. */ public record ChatOptions( @Nullable Integer maxOutputTokens, - @Nullable Double temperature, @Nullable ReasoningConfig reasoning, @Nullable CacheRetention cacheRetention, Map providerOptions) {} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java index ececa78836a..ac7dcd48698 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java @@ -14,8 +14,8 @@ /** * Inputs to a {@link ChatModelApi#complete} call assembled by {@code ChatClient}. Carries the * conversation messages, an optional system prompt, and the resolved tool definitions; per-call - * tunables (max tokens, temperature, reasoning, cache retention, vendor escape hatches) live on - * {@link ChatOptions} so a request can be reused while options vary. + * tunables (max output tokens, stop sequences, reasoning, cache retention, vendor escape hatches) + * live on {@link ChatOptions} so a request can be reused while options vary. * *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime — concrete fields * will be finalised when the first native {@code ChatModelApi} implementation lands. diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/event/ChatModelEvent.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/event/ChatModelEvent.java index 66fbe781cab..82db2b1476b 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/event/ChatModelEvent.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/event/ChatModelEvent.java @@ -33,7 +33,8 @@ public sealed interface ChatModelEvent ChatModelEvent.DoneEvent, ChatModelEvent.ErrorEvent { - record StartEvent(@Nullable String modelId, @Nullable String apiId) implements ChatModelEvent {} + record StartEvent(@Nullable String apiFamily, @Nullable String modelId) + implements ChatModelEvent {} record TextStartEvent(int blockIndex) implements ChatModelEvent {} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterImpl.java index 108fd8c4e9d..7f57aafc6fd 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterImpl.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterImpl.java @@ -133,7 +133,9 @@ protected AssistantMessageBuilder toAssistantMessageBuilder(ChatResponse chatRes Optional.ofNullable(metadata.modelName()) .filter(StringUtils::isNotBlank) .ifPresent(builder::modelId); - Optional.ofNullable(metadata.id()).filter(StringUtils::isNotBlank).ifPresent(builder::apiId); + Optional.ofNullable(metadata.id()) + .filter(StringUtils::isNotBlank) + .ifPresent(builder::messageId); Optional.ofNullable(metadata.finishReason()) .map(this::toStopReason) .ifPresent(builder::stopReason); diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/AgentResponse.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/AgentResponse.java index ae8fc94cf9b..d2405e6bd26 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/AgentResponse.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/AgentResponse.java @@ -90,7 +90,7 @@ public static AgentResponse exampleResultWithAssistantMessageResponse() { AssistantMessage.builder() .content(singleTextContent("This is a sample response text from the AI agent.")) .modelId("gpt-5") - .apiId("chatcmpl-123") + .messageId("chatcmpl-123") .stopReason(StopReason.STOP) .usage(AgentMetrics.TokenUsage.builder().inputTokenCount(5).outputTokenCount(6).build()) .metadata(Map.of("timestamp", ZonedDateTime.parse("2025-01-15T10:30:00Z"))) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/AssistantMessage.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/AssistantMessage.java index 0f578259e71..679d40c050a 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/AssistantMessage.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/model/message/AssistantMessage.java @@ -20,12 +20,12 @@ @AgenticAiRecord @JsonDeserialize(builder = AssistantMessage.AssistantMessageJacksonProxyBuilder.class) public record AssistantMessage( - @JsonInclude(JsonInclude.Include.NON_EMPTY) List content, - @JsonInclude(JsonInclude.Include.NON_EMPTY) List toolCalls, @Nullable String modelId, - @Nullable String apiId, + @Nullable String messageId, @Nullable StopReason stopReason, @Nullable AgentMetrics.TokenUsage usage, + @JsonInclude(JsonInclude.Include.NON_EMPTY) List content, + @JsonInclude(JsonInclude.Include.NON_EMPTY) List toolCalls, @JsonInclude(JsonInclude.Include.NON_EMPTY) Map metadata) implements AssistantMessageBuilder.With, Message, ContentMessage { diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterTest.java index 5c7a4998eb0..510b55b3e65 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatMessageConverterTest.java @@ -231,7 +231,7 @@ void toAssistantMessage_convertsFromChatResponse() { assertThat((ZonedDateTime) result.metadata().get("timestamp")) .isCloseTo(ZonedDateTime.now(), within(1, ChronoUnit.SECONDS)); - assertThat(result.apiId()).isEqualTo("chatcmpl-123"); + assertThat(result.messageId()).isEqualTo("chatcmpl-123"); assertThat(result.stopReason()).isEqualTo(StopReason.STOP); assertThat(result.usage()) .isEqualTo( @@ -257,7 +257,7 @@ void toAssistantMessage_convertsFromChatResponse_withoutContentText() { assertThat(result.content()).isEmpty(); assertThat(result.metadata()).containsOnlyKeys("timestamp"); - assertThat(result.apiId()).isEqualTo("chatcmpl-123"); + assertThat(result.messageId()).isEqualTo("chatcmpl-123"); assertThat(result.stopReason()).isEqualTo(StopReason.CONTENT_FILTERED); assertThat(result.usage()) .isEqualTo( @@ -283,7 +283,7 @@ void toAssistantMessage_convertsFromChatResponse_withoutMetadata() { final var result = chatMessageConverter.toAssistantMessage(chatResponse); assertThat(result.metadata()).containsOnlyKeys("timestamp"); - assertThat(result.apiId()).isNull(); + assertThat(result.messageId()).isNull(); assertThat(result.stopReason()).isNull(); assertThat(result.usage()).isNull(); } @@ -390,7 +390,7 @@ void toAssistantMessage_populatesTypedProvenanceFields() { final var result = chatMessageConverter.toAssistantMessage(chatResponse); - assertThat(result.apiId()).isEqualTo("msg-abc"); + assertThat(result.messageId()).isEqualTo("msg-abc"); assertThat(result.modelId()).isEqualTo("claude-3-7-sonnet"); assertThat(result.stopReason()).isEqualTo(StopReason.STOP); assertThat(result.usage()) From 55886caaa3026a25852266eba52089c8c26172da Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 11:35:26 +0200 Subject: [PATCH 46/81] feat(agentic-ai): add ResponseFormat to ChatOptions SPI Promote response format to a first-class field on the new ChatOptions SPI rather than tunneling it through providerOptions. Mirrors the sealed Text/Json shape of the existing ResponseFormatConfiguration so ChatClient can translate at the boundary; nullable so today's "no explicit format" behaviour stays the default. --- .../aiagent/framework/api/ChatOptions.java | 7 ++++ .../aiagent/framework/api/ResponseFormat.java | 32 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ResponseFormat.java diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java index 5cdff0724d1..9a34dd56b27 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java @@ -28,10 +28,17 @@ * caller-supplied value wins, otherwise the resolved {@link ModelCapabilities#maxOutputTokens()} is * used as a fallback, otherwise the implementation supplies its own per-API default. * + *

    Note on {@link #responseFormat}: {@code null} leaves the provider default in + * place (matches today's behaviour, where no explicit format is sent for the text case). + * Implementations translate the value onto the provider's native shape; providers without a native + * structured-output mode (Anthropic Messages today) treat {@link ResponseFormat.Json} as + * best-effort and rely on the system prompt to constrain output. + * *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. */ public record ChatOptions( @Nullable Integer maxOutputTokens, @Nullable ReasoningConfig reasoning, @Nullable CacheRetention cacheRetention, + @Nullable ResponseFormat responseFormat, Map providerOptions) {} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ResponseFormat.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ResponseFormat.java new file mode 100644 index 00000000000..c36576fbe10 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ResponseFormat.java @@ -0,0 +1,32 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.api; + +import java.util.Map; +import org.springframework.lang.Nullable; + +/** + * Universal response-format hint passed via {@link ChatOptions#responseFormat()}. {@link Text} + * requests free-form text output; {@link Json} requests JSON, optionally constrained by a JSON + * Schema. {@code null} on {@link ChatOptions} leaves the provider default in place — this matches + * today's behaviour where no explicit format is sent for the text case. + * + *

    The {@link Json#schema()} payload is the JSON Schema wire format ({@code Map}) + * as supplied by the connector configuration; implementations translate it onto the provider's + * native shape (e.g. OpenAI {@code response_format.json_schema}, Google {@code responseSchema}). + * Providers without a native structured-output mode (Anthropic Messages today) treat {@link Json} + * as best-effort and rely on the system prompt to constrain output. + * + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + */ +public sealed interface ResponseFormat permits ResponseFormat.Text, ResponseFormat.Json { + + record Text() implements ResponseFormat {} + + record Json(@Nullable String schemaName, @Nullable Map schema) + implements ResponseFormat {} +} From e7a079e808efae1673558b92c0dceb4c3b0fe7b6 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 11:39:35 +0200 Subject: [PATCH 47/81] refactor(agentic-ai): dispatch chat factories by providerType string Replace ChatModelApiFactory#configurationType with #supportedProviderTypes so the registry can mirror the existing ChatModelProviderRegistry shape: flat Map keyed by ProviderConfiguration#providerType, exact-match dispatch. Avoids class-based assignability lookups for the bridge factory which otherwise would have to claim ProviderConfiguration itself as its type and break exact matching for concrete subclasses. --- .../aiagent/framework/api/ChatModelApiFactory.java | 9 +++++---- .../aiagent/framework/api/ChatModelApiRegistry.java | 10 ++++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java index 9bfd55c6173..1ccb7a226f4 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java @@ -7,15 +7,16 @@ package io.camunda.connector.agenticai.aiagent.framework.api; import io.camunda.connector.agenticai.aiagent.model.request.provider.ProviderConfiguration; +import java.util.Set; /** * Stateless factory that produces per-job {@link ChatModelApi} instances for a single wire-protocol * family ({@code anthropic-messages}, {@code bedrock-converse}, {@code openai-responses}, {@code * openai-completions}, {@code google-genai}). * - *

    One factory bean per family. The {@link ChatModelApiRegistry} routes a {@link - * ProviderConfiguration} to the correct factory at request time using {@link #apiFamily} and {@link - * #configurationType} as discriminators. + *

    The {@link ChatModelApiRegistry} indexes factories by the {@link ProviderConfiguration#type + * providerType} strings each factory claims via {@link #supportedProviderTypes} and dispatches by + * exact match. {@link #apiFamily} is informational — used in logs and stream events. * *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. * @@ -25,7 +26,7 @@ public interface ChatModelApiFactory { String apiFamily(); - Class configurationType(); + Set supportedProviderTypes(); ChatModelApi create(C configuration); } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java index c9f487b79bf..408bccbbe85 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java @@ -10,11 +10,13 @@ /** * Resolves a {@link ChatModelApi} for a given {@link ProviderConfiguration}. Composed of all - * registered {@link ChatModelApiFactory} beans; factory selection is driven by the configuration's - * discriminator. + * registered {@link ChatModelApiFactory} beans, indexed by the {@link ProviderConfiguration#type + * providerType} strings each factory advertises via {@link + * ChatModelApiFactory#supportedProviderTypes}; lookup is exact-match on the configuration's + * provider type. * - *

    Unknown api families fail fast at validation rather than runtime — there is no factory bean to - * resolve, so requests can never start. + *

    Unknown provider types fail fast — there is no factory to resolve, so requests can never + * start. * *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. */ From f21efbb74a25da9f581498f05cc4671ce3370c3f Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 12:32:35 +0200 Subject: [PATCH 48/81] feat(agentic-ai): route chat calls through ChatClient SPI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the new framework SPI end-to-end so every chat call goes through ChatClient → ChatModelApiRegistry → ChatModelApi instead of the legacy AiFrameworkAdapter. Behaviour is unchanged: the only registered factory is the LangChain4j bridge, which absorbs the previous adapter logic and covers all six built-in provider configurations. Highlights: - ChatClientImpl assembles ChatRequest / ChatOptions from the runtime memory, agent context, and ResponseConfiguration, then dispatches via the registry. - ChatModelApiRegistryImpl indexes factories by ProviderConfiguration providerType strings, fail-loud on collision so user overrides go via @ConditionalOnMissingBean rather than silent shadowing. - Langchain4JChatModelApi(Factory) wraps a resolved L4J ChatModel and exposes the bridge logic for reuse by custom factories. - BaseAgentRequestHandler owns model-call metric tracking now that the framework no longer mutates AgentContext. - Old AiFrameworkAdapter / Langchain4JAiFrameworkAdapter and their tests removed; replaced by ChatClientImplTest, ChatModelApiRegistryImplTest, and Langchain4JChatModelApiTest. 1290 unit tests + the wire-format e2e tests stay green. --- .../agent/BaseAgentRequestHandler.java | 44 ++- .../agent/JobWorkerAgentRequestHandler.java | 6 +- .../OutboundConnectorAgentRequestHandler.java | 6 +- .../aiagent/framework/AiFrameworkAdapter.java | 18 - .../framework/AiFrameworkChatResponse.java | 16 - .../aiagent/framework/ChatClientImpl.java | 64 ++++ .../framework/ChatModelApiRegistryImpl.java | 63 ++++ .../aiagent/framework/api/CacheRetention.java | 3 +- .../aiagent/framework/api/ChatClient.java | 3 +- .../aiagent/framework/api/ChatModelApi.java | 3 +- .../framework/api/ChatModelApiFactory.java | 3 +- .../framework/api/ChatModelApiRegistry.java | 3 +- .../aiagent/framework/api/ChatOptions.java | 3 +- .../aiagent/framework/api/ChatRequest.java | 5 +- .../aiagent/framework/api/ChatResponse.java | 3 +- .../framework/api/ChatStreamListener.java | 3 +- .../framework/api/ModelCapabilities.java | 3 +- .../framework/api/ReasoningConfig.java | 3 +- .../aiagent/framework/api/ResponseFormat.java | 3 +- .../framework/api/event/ChatModelEvent.java | 3 +- .../Langchain4JAiFrameworkAdapter.java | 126 ------- .../Langchain4JAiFrameworkChatResponse.java | 15 - .../langchain4j/Langchain4JChatModelApi.java | 132 ++++++++ .../Langchain4JChatModelApiFactory.java | 72 ++++ ...icAiLangchain4JFrameworkConfiguration.java | 6 +- .../AgenticAiConnectorsAutoConfiguration.java | 27 +- .../JobWorkerAgentRequestHandlerTest.java | 43 +-- ...boundConnectorAgentRequestHandlerTest.java | 41 +-- .../aiagent/framework/ChatClientImplTest.java | 143 ++++++++ .../ChatModelApiRegistryImplTest.java | 90 +++++ .../Langchain4JAiFrameworkAdapterTest.java | 316 ------------------ .../Langchain4JChatModelApiTest.java | 249 ++++++++++++++ ...nticAiConnectorsAutoConfigurationTest.java | 8 +- 33 files changed, 946 insertions(+), 580 deletions(-) delete mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/AiFrameworkAdapter.java delete mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/AiFrameworkChatResponse.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImpl.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImpl.java delete mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkAdapter.java delete mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkChatResponse.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApi.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApiFactory.java create mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImplTest.java create mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImplTest.java delete mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkAdapterTest.java create mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApiTest.java diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/BaseAgentRequestHandler.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/BaseAgentRequestHandler.java index 0354a2344bf..55dae1f7076 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/BaseAgentRequestHandler.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/BaseAgentRequestHandler.java @@ -9,7 +9,9 @@ import io.camunda.connector.agenticai.aiagent.agent.AgentInitializationResult.AgentContextInitializationResult; import io.camunda.connector.agenticai.aiagent.agent.AgentInitializationResult.AgentDiscoveryInProgressInitializationResult; import io.camunda.connector.agenticai.aiagent.agent.AgentInitializationResult.AgentResponseInitializationResult; -import io.camunda.connector.agenticai.aiagent.framework.AiFrameworkAdapter; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatClient; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationSession; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStore; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStoreRegistry; @@ -17,6 +19,7 @@ import io.camunda.connector.agenticai.aiagent.memory.runtime.MessageWindowRuntimeMemory; import io.camunda.connector.agenticai.aiagent.model.AgentContext; import io.camunda.connector.agenticai.aiagent.model.AgentExecutionContext; +import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.agenticai.aiagent.model.AgentResponse; import io.camunda.connector.agenticai.aiagent.model.request.MemoryConfiguration; import io.camunda.connector.agenticai.aiagent.tool.GatewayToolHandlerRegistry; @@ -27,6 +30,8 @@ import io.camunda.connector.api.outbound.JobCompletionFailure; import java.util.List; import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +50,7 @@ public abstract class BaseAgentRequestHandler< private final AgentLimitsValidator limitsValidator; private final AgentMessagesHandler messagesHandler; private final GatewayToolHandlerRegistry gatewayToolHandlers; - private final AiFrameworkAdapter framework; + private final ChatClient chatClient; private final AgentResponseHandler responseHandler; public BaseAgentRequestHandler( @@ -54,14 +59,14 @@ public BaseAgentRequestHandler( AgentLimitsValidator limitsValidator, AgentMessagesHandler messagesHandler, GatewayToolHandlerRegistry gatewayToolHandlers, - AiFrameworkAdapter framework, + ChatClient chatClient, AgentResponseHandler responseHandler) { this.agentInitializer = agentInitializer; this.conversationStoreRegistry = conversationStoreRegistry; this.limitsValidator = limitsValidator; this.messagesHandler = messagesHandler; this.gatewayToolHandlers = gatewayToolHandlers; - this.framework = framework; + this.chatClient = chatClient; this.responseHandler = responseHandler; } @@ -166,13 +171,19 @@ private AgentResponse processConversation( handleAddedUserMessages(executionContext, agentContext, userMessages); - // call framework with memory - LOGGER.debug("Executing chat request with AI framework"); - final var frameworkChatResponse = - framework.executeChatRequest(executionContext, agentContext, runtimeMemory); - agentContext = frameworkChatResponse.agentContext(); + // dispatch via the chat client SPI; bridge / native impls live behind it + LOGGER.debug("Executing chat request"); + final var chatResponse = + joinChat( + chatClient.chat( + executionContext, agentContext, runtimeMemory, ChatStreamListener.NOOP)); + final var usage = + chatResponse.usage() != null ? chatResponse.usage() : AgentMetrics.TokenUsage.empty(); + agentContext = + agentContext.withMetrics( + agentContext.metrics().incrementModelCalls(1).incrementTokenUsage(usage)); - final var assistantMessage = frameworkChatResponse.assistantMessage(); + final var assistantMessage = chatResponse.assistantMessage(); LOGGER.debug( "Received assistant message containing {} tool call requests", assistantMessage.toolCalls() != null ? assistantMessage.toolCalls().size() : 0); @@ -196,6 +207,19 @@ private AgentResponse processConversation( executionContext, agentContext, assistantMessage, processVariableToolCalls); } + private static ChatResponse joinChat(CompletableFuture future) { + try { + return future.join(); + } catch (CompletionException e) { + // unwrap so callers see the original ConnectorException (or other RuntimeException) + // rather than the CompletableFuture wrapper + if (e.getCause() instanceof RuntimeException re) { + throw re; + } + throw e; + } + } + protected abstract boolean modelCallPrerequisitesFulfilled( C executionContext, AgentContext agentContext, List addedUserMessages); diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/JobWorkerAgentRequestHandler.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/JobWorkerAgentRequestHandler.java index e7b84914188..3ff76ebc8be 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/JobWorkerAgentRequestHandler.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/JobWorkerAgentRequestHandler.java @@ -9,7 +9,7 @@ import io.camunda.connector.agenticai.aiagent.AiAgentJobWorker; import io.camunda.connector.agenticai.aiagent.AiAgentSubProcessConnectorResponse; import io.camunda.connector.agenticai.aiagent.AiAgentSubProcessConnectorResponse.ToolCallElementActivation; -import io.camunda.connector.agenticai.aiagent.framework.AiFrameworkAdapter; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatClient; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStoreRegistry; import io.camunda.connector.agenticai.aiagent.model.AgentContext; import io.camunda.connector.agenticai.aiagent.model.AgentResponse; @@ -39,7 +39,7 @@ public JobWorkerAgentRequestHandler( AgentLimitsValidator limitsValidator, AgentMessagesHandler messagesHandler, GatewayToolHandlerRegistry gatewayToolHandlers, - AiFrameworkAdapter framework, + ChatClient chatClient, AgentResponseHandler responseHandler) { super( agentInitializer, @@ -47,7 +47,7 @@ public JobWorkerAgentRequestHandler( limitsValidator, messagesHandler, gatewayToolHandlers, - framework, + chatClient, responseHandler); } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/OutboundConnectorAgentRequestHandler.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/OutboundConnectorAgentRequestHandler.java index 93721a801ce..ac7b2a854f1 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/OutboundConnectorAgentRequestHandler.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/OutboundConnectorAgentRequestHandler.java @@ -9,7 +9,7 @@ import static io.camunda.connector.agenticai.aiagent.agent.AgentErrorCodes.ERROR_CODE_NO_USER_MESSAGE_CONTENT; import io.camunda.connector.agenticai.aiagent.AiAgentTaskConnectorResponse; -import io.camunda.connector.agenticai.aiagent.framework.AiFrameworkAdapter; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatClient; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStoreRegistry; import io.camunda.connector.agenticai.aiagent.model.AgentContext; import io.camunda.connector.agenticai.aiagent.model.AgentResponse; @@ -30,7 +30,7 @@ public OutboundConnectorAgentRequestHandler( AgentLimitsValidator limitsValidator, AgentMessagesHandler messagesHandler, GatewayToolHandlerRegistry gatewayToolHandlers, - AiFrameworkAdapter framework, + ChatClient chatClient, AgentResponseHandler responseHandler) { super( agentInitializer, @@ -38,7 +38,7 @@ public OutboundConnectorAgentRequestHandler( limitsValidator, messagesHandler, gatewayToolHandlers, - framework, + chatClient, responseHandler); } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/AiFrameworkAdapter.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/AiFrameworkAdapter.java deleted file mode 100644 index 8c36fdf03d0..00000000000 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/AiFrameworkAdapter.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH - * under one or more contributor license agreements. Licensed under a proprietary license. - * See the License.txt file for more information. You may not use this file - * except in compliance with the proprietary license. - */ -package io.camunda.connector.agenticai.aiagent.framework; - -import io.camunda.connector.agenticai.aiagent.memory.runtime.RuntimeMemory; -import io.camunda.connector.agenticai.aiagent.model.AgentContext; -import io.camunda.connector.agenticai.aiagent.model.AgentExecutionContext; - -public interface AiFrameworkAdapter { - R executeChatRequest( - AgentExecutionContext executionContext, - AgentContext agentContext, - RuntimeMemory runtimeMemory); -} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/AiFrameworkChatResponse.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/AiFrameworkChatResponse.java deleted file mode 100644 index 66a1bb4b8f9..00000000000 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/AiFrameworkChatResponse.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH - * under one or more contributor license agreements. Licensed under a proprietary license. - * See the License.txt file for more information. You may not use this file - * except in compliance with the proprietary license. - */ -package io.camunda.connector.agenticai.aiagent.framework; - -import io.camunda.connector.agenticai.aiagent.model.AgentContext; -import io.camunda.connector.agenticai.model.message.AssistantMessage; - -public interface AiFrameworkChatResponse { - AgentContext agentContext(); - - AssistantMessage assistantMessage(); -} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImpl.java new file mode 100644 index 00000000000..8425f724c1b --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImpl.java @@ -0,0 +1,64 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework; + +import io.camunda.connector.agenticai.aiagent.framework.api.ChatClient; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiRegistry; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatOptions; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; +import io.camunda.connector.agenticai.aiagent.framework.api.ResponseFormat; +import io.camunda.connector.agenticai.aiagent.memory.runtime.RuntimeMemory; +import io.camunda.connector.agenticai.aiagent.model.AgentContext; +import io.camunda.connector.agenticai.aiagent.model.AgentExecutionContext; +import io.camunda.connector.agenticai.aiagent.model.request.ResponseConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.ResponseFormatConfiguration.JsonResponseFormatConfiguration; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import org.springframework.lang.Nullable; + +public class ChatClientImpl implements ChatClient { + + private final ChatModelApiRegistry registry; + + public ChatClientImpl(ChatModelApiRegistry registry) { + this.registry = registry; + } + + @Override + public CompletableFuture chat( + AgentExecutionContext executionContext, + AgentContext agentContext, + RuntimeMemory runtimeMemory, + ChatStreamListener listener) { + try { + final var api = registry.resolve(executionContext.provider()); + final var request = + new ChatRequest(runtimeMemory.filteredMessages(), null, agentContext.toolDefinitions()); + final var options = + new ChatOptions( + null, null, null, toResponseFormat(executionContext.response()), Map.of()); + return api.complete(request, options, listener != null ? listener : ChatStreamListener.NOOP); + } catch (RuntimeException e) { + return CompletableFuture.failedFuture(e); + } + } + + private static @Nullable ResponseFormat toResponseFormat( + @Nullable ResponseConfiguration response) { + if (response == null || response.format() == null) { + return null; + } + if (response.format() instanceof JsonResponseFormatConfiguration json) { + return new ResponseFormat.Json(json.schemaName(), json.schema()); + } + // TextResponseFormatConfiguration → null preserves "no explicit format on the wire" + // behaviour. Implementations choose their default text mode. + return null; + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImpl.java new file mode 100644 index 00000000000..ec30f2c5dd6 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImpl.java @@ -0,0 +1,63 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework; + +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiRegistry; +import io.camunda.connector.agenticai.aiagent.model.request.provider.ProviderConfiguration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ChatModelApiRegistryImpl implements ChatModelApiRegistry { + + private static final Logger LOGGER = LoggerFactory.getLogger(ChatModelApiRegistryImpl.class); + + private final Map> factoriesByProviderType; + + public ChatModelApiRegistryImpl(List> factories) { + this.factoriesByProviderType = indexByProviderType(factories); + } + + private static Map> indexByProviderType( + List> factories) { + final Map> index = new HashMap<>(); + for (ChatModelApiFactory factory : factories) { + for (String providerType : factory.supportedProviderTypes()) { + final var existing = index.get(providerType); + if (existing != null) { + throw new IllegalStateException( + "Two chat model API factories claim provider type '%s': %s and %s. To override the default factory, exclude the built-in bean (e.g. via @ConditionalOnMissingBean) before contributing your own." + .formatted( + providerType, existing.getClass().getName(), factory.getClass().getName())); + } + index.put(providerType, factory); + LOGGER.debug( + "Registered chat model API factory for provider type '{}': {} (apiFamily={})", + providerType, + factory.getClass().getName(), + factory.apiFamily()); + } + } + return Map.copyOf(index); + } + + @Override + @SuppressWarnings("unchecked") + public ChatModelApi resolve(ProviderConfiguration configuration) { + final var providerType = configuration.providerType(); + final var factory = factoriesByProviderType.get(providerType); + if (factory == null) { + throw new IllegalStateException( + "No chat model API factory registered for provider type '%s'".formatted(providerType)); + } + return ((ChatModelApiFactory) factory).create(configuration); + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/CacheRetention.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/CacheRetention.java index 01b3285b9ea..b4411c8c065 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/CacheRetention.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/CacheRetention.java @@ -13,7 +13,8 @@ * retention; {@code NONE} strips cache markers entirely. Concrete breakpoint placement is * implementation-specific. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + * ChatModelApiRegistry. */ public enum CacheRetention { NONE, diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClient.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClient.java index 1766b4a3b88..51a3253759f 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClient.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClient.java @@ -20,7 +20,8 @@ *

    Replaces today's {@code AiFrameworkAdapter} call site. Mirroring its signature keeps the * cutover diff small. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + * ChatModelApiRegistry. */ public interface ChatClient { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApi.java index 79787938ef9..6330ae735bb 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApi.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApi.java @@ -17,7 +17,8 @@ * {@link CompletableFuture} surface to callers. The optional {@link ChatStreamListener} receives * discriminated stream events for in-process observability. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + * ChatModelApiRegistry. */ public interface ChatModelApi { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java index 1ccb7a226f4..e4531731f9e 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java @@ -18,7 +18,8 @@ * providerType} strings each factory claims via {@link #supportedProviderTypes} and dispatches by * exact match. {@link #apiFamily} is informational — used in logs and stream events. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + * ChatModelApiRegistry. * * @param the {@link ProviderConfiguration} subtype this factory handles */ diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java index 408bccbbe85..0eb1915d7d8 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java @@ -18,7 +18,8 @@ *

    Unknown provider types fail fast — there is no factory to resolve, so requests can never * start. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + * ChatModelApiRegistry. */ public interface ChatModelApiRegistry { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java index 9a34dd56b27..ca3c03226a8 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java @@ -34,7 +34,8 @@ * structured-output mode (Anthropic Messages today) treat {@link ResponseFormat.Json} as * best-effort and rely on the system prompt to constrain output. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + * ChatModelApiRegistry. */ public record ChatOptions( @Nullable Integer maxOutputTokens, diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java index ac7dcd48698..d992e605fca 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java @@ -17,8 +17,9 @@ * tunables (max output tokens, stop sequences, reasoning, cache retention, vendor escape hatches) * live on {@link ChatOptions} so a request can be reused while options vary. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime — concrete fields - * will be finalised when the first native {@code ChatModelApi} implementation lands. + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + * ChatModelApiRegistry — concrete fields will be finalised when the first native {@code + * ChatModelApi} implementation lands. */ public record ChatRequest( List messages, @Nullable String systemPrompt, List tools) {} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatResponse.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatResponse.java index af6388bd043..730374e57b9 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatResponse.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatResponse.java @@ -18,7 +18,8 @@ * where {@code stopReason == ERROR}; transport / SDK / auth failures complete the future * exceptionally instead. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + * ChatModelApiRegistry. */ public record ChatResponse( AssistantMessage assistantMessage, diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatStreamListener.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatStreamListener.java index bab42b4c654..589a612617f 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatStreamListener.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatStreamListener.java @@ -14,7 +14,8 @@ * listener is intentionally not exposed as a reactive type — the public chat surface remains a * blocking {@code CompletableFuture}. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + * ChatModelApiRegistry. */ @FunctionalInterface public interface ChatStreamListener { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java index 2b091b2ded2..c83980bc188 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java @@ -15,7 +15,8 @@ * negotiation, and cache-marker placement. The vocabulary for {@link Modality} is fixed; modality * lists per location are symmetric so every location has an explicit answer. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + * ChatModelApiRegistry. */ public record ModelCapabilities( List userMessageModalities, diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ReasoningConfig.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ReasoningConfig.java index f7736e09fae..3134a85ff7c 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ReasoningConfig.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ReasoningConfig.java @@ -12,7 +12,8 @@ * this onto provider-native fields (Anthropic adaptive effort / thinking budget, OpenAI Responses * reasoning effort, Gemini thinking budget, etc.). * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + * ChatModelApiRegistry. */ public sealed interface ReasoningConfig permits ReasoningConfig.ReasoningEffort, diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ResponseFormat.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ResponseFormat.java index c36576fbe10..8c62c7ae949 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ResponseFormat.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ResponseFormat.java @@ -21,7 +21,8 @@ * Providers without a native structured-output mode (Anthropic Messages today) treat {@link Json} * as best-effort and rely on the system prompt to constrain output. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + * ChatModelApiRegistry. */ public sealed interface ResponseFormat permits ResponseFormat.Text, ResponseFormat.Json { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/event/ChatModelEvent.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/event/ChatModelEvent.java index 82db2b1476b..9016529a1ad 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/event/ChatModelEvent.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/event/ChatModelEvent.java @@ -16,7 +16,8 @@ * the assembled {@link ChatResponse}, while {@link ErrorEvent} carries the error message plus any * partial content / usage accumulated before the failure. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Not yet wired into the runtime. + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + * ChatModelApiRegistry. */ public sealed interface ChatModelEvent permits ChatModelEvent.StartEvent, diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkAdapter.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkAdapter.java deleted file mode 100644 index 8769794d358..00000000000 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkAdapter.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH - * under one or more contributor license agreements. Licensed under a proprietary license. - * See the License.txt file for more information. You may not use this file - * except in compliance with the proprietary license. - */ -package io.camunda.connector.agenticai.aiagent.framework.langchain4j; - -import static io.camunda.connector.agenticai.aiagent.agent.AgentErrorCodes.ERROR_CODE_FAILED_MODEL_CALL; - -import dev.langchain4j.model.chat.ChatModel; -import dev.langchain4j.model.chat.request.ChatRequest; -import dev.langchain4j.model.chat.request.ResponseFormat; -import dev.langchain4j.model.chat.request.ResponseFormatType; -import dev.langchain4j.model.chat.request.json.JsonSchema; -import dev.langchain4j.model.chat.response.ChatResponse; -import io.camunda.connector.agenticai.aiagent.framework.AiFrameworkAdapter; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.jsonschema.JsonSchemaConverter; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverter; -import io.camunda.connector.agenticai.aiagent.memory.runtime.RuntimeMemory; -import io.camunda.connector.agenticai.aiagent.model.AgentContext; -import io.camunda.connector.agenticai.aiagent.model.AgentExecutionContext; -import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; -import io.camunda.connector.agenticai.aiagent.model.request.ResponseConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.ResponseFormatConfiguration.JsonResponseFormatConfiguration; -import io.camunda.connector.agenticai.model.message.AssistantMessage; -import io.camunda.connector.api.error.ConnectorException; -import java.util.Optional; -import org.apache.commons.lang3.StringUtils; - -public class Langchain4JAiFrameworkAdapter - implements AiFrameworkAdapter { - - private final ChatModelFactory chatModelFactory; - private final ChatMessageConverter chatMessageConverter; - private final ToolSpecificationConverter toolSpecificationConverter; - private final JsonSchemaConverter jsonSchemaConverter; - - public Langchain4JAiFrameworkAdapter( - ChatModelFactory chatModelFactory, - ChatMessageConverter chatMessageConverter, - ToolSpecificationConverter toolSpecificationConverter, - JsonSchemaConverter jsonSchemaConverter) { - this.chatModelFactory = chatModelFactory; - this.chatMessageConverter = chatMessageConverter; - this.toolSpecificationConverter = toolSpecificationConverter; - this.jsonSchemaConverter = jsonSchemaConverter; - } - - @Override - public Langchain4JAiFrameworkChatResponse executeChatRequest( - AgentExecutionContext executionContext, - AgentContext agentContext, - RuntimeMemory runtimeMemory) { - final var messages = chatMessageConverter.map(runtimeMemory.filteredMessages()); - final var toolSpecifications = - toolSpecificationConverter.asToolSpecifications(agentContext.toolDefinitions()); - - final var chatRequestBuilder = - ChatRequest.builder().messages(messages).toolSpecifications(toolSpecifications); - configureResponseFormat(chatRequestBuilder, executionContext.response()); - - final ChatModel chatModel = chatModelFactory.createChatModel(executionContext.provider()); - final ChatResponse chatResponse = doChat(chatModel, chatRequestBuilder); - final AssistantMessage assistantMessage = chatMessageConverter.toAssistantMessage(chatResponse); - - final var updatedAgentContext = - agentContext.withMetrics( - agentContext - .metrics() - .incrementModelCalls(1) - .incrementTokenUsage( - assistantMessage.usage() != null - ? assistantMessage.usage() - : AgentMetrics.TokenUsage.empty())); - - return new Langchain4JAiFrameworkChatResponse(updatedAgentContext, assistantMessage); - } - - private void configureResponseFormat( - ChatRequest.Builder chatRequestBuilder, ResponseConfiguration responseConfiguration) { - final var responseFormat = createResponseFormat(responseConfiguration); - if (responseFormat != null) { - chatRequestBuilder.responseFormat(responseFormat); - } - } - - private ResponseFormat createResponseFormat(ResponseConfiguration responseConfiguration) { - // do not explicitely configure response format to TEXT as (depending on the model) this might - // lead to exceptions - if (responseConfiguration != null - && responseConfiguration.format() != null - && responseConfiguration.format() instanceof JsonResponseFormatConfiguration jsonFormat) { - final var builder = ResponseFormat.builder().type(ResponseFormatType.JSON); - if (jsonFormat.schema() != null) { - final var jsonSchema = - JsonSchema.builder() - .name( - Optional.ofNullable(jsonFormat.schemaName()) - .filter(StringUtils::isNotBlank) - .orElse("Response")) - .rootElement(jsonSchemaConverter.mapToSchema(jsonFormat.schema())) - .build(); - builder.jsonSchema(jsonSchema); - } - - return builder.build(); - } - - return null; - } - - private ChatResponse doChat(ChatModel chatModel, ChatRequest.Builder chatRequestBuilder) { - try { - return chatModel.chat(chatRequestBuilder.build()); - } catch (Exception e) { - final var message = - Optional.ofNullable(e.getMessage()) - .filter(StringUtils::isNotBlank) - .orElseGet(() -> e.getClass().getSimpleName()); - - throw new ConnectorException( - ERROR_CODE_FAILED_MODEL_CALL, "Model call failed: %s".formatted(message), e); - } - } -} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkChatResponse.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkChatResponse.java deleted file mode 100644 index 09e00a87cd0..00000000000 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkChatResponse.java +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH - * under one or more contributor license agreements. Licensed under a proprietary license. - * See the License.txt file for more information. You may not use this file - * except in compliance with the proprietary license. - */ -package io.camunda.connector.agenticai.aiagent.framework.langchain4j; - -import io.camunda.connector.agenticai.aiagent.framework.AiFrameworkChatResponse; -import io.camunda.connector.agenticai.aiagent.model.AgentContext; -import io.camunda.connector.agenticai.model.message.AssistantMessage; - -public record Langchain4JAiFrameworkChatResponse( - AgentContext agentContext, AssistantMessage assistantMessage) - implements AiFrameworkChatResponse {} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApi.java new file mode 100644 index 00000000000..968428f931f --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApi.java @@ -0,0 +1,132 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.langchain4j; + +import static io.camunda.connector.agenticai.aiagent.agent.AgentErrorCodes.ERROR_CODE_FAILED_MODEL_CALL; + +import dev.langchain4j.model.chat.ChatModel; +import dev.langchain4j.model.chat.request.ChatRequest; +import dev.langchain4j.model.chat.request.ResponseFormatType; +import dev.langchain4j.model.chat.request.json.JsonSchema; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatOptions; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; +import io.camunda.connector.agenticai.aiagent.framework.api.ResponseFormat; +import io.camunda.connector.agenticai.aiagent.framework.langchain4j.jsonschema.JsonSchemaConverter; +import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverter; +import io.camunda.connector.api.error.ConnectorException; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import org.apache.commons.lang3.StringUtils; +import org.springframework.lang.Nullable; + +/** + * LangChain4j-backed {@link ChatModelApi} used for every wire-protocol family today. Wraps a + * pre-resolved {@link ChatModel}; one instance per chat invocation, produced by {@link + * Langchain4JChatModelApiFactory#create}. + * + *

    Public so customers can compose it from their own {@link + * io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory ChatModelApiFactory} + * bean — e.g. to wire up a LangChain4j-supported provider we don't ship by passing in their own + * resolved {@link ChatModel} alongside the framework's converter beans. + */ +public class Langchain4JChatModelApi implements ChatModelApi { + + private final ChatModel chatModel; + private final ChatMessageConverter chatMessageConverter; + private final ToolSpecificationConverter toolSpecificationConverter; + private final JsonSchemaConverter jsonSchemaConverter; + + public Langchain4JChatModelApi( + ChatModel chatModel, + ChatMessageConverter chatMessageConverter, + ToolSpecificationConverter toolSpecificationConverter, + JsonSchemaConverter jsonSchemaConverter) { + this.chatModel = chatModel; + this.chatMessageConverter = chatMessageConverter; + this.toolSpecificationConverter = toolSpecificationConverter; + this.jsonSchemaConverter = jsonSchemaConverter; + } + + @Override + public ModelCapabilities capabilities() { + // Conservative defaults — the bridge is model-agnostic. Native implementations will resolve + // capabilities from the matrix. + return new ModelCapabilities( + List.of(Modality.TEXT), + List.of(Modality.TEXT), + List.of(Modality.TEXT), + false, + false, + false, + true, + null, + null); + } + + @Override + public CompletableFuture complete( + io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest request, + ChatOptions options, + ChatStreamListener listener) { + try { + final var l4jMessages = chatMessageConverter.map(request.messages()); + final var toolSpecifications = + toolSpecificationConverter.asToolSpecifications(request.tools()); + + final var l4jRequestBuilder = + ChatRequest.builder().messages(l4jMessages).toolSpecifications(toolSpecifications); + + final var l4jResponseFormat = toL4jResponseFormat(options.responseFormat()); + if (l4jResponseFormat != null) { + l4jRequestBuilder.responseFormat(l4jResponseFormat); + } + + final var l4jResponse = chatModel.chat(l4jRequestBuilder.build()); + final var assistantMessage = chatMessageConverter.toAssistantMessage(l4jResponse); + + return CompletableFuture.completedFuture( + new ChatResponse( + assistantMessage, assistantMessage.stopReason(), assistantMessage.usage(), null)); + } catch (Exception e) { + final var message = + Optional.ofNullable(e.getMessage()) + .filter(StringUtils::isNotBlank) + .orElseGet(() -> e.getClass().getSimpleName()); + return CompletableFuture.failedFuture( + new ConnectorException( + ERROR_CODE_FAILED_MODEL_CALL, "Model call failed: %s".formatted(message), e)); + } + } + + private @Nullable dev.langchain4j.model.chat.request.ResponseFormat toL4jResponseFormat( + @Nullable ResponseFormat responseFormat) { + if (!(responseFormat instanceof ResponseFormat.Json json)) { + // Both null and ResponseFormat.Text leave the format unset — matches the previous adapter, + // which avoided sending TEXT explicitly because some models reject it. + return null; + } + + final var builder = + dev.langchain4j.model.chat.request.ResponseFormat.builder().type(ResponseFormatType.JSON); + + if (json.schema() != null) { + final var name = StringUtils.isNotBlank(json.schemaName()) ? json.schemaName() : "Response"; + builder.jsonSchema( + JsonSchema.builder() + .name(name) + .rootElement(jsonSchemaConverter.mapToSchema(json.schema())) + .build()); + } + + return builder.build(); + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApiFactory.java new file mode 100644 index 00000000000..9eeeae77157 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApiFactory.java @@ -0,0 +1,72 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.langchain4j; + +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; +import io.camunda.connector.agenticai.aiagent.framework.langchain4j.jsonschema.JsonSchemaConverter; +import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverter; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.ProviderConfiguration; +import java.util.Set; + +/** + * Bridge factory that produces a {@link Langchain4JChatModelApi} for every built-in {@link + * ProviderConfiguration} subtype. Until per-family native implementations ship, every chat call + * lands here. + */ +public class Langchain4JChatModelApiFactory implements ChatModelApiFactory { + + public static final String API_FAMILY = "langchain4j"; + + private static final Set SUPPORTED_PROVIDER_TYPES = + Set.of( + AnthropicProviderConfiguration.ANTHROPIC_ID, + BedrockProviderConfiguration.BEDROCK_ID, + AzureOpenAiProviderConfiguration.AZURE_OPENAI_ID, + GoogleVertexAiProviderConfiguration.GOOGLE_VERTEX_AI_ID, + OpenAiProviderConfiguration.OPENAI_ID, + OpenAiCompatibleProviderConfiguration.OPENAI_COMPATIBLE_ID); + + private final ChatModelFactory chatModelFactory; + private final ChatMessageConverter chatMessageConverter; + private final ToolSpecificationConverter toolSpecificationConverter; + private final JsonSchemaConverter jsonSchemaConverter; + + public Langchain4JChatModelApiFactory( + ChatModelFactory chatModelFactory, + ChatMessageConverter chatMessageConverter, + ToolSpecificationConverter toolSpecificationConverter, + JsonSchemaConverter jsonSchemaConverter) { + this.chatModelFactory = chatModelFactory; + this.chatMessageConverter = chatMessageConverter; + this.toolSpecificationConverter = toolSpecificationConverter; + this.jsonSchemaConverter = jsonSchemaConverter; + } + + @Override + public String apiFamily() { + return API_FAMILY; + } + + @Override + public Set supportedProviderTypes() { + return SUPPORTED_PROVIDER_TYPES; + } + + @Override + public ChatModelApi create(ProviderConfiguration configuration) { + final var chatModel = chatModelFactory.createChatModel(configuration); + return new Langchain4JChatModelApi( + chatModel, chatMessageConverter, toolSpecificationConverter, jsonSchemaConverter); + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java index ae93ad14f2a..6bc9515990f 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java @@ -12,7 +12,7 @@ import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelFactory; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ContentConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ContentConverterImpl; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.Langchain4JAiFrameworkAdapter; +import io.camunda.connector.agenticai.aiagent.framework.langchain4j.Langchain4JChatModelApiFactory; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.document.DocumentToContentConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.document.DocumentToContentConverterImpl; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.jsonschema.JsonSchemaConverter; @@ -79,12 +79,12 @@ public ChatMessageConverter langchain4JChatMessageConverter( @Bean @ConditionalOnMissingBean - public Langchain4JAiFrameworkAdapter langchain4JAiFrameworkAdapter( + public Langchain4JChatModelApiFactory langchain4JChatModelApiFactory( ChatModelFactory chatModelFactory, ChatMessageConverter chatMessageConverter, ToolSpecificationConverter toolSpecificationConverter, JsonSchemaConverter jsonSchemaConverter) { - return new Langchain4JAiFrameworkAdapter( + return new Langchain4JChatModelApiFactory( chatModelFactory, chatMessageConverter, toolSpecificationConverter, jsonSchemaConverter); } } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java index 9d2efd3f823..ec406f07002 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java @@ -39,7 +39,11 @@ import io.camunda.connector.agenticai.aiagent.agent.JobWorkerAgentRequestHandler; import io.camunda.connector.agenticai.aiagent.agent.OutboundConnectorAgentRequestHandler; import io.camunda.connector.agenticai.aiagent.agent.ToolCallResultDocumentExtractor; -import io.camunda.connector.agenticai.aiagent.framework.AiFrameworkAdapter; +import io.camunda.connector.agenticai.aiagent.framework.ChatClientImpl; +import io.camunda.connector.agenticai.aiagent.framework.ChatModelApiRegistryImpl; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatClient; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiRegistry; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelHttpProxySupport; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.configuration.AgenticAiLangchain4JFrameworkConfiguration; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStore; @@ -255,6 +259,19 @@ public AgentResponseHandler aiAgentResponseHandler( return new AgentResponseHandlerImpl(objectMapper); } + @Bean + @ConditionalOnMissingBean + public ChatModelApiRegistry aiAgentChatModelApiRegistry( + List> chatModelApiFactories) { + return new ChatModelApiRegistryImpl(chatModelApiFactories); + } + + @Bean + @ConditionalOnMissingBean + public ChatClient aiAgentChatClient(ChatModelApiRegistry chatModelApiRegistry) { + return new ChatClientImpl(chatModelApiRegistry); + } + @Bean @ConditionalOnMissingBean @ConditionalOnBooleanProperty( @@ -266,7 +283,7 @@ public OutboundConnectorAgentRequestHandler aiAgentOutboundConnectorAgentRequest AgentLimitsValidator limitsValidator, AgentMessagesHandler messagesHandler, GatewayToolHandlerRegistry gatewayToolHandlers, - AiFrameworkAdapter aiFrameworkAdapter, + ChatClient chatClient, AgentResponseHandler responseHandler) { return new OutboundConnectorAgentRequestHandler( agentInitializer, @@ -274,7 +291,7 @@ public OutboundConnectorAgentRequestHandler aiAgentOutboundConnectorAgentRequest limitsValidator, messagesHandler, gatewayToolHandlers, - aiFrameworkAdapter, + chatClient, responseHandler); } @@ -300,7 +317,7 @@ public JobWorkerAgentRequestHandler aiAgentJobWorkerAgentRequestHandler( AgentLimitsValidator limitsValidator, AgentMessagesHandler messagesHandler, GatewayToolHandlerRegistry gatewayToolHandlers, - AiFrameworkAdapter aiFrameworkAdapter, + ChatClient chatClient, AgentResponseHandler responseHandler) { return new JobWorkerAgentRequestHandler( agentInitializer, @@ -308,7 +325,7 @@ public JobWorkerAgentRequestHandler aiAgentJobWorkerAgentRequestHandler( limitsValidator, messagesHandler, gatewayToolHandlers, - aiFrameworkAdapter, + chatClient, responseHandler); } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/JobWorkerAgentRequestHandlerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/JobWorkerAgentRequestHandlerTest.java index ba8b3810606..be5c762f152 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/JobWorkerAgentRequestHandlerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/JobWorkerAgentRequestHandlerTest.java @@ -29,8 +29,8 @@ import io.camunda.connector.agenticai.aiagent.AiAgentJobWorker; import io.camunda.connector.agenticai.aiagent.agent.AgentInitializationResult.AgentContextInitializationResult; import io.camunda.connector.agenticai.aiagent.agent.AgentInitializationResult.AgentResponseInitializationResult; -import io.camunda.connector.agenticai.aiagent.framework.AiFrameworkAdapter; -import io.camunda.connector.agenticai.aiagent.framework.AiFrameworkChatResponse; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatClient; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStore; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStoreRegistry; import io.camunda.connector.agenticai.aiagent.memory.conversation.inprocess.InProcessConversationContext; @@ -55,6 +55,7 @@ import io.camunda.connector.agenticai.model.tool.ToolCallResult; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -93,7 +94,7 @@ class JobWorkerAgentRequestHandlerTest { @Mock private AgentLimitsValidator limitsValidator; @Mock private AgentMessagesHandler messagesHandler; @Mock private GatewayToolHandlerRegistry gatewayToolHandlers; - @Mock private AiFrameworkAdapter framework; + @Mock private ChatClient chatClient; @Mock private AgentResponseHandler responseHandler; private ConversationStore conversationStore; @@ -136,7 +137,7 @@ void directlyReturnsAgentResponseWhenInitializationReturnsResponse() { assertThat(response.responseValue()).isNotNull().isEqualTo(agentResponse); verifyNoInteractions( - limitsValidator, messagesHandler, gatewayToolHandlers, framework, responseHandler); + limitsValidator, messagesHandler, gatewayToolHandlers, chatClient, responseHandler); } @Test @@ -430,7 +431,7 @@ void silentlyCompletesJobWhenNoUserMessageContent() { assertThat(response.cancelRemainingInstances()).isFalse(); assertThat(response.elementActivations()).isEmpty(); - verifyNoInteractions(framework); + verifyNoInteractions(chatClient); } private RuntimeMemory setupRuntimeMemorySizeTest(MemoryConfiguration memoryConfiguration) { @@ -524,26 +525,18 @@ private void mockInterruptedToolCall(List toolCallResults) { } private void mockFrameworkExecution(AssistantMessage assistantMessage) { - when(framework.executeChatRequest( - eq(agentExecutionContext), any(AgentContext.class), runtimeMemoryCaptor.capture())) + when(chatClient.chat( + eq(agentExecutionContext), + any(AgentContext.class), + runtimeMemoryCaptor.capture(), + any())) .thenAnswer( - i -> { - final var agentContext = i.getArgument(1, AgentContext.class); - return new TestFrameworkChatResponse( - agentContext.withMetrics( - agentContext - .metrics() - .incrementModelCalls(1) - .incrementTokenUsage( - TokenUsage.builder() - .inputTokenCount(10) - .outputTokenCount(20) - .build())), - assistantMessage); - }); + i -> + CompletableFuture.completedFuture( + new ChatResponse( + assistantMessage, + assistantMessage.stopReason(), + TokenUsage.builder().inputTokenCount(10).outputTokenCount(20).build(), + null))); } - - private record TestFrameworkChatResponse( - AgentContext agentContext, AssistantMessage assistantMessage) - implements AiFrameworkChatResponse {} } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/OutboundConnectorAgentRequestHandlerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/OutboundConnectorAgentRequestHandlerTest.java index bd93d6f7963..d1811736c25 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/OutboundConnectorAgentRequestHandlerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/OutboundConnectorAgentRequestHandlerTest.java @@ -25,8 +25,8 @@ import io.camunda.connector.agenticai.aiagent.agent.AgentInitializationResult.AgentContextInitializationResult; import io.camunda.connector.agenticai.aiagent.agent.AgentInitializationResult.AgentResponseInitializationResult; -import io.camunda.connector.agenticai.aiagent.framework.AiFrameworkAdapter; -import io.camunda.connector.agenticai.aiagent.framework.AiFrameworkChatResponse; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatClient; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStoreRegistry; import io.camunda.connector.agenticai.aiagent.memory.conversation.inprocess.InProcessConversationContext; import io.camunda.connector.agenticai.aiagent.memory.conversation.inprocess.InProcessConversationStore; @@ -50,6 +50,7 @@ import io.camunda.connector.api.error.ConnectorException; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -83,7 +84,7 @@ class OutboundConnectorAgentRequestHandlerTest { @Mock private AgentLimitsValidator limitsValidator; @Mock private AgentMessagesHandler messagesHandler; @Mock private GatewayToolHandlerRegistry gatewayToolHandlers; - @Mock private AiFrameworkAdapter framework; + @Mock private ChatClient chatClient; @Mock private AgentResponseHandler responseHandler; @Mock private OutboundConnectorAgentExecutionContext agentExecutionContext; @@ -118,7 +119,7 @@ void directlyReturnsAgentResponseWhenInitializationReturnsResponse() { assertThat(response.agentResponse()).isEqualTo(agentResponse); verifyNoInteractions( - limitsValidator, messagesHandler, gatewayToolHandlers, framework, responseHandler); + limitsValidator, messagesHandler, gatewayToolHandlers, chatClient, responseHandler); } @Test @@ -384,26 +385,18 @@ private void mockUserPrompt( } private void mockFrameworkExecution(AssistantMessage assistantMessage) { - when(framework.executeChatRequest( - eq(agentExecutionContext), any(AgentContext.class), runtimeMemoryCaptor.capture())) + when(chatClient.chat( + eq(agentExecutionContext), + any(AgentContext.class), + runtimeMemoryCaptor.capture(), + any())) .thenAnswer( - i -> { - final var agentContext = i.getArgument(1, AgentContext.class); - return new TestFrameworkChatResponse( - agentContext.withMetrics( - agentContext - .metrics() - .incrementModelCalls(1) - .incrementTokenUsage( - TokenUsage.builder() - .inputTokenCount(10) - .outputTokenCount(20) - .build())), - assistantMessage); - }); + i -> + CompletableFuture.completedFuture( + new ChatResponse( + assistantMessage, + assistantMessage.stopReason(), + TokenUsage.builder().inputTokenCount(10).outputTokenCount(20).build(), + null))); } - - private record TestFrameworkChatResponse( - AgentContext agentContext, AssistantMessage assistantMessage) - implements AiFrameworkChatResponse {} } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImplTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImplTest.java new file mode 100644 index 00000000000..73b82a94094 --- /dev/null +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImplTest.java @@ -0,0 +1,143 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework; + +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.systemMessage; +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.userMessage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiRegistry; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatOptions; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; +import io.camunda.connector.agenticai.aiagent.framework.api.ResponseFormat; +import io.camunda.connector.agenticai.aiagent.memory.runtime.DefaultRuntimeMemory; +import io.camunda.connector.agenticai.aiagent.memory.runtime.RuntimeMemory; +import io.camunda.connector.agenticai.aiagent.model.AgentContext; +import io.camunda.connector.agenticai.aiagent.model.AgentExecutionContext; +import io.camunda.connector.agenticai.aiagent.model.AgentState; +import io.camunda.connector.agenticai.aiagent.model.request.OutboundConnectorResponseConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.ResponseConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.ResponseFormatConfiguration.JsonResponseFormatConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.ResponseFormatConfiguration.TextResponseFormatConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicConnection; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicModel; +import io.camunda.connector.agenticai.model.tool.ToolDefinition; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ChatClientImplTest { + + private static final List TOOL_DEFINITIONS = + List.of(ToolDefinition.builder().name("Tool").description("desc").build()); + + private static final AgentContext AGENT_CONTEXT = + AgentContext.empty().withState(AgentState.READY).withToolDefinitions(TOOL_DEFINITIONS); + + private static final AnthropicProviderConfiguration PROVIDER_CONFIG = + new AnthropicProviderConfiguration( + new AnthropicConnection( + null, + new AnthropicAuthentication("api-key"), + null, + new AnthropicModel("claude", null))); + + @Mock private ChatModelApiRegistry registry; + @Mock private ChatModelApi chatModelApi; + @Mock private AgentExecutionContext executionContext; + @Mock private ChatResponse chatResponse; + + @Captor private ArgumentCaptor requestCaptor; + @Captor private ArgumentCaptor optionsCaptor; + + private RuntimeMemory runtimeMemory; + private ChatClientImpl chatClient; + + @BeforeEach + void setUp() { + runtimeMemory = new DefaultRuntimeMemory(); + runtimeMemory.addMessages(List.of(systemMessage("system"), userMessage("hi"))); + + when(executionContext.provider()).thenReturn(PROVIDER_CONFIG); + when(registry.resolve(PROVIDER_CONFIG)).thenReturn(chatModelApi); + when(chatModelApi.complete(requestCaptor.capture(), optionsCaptor.capture(), any())) + .thenReturn(CompletableFuture.completedFuture(chatResponse)); + + chatClient = new ChatClientImpl(registry); + } + + @Test + void buildsRequestFromRuntimeMemoryAndAgentContext() { + final var future = chatClient.chat(executionContext, AGENT_CONTEXT, runtimeMemory, null); + + assertThat(future).isCompleted(); + assertThat(future.join()).isSameAs(chatResponse); + assertThat(requestCaptor.getValue().messages()) + .containsExactlyElementsOf(runtimeMemory.filteredMessages()); + assertThat(requestCaptor.getValue().tools()).containsExactlyElementsOf(TOOL_DEFINITIONS); + assertThat(requestCaptor.getValue().systemPrompt()).isNull(); + } + + @Test + void translatesJsonResponseFormatConfiguration() { + when(executionContext.response()) + .thenReturn( + new OutboundConnectorResponseConfiguration( + new JsonResponseFormatConfiguration(Map.of("type", "object"), "MySchema"), false)); + + chatClient.chat(executionContext, AGENT_CONTEXT, runtimeMemory, ChatStreamListener.NOOP); + + final var responseFormat = optionsCaptor.getValue().responseFormat(); + assertThat(responseFormat).isInstanceOf(ResponseFormat.Json.class); + final var json = (ResponseFormat.Json) responseFormat; + assertThat(json.schemaName()).isEqualTo("MySchema"); + assertThat(json.schema()).containsEntry("type", "object"); + } + + @Test + void leavesResponseFormatNullForTextConfiguration() { + when(executionContext.response()) + .thenReturn( + new OutboundConnectorResponseConfiguration( + new TextResponseFormatConfiguration(false), false)); + + chatClient.chat(executionContext, AGENT_CONTEXT, runtimeMemory, ChatStreamListener.NOOP); + + assertThat(optionsCaptor.getValue().responseFormat()).isNull(); + } + + @Test + void leavesResponseFormatNullWhenResponseConfigurationMissing() { + when(executionContext.response()).thenReturn((ResponseConfiguration) null); + + chatClient.chat(executionContext, AGENT_CONTEXT, runtimeMemory, ChatStreamListener.NOOP); + + assertThat(optionsCaptor.getValue().responseFormat()).isNull(); + } + + @Test + void usesNoopListenerWhenCallerPassesNull() { + chatClient.chat(executionContext, AGENT_CONTEXT, runtimeMemory, null); + + // no NPE on the underlying ChatModelApi means the null listener was substituted with NOOP + } +} diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImplTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImplTest.java new file mode 100644 index 00000000000..a2500aa150f --- /dev/null +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImplTest.java @@ -0,0 +1,90 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicConnection; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicModel; +import io.camunda.connector.agenticai.aiagent.model.request.provider.ProviderConfiguration; +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class ChatModelApiRegistryImplTest { + + @Test + void resolvesFactoryByProviderType() { + final var anthropicFactory = factoryFor(AnthropicProviderConfiguration.ANTHROPIC_ID); + final var resolvedApi = mock(ChatModelApi.class); + when(anthropicFactory.create(any())).thenReturn(resolvedApi); + + final var otherFactory = factoryFor("other"); + + final var registry = new ChatModelApiRegistryImpl(List.of(anthropicFactory, otherFactory)); + + assertThat(registry.resolve(validAnthropicConfig())).isSameAs(resolvedApi); + } + + @Test + void resolvesFactoryClaimingMultipleProviderTypes() { + final var multiTypeFactory = + factoryFor(AnthropicProviderConfiguration.ANTHROPIC_ID, "openai", "bedrock"); + final var resolvedApi = mock(ChatModelApi.class); + when(multiTypeFactory.create(any())).thenReturn(resolvedApi); + + final var registry = new ChatModelApiRegistryImpl(List.of(multiTypeFactory)); + + assertThat(registry.resolve(validAnthropicConfig())).isSameAs(resolvedApi); + } + + @Test + void throwsWhenNoFactoryRegisteredForType() { + final var registry = new ChatModelApiRegistryImpl(List.of()); + + assertThatThrownBy(() -> registry.resolve(validAnthropicConfig())) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining( + "No chat model API factory registered for provider type '%s'" + .formatted(AnthropicProviderConfiguration.ANTHROPIC_ID)); + } + + @Test + void throwsWhenTwoFactoriesClaimSameProviderType() { + final var first = factoryFor("duplicate"); + final var second = factoryFor("duplicate"); + + assertThatThrownBy(() -> new ChatModelApiRegistryImpl(List.of(first, second))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Two chat model API factories claim provider type 'duplicate'"); + } + + @SuppressWarnings("unchecked") + private static ChatModelApiFactory factoryFor(String... providerTypes) { + final ChatModelApiFactory factory = mock(ChatModelApiFactory.class); + when(factory.supportedProviderTypes()).thenReturn(Set.of(providerTypes)); + when(factory.apiFamily()).thenReturn("test"); + return factory; + } + + private static AnthropicProviderConfiguration validAnthropicConfig() { + return new AnthropicProviderConfiguration( + new AnthropicConnection( + null, + new AnthropicAuthentication("api-key"), + null, + new AnthropicModel("claude", null))); + } +} diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkAdapterTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkAdapterTest.java deleted file mode 100644 index 0cd1c150caa..00000000000 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JAiFrameworkAdapterTest.java +++ /dev/null @@ -1,316 +0,0 @@ -/* - * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH - * under one or more contributor license agreements. Licensed under a proprietary license. - * See the License.txt file for more information. You may not use this file - * except in compliance with the proprietary license. - */ -package io.camunda.connector.agenticai.aiagent.framework.langchain4j; - -import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.assistantMessage; -import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.systemMessage; -import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.userMessage; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.when; - -import dev.langchain4j.agent.tool.ToolSpecification; -import dev.langchain4j.data.message.ChatMessage; -import dev.langchain4j.exception.ModelNotFoundException; -import dev.langchain4j.exception.UnresolvedModelServerException; -import dev.langchain4j.model.chat.ChatModel; -import dev.langchain4j.model.chat.request.ChatRequest; -import dev.langchain4j.model.chat.request.ResponseFormatType; -import dev.langchain4j.model.chat.request.json.JsonObjectSchema; -import dev.langchain4j.model.chat.response.ChatResponse; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.jsonschema.JsonSchemaConverter; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverter; -import io.camunda.connector.agenticai.aiagent.memory.runtime.DefaultRuntimeMemory; -import io.camunda.connector.agenticai.aiagent.memory.runtime.RuntimeMemory; -import io.camunda.connector.agenticai.aiagent.model.AgentContext; -import io.camunda.connector.agenticai.aiagent.model.AgentExecutionContext; -import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; -import io.camunda.connector.agenticai.aiagent.model.AgentState; -import io.camunda.connector.agenticai.aiagent.model.request.OutboundConnectorResponseConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.ResponseConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.ResponseFormatConfiguration.JsonResponseFormatConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.ResponseFormatConfiguration.TextResponseFormatConfiguration; -import io.camunda.connector.agenticai.model.message.AssistantMessage; -import io.camunda.connector.agenticai.model.message.Message; -import io.camunda.connector.agenticai.model.tool.ToolDefinition; -import io.camunda.connector.api.error.ConnectorException; -import java.util.List; -import java.util.Map; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.NullAndEmptySource; -import org.junit.jupiter.params.provider.ValueSource; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -class Langchain4JAiFrameworkAdapterTest { - - private static final String SYSTEM_PROMPT = "You are a helpful assistant. Be nice."; - private static final String USER_PROMPT = "Write a haiku about the sea."; - private static final String RESPONSE_TEXT = - "Endless waves whisper | moonlight dances on the tide | secrets drift below."; - - private static final List INPUT_MESSAGES = - List.of(systemMessage(SYSTEM_PROMPT), userMessage(USER_PROMPT)); - - private static final List L4J_MESSAGES = - List.of( - dev.langchain4j.data.message.SystemMessage.systemMessage(SYSTEM_PROMPT), - dev.langchain4j.data.message.UserMessage.userMessage(USER_PROMPT)); - - private static final AssistantMessage ASSISTANT_MESSAGE = assistantMessage(RESPONSE_TEXT); - - private static final List TOOL_DEFINITIONS = - List.of( - ToolDefinition.builder().name("GetTime").description("Returns the current time").build()); - - private static final List L4J_TOOL_SPECIFICATIONS = - List.of( - ToolSpecification.builder() - .name("GetTime") - .description("Returns the current time") - .build()); - - private static final AgentContext AGENT_CONTEXT = - AgentContext.empty() - .withState(AgentState.READY) - .withToolDefinitions(TOOL_DEFINITIONS) - .withMetrics( - AgentMetrics.empty() - .withModelCalls(5) - .withTokenUsage( - AgentMetrics.TokenUsage.empty() - .withInputTokenCount(10) - .withOutputTokenCount(20))); - - @Mock private ChatModelFactory chatModelFactory; - @Mock private ChatMessageConverter chatMessageConverter; - @Mock private ToolSpecificationConverter toolSpecificationConverter; - @Mock private JsonSchemaConverter jsonSchemaConverter; - - @Mock private ChatModel chatModel; - @Mock private ChatResponse chatResponse; - - @Captor private ArgumentCaptor chatRequestCaptor; - - private RuntimeMemory runtimeMemory; - private Langchain4JAiFrameworkAdapter adapter; - - @BeforeEach - void setUp() { - runtimeMemory = new DefaultRuntimeMemory(); - runtimeMemory.addMessages(INPUT_MESSAGES); - when(chatMessageConverter.map(runtimeMemory.filteredMessages())).thenReturn(L4J_MESSAGES); - - when(toolSpecificationConverter.asToolSpecifications(TOOL_DEFINITIONS)) - .thenReturn(L4J_TOOL_SPECIFICATIONS); - - when(chatModelFactory.createChatModel(any())).thenReturn(chatModel); - when(chatModel.chat(chatRequestCaptor.capture())).thenReturn(chatResponse); - when(chatMessageConverter.toAssistantMessage(chatResponse)) - .thenReturn( - ASSISTANT_MESSAGE.withUsage( - AgentMetrics.TokenUsage.builder().inputTokenCount(5).outputTokenCount(6).build())); - - adapter = - new Langchain4JAiFrameworkAdapter( - chatModelFactory, - chatMessageConverter, - toolSpecificationConverter, - jsonSchemaConverter); - } - - @Test - void modelRequestContainsMessagesAndToolSpecifications() { - adapter.executeChatRequest(createExecutionContext(), AGENT_CONTEXT, runtimeMemory); - - final var chatRequest = chatRequestCaptor.getValue(); - assertThat(chatRequest.messages()).containsExactlyElementsOf(L4J_MESSAGES); - assertThat(chatRequest.toolSpecifications()).containsExactlyElementsOf(L4J_TOOL_SPECIFICATIONS); - } - - @Test - void wrapsUnderlyingExceptionsInConnectorException() { - reset(chatModel, chatResponse, chatMessageConverter); - when(chatMessageConverter.map(runtimeMemory.filteredMessages())).thenReturn(L4J_MESSAGES); - - final var cause = new ModelNotFoundException("Model 'dummy' was not found"); - doThrow(cause).when(chatModel).chat(any(ChatRequest.class)); - - assertThatThrownBy( - () -> - adapter.executeChatRequest(createExecutionContext(), AGENT_CONTEXT, runtimeMemory)) - .isInstanceOfSatisfying( - ConnectorException.class, - ex -> { - assertThat(ex.getErrorCode()).isEqualTo("FAILED_MODEL_CALL"); - assertThat(ex.getMessage()) - .isEqualTo("Model call failed: Model 'dummy' was not found"); - assertThat(ex.getCause()).isEqualTo(cause); - }); - } - - @Test - void usesExceptionClassIfNoMessageIncludedInException() { - reset(chatModel, chatResponse, chatMessageConverter); - when(chatMessageConverter.map(runtimeMemory.filteredMessages())).thenReturn(L4J_MESSAGES); - - final var cause = new UnresolvedModelServerException((String) null); - doThrow(cause).when(chatModel).chat(any(ChatRequest.class)); - - assertThatThrownBy( - () -> - adapter.executeChatRequest(createExecutionContext(), AGENT_CONTEXT, runtimeMemory)) - .isInstanceOfSatisfying( - ConnectorException.class, - ex -> { - assertThat(ex.getErrorCode()).isEqualTo("FAILED_MODEL_CALL"); - assertThat(ex.getMessage()) - .isEqualTo("Model call failed: UnresolvedModelServerException"); - assertThat(ex.getCause()).isEqualTo(cause); - }); - } - - @Test - void doesNotExplicitelyConfigureResponseFormatWhenTextFormatIsConfigured() { - adapter.executeChatRequest(createExecutionContext(), AGENT_CONTEXT, runtimeMemory); - - final var chatRequest = chatRequestCaptor.getValue(); - assertThat(chatRequest.responseFormat()).isNull(); - } - - @Test - void doesNotExplicitelyConfigureResponseFormatWhenResponseConfigurationIsMissing() { - adapter.executeChatRequest(createExecutionContext(null), AGENT_CONTEXT, runtimeMemory); - - final var chatRequest = chatRequestCaptor.getValue(); - assertThat(chatRequest.responseFormat()).isNull(); - } - - @Test - void doesNotExplicitelyConfigureResponseFormatWhenResponseFormatConfigurationIsMissing() { - adapter.executeChatRequest( - createExecutionContext(new OutboundConnectorResponseConfiguration(null, false)), - AGENT_CONTEXT, - runtimeMemory); - - final var chatRequest = chatRequestCaptor.getValue(); - assertThat(chatRequest.responseFormat()).isNull(); - } - - @Test - void requestsJsonResponseWhenConfigured() { - adapter.executeChatRequest( - createExecutionContext( - new OutboundConnectorResponseConfiguration( - new JsonResponseFormatConfiguration(null, null), false)), - AGENT_CONTEXT, - runtimeMemory); - - final var chatRequest = chatRequestCaptor.getValue(); - assertThat(chatRequest.responseFormat().type()).isEqualTo(ResponseFormatType.JSON); - assertThat(chatRequest.responseFormat().jsonSchema()).isNull(); - } - - @Test - void requestsJsonResponseWithSchemaWhenConfigured() { - final Map schema = Map.of("type", "object", "description", "My schema"); - final var schemaName = "Foo"; - - final var jsonObjectSchema = JsonObjectSchema.builder().description("My schema").build(); - when(jsonSchemaConverter.mapToSchema(schema)).thenReturn(jsonObjectSchema); - - adapter.executeChatRequest( - createExecutionContext( - new OutboundConnectorResponseConfiguration( - new JsonResponseFormatConfiguration(schema, schemaName), false)), - AGENT_CONTEXT, - runtimeMemory); - - final var chatRequest = chatRequestCaptor.getValue(); - assertThat(chatRequest.responseFormat().type()).isEqualTo(ResponseFormatType.JSON); - assertThat(chatRequest.responseFormat().jsonSchema()).isNotNull(); - assertThat(chatRequest.responseFormat().jsonSchema().name()).isEqualTo(schemaName); - assertThat(chatRequest.responseFormat().jsonSchema().rootElement()).isEqualTo(jsonObjectSchema); - } - - @ParameterizedTest - @NullAndEmptySource - @ValueSource(strings = {" "}) - void usesDefaultSchemaNameIfNullOrBlank(String schemaName) { - final Map schema = Map.of("type", "object", "description", "My schema"); - - final var jsonObjectSchema = JsonObjectSchema.builder().description("My schema").build(); - when(jsonSchemaConverter.mapToSchema(schema)).thenReturn(jsonObjectSchema); - - adapter.executeChatRequest( - createExecutionContext( - new OutboundConnectorResponseConfiguration( - new JsonResponseFormatConfiguration(schema, schemaName), false)), - AGENT_CONTEXT, - runtimeMemory); - - final var chatRequest = chatRequestCaptor.getValue(); - assertThat(chatRequest.responseFormat().type()).isEqualTo(ResponseFormatType.JSON); - assertThat(chatRequest.responseFormat().jsonSchema()).isNotNull(); - assertThat(chatRequest.responseFormat().jsonSchema().name()).isEqualTo("Response"); - assertThat(chatRequest.responseFormat().jsonSchema().rootElement()).isEqualTo(jsonObjectSchema); - } - - @Test - void incrementsMetricsFromResponse() { - final var adapterResponse = - adapter.executeChatRequest(createExecutionContext(), AGENT_CONTEXT, runtimeMemory); - final var expectedMetrics = - AGENT_CONTEXT - .metrics() - .withModelCalls(6) - .withTokenUsage( - AgentMetrics.TokenUsage.empty() - .withInputTokenCount(15) // 10 from context + 5 from response - .withOutputTokenCount(26)); // 20 from context + 6 from response - - assertThat(adapterResponse.agentContext()) - .usingRecursiveComparison() - .isEqualTo(AGENT_CONTEXT.withMetrics(expectedMetrics)); - } - - @Test - void tokenUsageIsUnchangedIfMissingInResponse() { - when(chatMessageConverter.toAssistantMessage(chatResponse)).thenReturn(ASSISTANT_MESSAGE); - - final var adapterResponse = - adapter.executeChatRequest(createExecutionContext(), AGENT_CONTEXT, runtimeMemory); - - assertThat(adapterResponse.agentContext()) - .usingRecursiveComparison() - .isEqualTo(AGENT_CONTEXT.withMetrics(AGENT_CONTEXT.metrics().withModelCalls(6))); - } - - private AgentExecutionContext createExecutionContext() { - return createExecutionContext( - new OutboundConnectorResponseConfiguration( - new TextResponseFormatConfiguration(false), false)); - } - - private AgentExecutionContext createExecutionContext( - ResponseConfiguration responseConfiguration) { - final var executionContext = mock(AgentExecutionContext.class); - when(executionContext.response()).thenReturn(responseConfiguration); - - return executionContext; - } -} diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApiTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApiTest.java new file mode 100644 index 00000000000..48cb7552780 --- /dev/null +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApiTest.java @@ -0,0 +1,249 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.langchain4j; + +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.assistantMessage; +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.systemMessage; +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.userMessage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.when; + +import dev.langchain4j.agent.tool.ToolSpecification; +import dev.langchain4j.data.message.ChatMessage; +import dev.langchain4j.exception.ModelNotFoundException; +import dev.langchain4j.exception.UnresolvedModelServerException; +import dev.langchain4j.model.chat.ChatModel; +import dev.langchain4j.model.chat.request.ChatRequest; +import dev.langchain4j.model.chat.request.ResponseFormatType; +import dev.langchain4j.model.chat.request.json.JsonObjectSchema; +import dev.langchain4j.model.chat.response.ChatResponse; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatOptions; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; +import io.camunda.connector.agenticai.aiagent.framework.api.ResponseFormat; +import io.camunda.connector.agenticai.aiagent.framework.langchain4j.jsonschema.JsonSchemaConverter; +import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverter; +import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; +import io.camunda.connector.agenticai.model.message.AssistantMessage; +import io.camunda.connector.agenticai.model.message.Message; +import io.camunda.connector.agenticai.model.tool.ToolDefinition; +import io.camunda.connector.api.error.ConnectorException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletionException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class Langchain4JChatModelApiTest { + + private static final String SYSTEM_PROMPT = "You are a helpful assistant. Be nice."; + private static final String USER_PROMPT = "Write a haiku about the sea."; + private static final String RESPONSE_TEXT = + "Endless waves whisper | moonlight dances on the tide | secrets drift below."; + + private static final List INPUT_MESSAGES = + List.of(systemMessage(SYSTEM_PROMPT), userMessage(USER_PROMPT)); + + private static final List L4J_MESSAGES = + List.of( + dev.langchain4j.data.message.SystemMessage.systemMessage(SYSTEM_PROMPT), + dev.langchain4j.data.message.UserMessage.userMessage(USER_PROMPT)); + + private static final AssistantMessage ASSISTANT_MESSAGE = assistantMessage(RESPONSE_TEXT); + + private static final List TOOL_DEFINITIONS = + List.of( + ToolDefinition.builder().name("GetTime").description("Returns the current time").build()); + + private static final List L4J_TOOL_SPECIFICATIONS = + List.of( + ToolSpecification.builder() + .name("GetTime") + .description("Returns the current time") + .build()); + + @Mock private ChatModel chatModel; + @Mock private ChatMessageConverter chatMessageConverter; + @Mock private ToolSpecificationConverter toolSpecificationConverter; + @Mock private JsonSchemaConverter jsonSchemaConverter; + + @Mock private ChatResponse chatResponse; + + @Captor private ArgumentCaptor chatRequestCaptor; + + private Langchain4JChatModelApi api; + + @BeforeEach + void setUp() { + when(chatMessageConverter.map(INPUT_MESSAGES)).thenReturn(L4J_MESSAGES); + when(toolSpecificationConverter.asToolSpecifications(TOOL_DEFINITIONS)) + .thenReturn(L4J_TOOL_SPECIFICATIONS); + when(chatModel.chat(chatRequestCaptor.capture())).thenReturn(chatResponse); + when(chatMessageConverter.toAssistantMessage(chatResponse)) + .thenReturn( + ASSISTANT_MESSAGE.withUsage( + AgentMetrics.TokenUsage.builder().inputTokenCount(5).outputTokenCount(6).build())); + + api = + new Langchain4JChatModelApi( + chatModel, chatMessageConverter, toolSpecificationConverter, jsonSchemaConverter); + } + + @Test + void modelRequestContainsMessagesAndToolSpecifications() { + complete(textOptions()); + + final var chatRequest = chatRequestCaptor.getValue(); + assertThat(chatRequest.messages()).containsExactlyElementsOf(L4J_MESSAGES); + assertThat(chatRequest.toolSpecifications()).containsExactlyElementsOf(L4J_TOOL_SPECIFICATIONS); + } + + @Test + void wrapsUnderlyingExceptionsInConnectorException() { + reset(chatModel, chatResponse, chatMessageConverter); + when(chatMessageConverter.map(INPUT_MESSAGES)).thenReturn(L4J_MESSAGES); + + final var cause = new ModelNotFoundException("Model 'dummy' was not found"); + doThrow(cause).when(chatModel).chat(any(ChatRequest.class)); + + assertThatThrownBy(() -> complete(textOptions()).join()) + .isInstanceOfSatisfying( + CompletionException.class, + wrapper -> + assertThat(wrapper.getCause()) + .isInstanceOfSatisfying( + ConnectorException.class, + ex -> { + assertThat(ex.getErrorCode()).isEqualTo("FAILED_MODEL_CALL"); + assertThat(ex.getMessage()) + .isEqualTo("Model call failed: Model 'dummy' was not found"); + assertThat(ex.getCause()).isEqualTo(cause); + })); + } + + @Test + void usesExceptionClassIfNoMessageIncludedInException() { + reset(chatModel, chatResponse, chatMessageConverter); + when(chatMessageConverter.map(INPUT_MESSAGES)).thenReturn(L4J_MESSAGES); + + final var cause = new UnresolvedModelServerException((String) null); + doThrow(cause).when(chatModel).chat(any(ChatRequest.class)); + + assertThatThrownBy(() -> complete(textOptions()).join()) + .isInstanceOfSatisfying( + CompletionException.class, + wrapper -> + assertThat(wrapper.getCause()) + .isInstanceOfSatisfying( + ConnectorException.class, + ex -> { + assertThat(ex.getErrorCode()).isEqualTo("FAILED_MODEL_CALL"); + assertThat(ex.getMessage()) + .isEqualTo("Model call failed: UnresolvedModelServerException"); + assertThat(ex.getCause()).isEqualTo(cause); + })); + } + + @Test + void doesNotExplicitlyConfigureResponseFormatForText() { + complete(textOptions()); + + assertThat(chatRequestCaptor.getValue().responseFormat()).isNull(); + } + + @Test + void doesNotExplicitlyConfigureResponseFormatWhenNull() { + complete(options(null)); + + assertThat(chatRequestCaptor.getValue().responseFormat()).isNull(); + } + + @Test + void requestsJsonResponseWhenConfigured() { + complete(options(new ResponseFormat.Json(null, null))); + + final var format = chatRequestCaptor.getValue().responseFormat(); + assertThat(format.type()).isEqualTo(ResponseFormatType.JSON); + assertThat(format.jsonSchema()).isNull(); + } + + @Test + void requestsJsonResponseWithSchemaWhenConfigured() { + final Map schema = Map.of("type", "object", "description", "My schema"); + final var schemaName = "Foo"; + + final var jsonObjectSchema = JsonObjectSchema.builder().description("My schema").build(); + when(jsonSchemaConverter.mapToSchema(schema)).thenReturn(jsonObjectSchema); + + complete(options(new ResponseFormat.Json(schemaName, schema))); + + final var format = chatRequestCaptor.getValue().responseFormat(); + assertThat(format.type()).isEqualTo(ResponseFormatType.JSON); + assertThat(format.jsonSchema()).isNotNull(); + assertThat(format.jsonSchema().name()).isEqualTo(schemaName); + assertThat(format.jsonSchema().rootElement()).isEqualTo(jsonObjectSchema); + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource(strings = {" "}) + void usesDefaultSchemaNameIfNullOrBlank(String schemaName) { + final Map schema = Map.of("type", "object", "description", "My schema"); + + final var jsonObjectSchema = JsonObjectSchema.builder().description("My schema").build(); + when(jsonSchemaConverter.mapToSchema(schema)).thenReturn(jsonObjectSchema); + + complete(options(new ResponseFormat.Json(schemaName, schema))); + + final var format = chatRequestCaptor.getValue().responseFormat(); + assertThat(format.type()).isEqualTo(ResponseFormatType.JSON); + assertThat(format.jsonSchema()).isNotNull(); + assertThat(format.jsonSchema().name()).isEqualTo("Response"); + assertThat(format.jsonSchema().rootElement()).isEqualTo(jsonObjectSchema); + } + + @Test + void responseCarriesUsageAndStopReasonFromAssistantMessage() { + final var response = complete(textOptions()).join(); + + assertThat(response.assistantMessage()).isNotNull(); + assertThat(response.usage()) + .isEqualTo( + AgentMetrics.TokenUsage.builder().inputTokenCount(5).outputTokenCount(6).build()); + assertThat(response.errorMessage()).isNull(); + } + + private java.util.concurrent.CompletableFuture< + io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse> + complete(ChatOptions options) { + return api.complete( + new io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest( + INPUT_MESSAGES, null, TOOL_DEFINITIONS), + options, + ChatStreamListener.NOOP); + } + + private static ChatOptions textOptions() { + return options(new ResponseFormat.Text()); + } + + private static ChatOptions options(ResponseFormat responseFormat) { + return new ChatOptions(null, null, null, responseFormat, Map.of()); + } +} diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java index 2931f54e766..b842ede2f44 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java @@ -28,11 +28,13 @@ import io.camunda.connector.agenticai.aiagent.agent.AgentToolsResolver; import io.camunda.connector.agenticai.aiagent.agent.JobWorkerAgentRequestHandler; import io.camunda.connector.agenticai.aiagent.agent.OutboundConnectorAgentRequestHandler; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatClient; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiRegistry; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatMessageConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelFactory; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelHttpProxySupport; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ContentConverter; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.Langchain4JAiFrameworkAdapter; +import io.camunda.connector.agenticai.aiagent.framework.langchain4j.Langchain4JChatModelApiFactory; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.document.DocumentToContentConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.jsonschema.JsonSchemaConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.AnthropicChatModelProvider; @@ -101,6 +103,8 @@ class AgenticAiConnectorsAutoConfigurationTest { AgentLimitsValidator.class, AgentMessagesHandler.class, AgentResponseHandler.class, + ChatModelApiRegistry.class, + ChatClient.class, OutboundConnectorAgentRequestHandler.class, AiAgentFunction.class, JobWorkerAgentRequestHandler.class, @@ -123,7 +127,7 @@ class AgenticAiConnectorsAutoConfigurationTest { JsonSchemaConverter.class, ToolSpecificationConverter.class, ChatMessageConverter.class, - Langchain4JAiFrameworkAdapter.class); + Langchain4JChatModelApiFactory.class); // this will need to be updated in case we support different frameworks private static final List> ALL_BEANS = From 2d0f303369a04a683552a5a0eb8635964830346c Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 14:01:34 +0200 Subject: [PATCH 49/81] refactor(agentic-ai): align ChatClient SPI with the phased plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bring Phase A in line with the agreed phased implementation plan: - ChatClient.chat(...) becomes synchronous and returns ChatClientResult (AgentContext + AssistantMessage). The CompletableFuture from ChatModelApi.complete() stays an internal detail of ChatClientImpl. Metric increment moves into ChatClientImpl. BaseAgentRequestHandler's call site is now a 1:1 swap of the old AiFrameworkChatResponse path. - ChatRequest carries (messages, toolDefinitions, responseFormat) where responseFormat reuses the existing ResponseFormatConfiguration. Removed the SPI-side ResponseFormat type; dropped systemPrompt. - ChatOptions loses responseFormat. - ChatResponse simplifies to a single AssistantMessage field; stop reason / usage / errors live on the message itself. - ChatModelApiFactory keeps a single canonical providerType() and gains configurationType() back (defensive runtime check before unchecked cast). The bridge is now registered as 6 separate factory beans, one per ProviderConfiguration discriminator, each composing the matching ChatModelProvider. Bean parameters use the parameterized provider type so customer-provided ChatModelProvider beans (covered by an existing test) still override the default 1:1 without extra wiring. - ADR text updated: apiId → messageId (already renamed in earlier Phase 0 work), error-class table reflects ChatResponse without a separate errorMessage field, Phase 1 SPI list now mentions the ChatClientResult facade and the per-provider bridge bean pattern. 1290 unit tests + the wire-format e2e tests stay green. --- .../adr/004-replace-langchain4j-framework.md | 10 +- .../agent/BaseAgentRequestHandler.java | 31 +---- .../aiagent/framework/ChatClientImpl.java | 69 ++++++----- .../framework/ChatModelApiRegistryImpl.java | 36 +++--- .../aiagent/framework/api/ChatClient.java | 17 +-- .../framework/api/ChatClientResult.java | 21 ++++ .../framework/api/ChatModelApiFactory.java | 23 ++-- .../framework/api/ChatModelApiRegistry.java | 5 +- .../aiagent/framework/api/ChatOptions.java | 7 -- .../aiagent/framework/api/ChatRequest.java | 21 ++-- .../aiagent/framework/api/ChatResponse.java | 19 +-- .../aiagent/framework/api/ResponseFormat.java | 33 ------ .../langchain4j/Langchain4JChatModelApi.java | 25 ++-- .../Langchain4JChatModelApiFactory.java | 57 ++++----- ...icAiLangchain4JFrameworkConfiguration.java | 108 +++++++++++++++++- .../JobWorkerAgentRequestHandlerTest.java | 24 ++-- ...boundConnectorAgentRequestHandlerTest.java | 24 ++-- .../aiagent/framework/ChatClientImplTest.java | 90 +++++++++++---- .../ChatModelApiRegistryImplTest.java | 20 +--- .../Langchain4JChatModelApiTest.java | 45 +++----- ...nticAiConnectorsAutoConfigurationTest.java | 4 +- 21 files changed, 406 insertions(+), 283 deletions(-) create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClientResult.java delete mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ResponseFormat.java diff --git a/connectors/agentic-ai/docs/adr/004-replace-langchain4j-framework.md b/connectors/agentic-ai/docs/adr/004-replace-langchain4j-framework.md index 7ca8c9a3896..d70f2645bfe 100644 --- a/connectors/agentic-ai/docs/adr/004-replace-langchain4j-framework.md +++ b/connectors/agentic-ai/docs/adr/004-replace-langchain4j-framework.md @@ -185,7 +185,7 @@ Three classes of failure with distinct surface behavior: | Class | Examples | Surface | |----------------------------|--------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------| -| Model-side terminal | `stop_reason: refusal`, content filter, max-tokens hit, malformed tool-use | `CompletableFuture` completes normally with `ChatResponse{stopReason=ERROR, errorMessage, content, usage}` | +| Model-side terminal | `stop_reason: refusal`, content filter, max-tokens hit, malformed tool-use | `CompletableFuture` completes normally with `ChatResponse{assistantMessage{stopReason=ERROR, content, usage, ...}}` | | Transport / SDK / I/O | Connection refused, read timeout, TLS failure, malformed wire response | `CompletableFuture` completes exceptionally with `ConnectorException(ERROR_CODE_FAILED_MODEL_CALL, ...)` | | Auth / config | Bad API key, region not enabled, model not found | `CompletableFuture` completes exceptionally with a distinct error code | @@ -233,7 +233,7 @@ public record AssistantMessage( List content, List toolCalls, @Nullable String modelId, // NEW - @Nullable String apiId, // NEW + @Nullable String messageId, // NEW (provider-assigned message id) @Nullable StopReason stopReason, // NEW @Nullable TokenUsage usage, // NEW (per-message) Map metadata // existing — escape hatch @@ -510,7 +510,7 @@ Two phases: ### Phase 0 — Domain model extensions (additive, behavior-preserving) * Add `ReasoningContent` to the `Content` sealed hierarchy. -* Add optional `modelId`, `apiId`, `stopReason`, `usage` fields to `AssistantMessage`. +* Add optional `modelId`, `messageId`, `stopReason`, `usage` fields to `AssistantMessage`. * Add optional `contentBlocks` field to `ToolCallResult`. * Add `cacheReadInputTokens`, `cacheCreationInputTokens`, `reasoningTokens` to `TokenUsage`. * Drop `AiFrameworkChatResponse#rawChatResponse()`. @@ -522,7 +522,9 @@ No call-site changes. Existing tests pass with no behavior change. ### Phase 1 — Complete replacement (one shipping unit) * New SPI: `ChatModelApiFactory`, `ChatModelApiRegistry`, `ChatModelApi`, `ChatClient` - facade, `ChatModelEvent` sealed hierarchy, `ChatStreamListener`. + facade returning `ChatClientResult`, `ChatModelEvent` sealed hierarchy, `ChatStreamListener`. + Multi-provider bridges (e.g. LangChain4j) register one `ChatModelApiFactory` bean per + `providerType()` discriminator. * Capability matrix YAML + resolution chain (config override → exact id / alias → pattern → conservative defaults). * `ToolCallResultStrategy` (multimodal-native and user-message-fallback policies). diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/BaseAgentRequestHandler.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/BaseAgentRequestHandler.java index 55dae1f7076..721cbedbd65 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/BaseAgentRequestHandler.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/BaseAgentRequestHandler.java @@ -10,7 +10,6 @@ import io.camunda.connector.agenticai.aiagent.agent.AgentInitializationResult.AgentDiscoveryInProgressInitializationResult; import io.camunda.connector.agenticai.aiagent.agent.AgentInitializationResult.AgentResponseInitializationResult; import io.camunda.connector.agenticai.aiagent.framework.api.ChatClient; -import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationSession; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStore; @@ -19,7 +18,6 @@ import io.camunda.connector.agenticai.aiagent.memory.runtime.MessageWindowRuntimeMemory; import io.camunda.connector.agenticai.aiagent.model.AgentContext; import io.camunda.connector.agenticai.aiagent.model.AgentExecutionContext; -import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.agenticai.aiagent.model.AgentResponse; import io.camunda.connector.agenticai.aiagent.model.request.MemoryConfiguration; import io.camunda.connector.agenticai.aiagent.tool.GatewayToolHandlerRegistry; @@ -30,8 +28,6 @@ import io.camunda.connector.api.outbound.JobCompletionFailure; import java.util.List; import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -173,17 +169,11 @@ private AgentResponse processConversation( // dispatch via the chat client SPI; bridge / native impls live behind it LOGGER.debug("Executing chat request"); - final var chatResponse = - joinChat( - chatClient.chat( - executionContext, agentContext, runtimeMemory, ChatStreamListener.NOOP)); - final var usage = - chatResponse.usage() != null ? chatResponse.usage() : AgentMetrics.TokenUsage.empty(); - agentContext = - agentContext.withMetrics( - agentContext.metrics().incrementModelCalls(1).incrementTokenUsage(usage)); + final var chatClientResult = + chatClient.chat(executionContext, agentContext, runtimeMemory, ChatStreamListener.NOOP); + agentContext = chatClientResult.agentContext(); - final var assistantMessage = chatResponse.assistantMessage(); + final var assistantMessage = chatClientResult.assistantMessage(); LOGGER.debug( "Received assistant message containing {} tool call requests", assistantMessage.toolCalls() != null ? assistantMessage.toolCalls().size() : 0); @@ -207,19 +197,6 @@ private AgentResponse processConversation( executionContext, agentContext, assistantMessage, processVariableToolCalls); } - private static ChatResponse joinChat(CompletableFuture future) { - try { - return future.join(); - } catch (CompletionException e) { - // unwrap so callers see the original ConnectorException (or other RuntimeException) - // rather than the CompletableFuture wrapper - if (e.getCause() instanceof RuntimeException re) { - throw re; - } - throw e; - } - } - protected abstract boolean modelCallPrerequisitesFulfilled( C executionContext, AgentContext agentContext, List addedUserMessages); diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImpl.java index 8425f724c1b..8518712a92e 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImpl.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImpl.java @@ -7,20 +7,19 @@ package io.camunda.connector.agenticai.aiagent.framework; import io.camunda.connector.agenticai.aiagent.framework.api.ChatClient; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatClientResult; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiRegistry; import io.camunda.connector.agenticai.aiagent.framework.api.ChatOptions; import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; -import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; -import io.camunda.connector.agenticai.aiagent.framework.api.ResponseFormat; import io.camunda.connector.agenticai.aiagent.memory.runtime.RuntimeMemory; import io.camunda.connector.agenticai.aiagent.model.AgentContext; import io.camunda.connector.agenticai.aiagent.model.AgentExecutionContext; +import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.agenticai.aiagent.model.request.ResponseConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.ResponseFormatConfiguration.JsonResponseFormatConfiguration; import java.util.Map; -import java.util.concurrent.CompletableFuture; -import org.springframework.lang.Nullable; +import java.util.Optional; +import java.util.concurrent.CompletionException; public class ChatClientImpl implements ChatClient { @@ -31,34 +30,50 @@ public ChatClientImpl(ChatModelApiRegistry registry) { } @Override - public CompletableFuture chat( + public ChatClientResult chat( AgentExecutionContext executionContext, AgentContext agentContext, RuntimeMemory runtimeMemory, ChatStreamListener listener) { - try { - final var api = registry.resolve(executionContext.provider()); - final var request = - new ChatRequest(runtimeMemory.filteredMessages(), null, agentContext.toolDefinitions()); - final var options = - new ChatOptions( - null, null, null, toResponseFormat(executionContext.response()), Map.of()); - return api.complete(request, options, listener != null ? listener : ChatStreamListener.NOOP); - } catch (RuntimeException e) { - return CompletableFuture.failedFuture(e); - } + final var api = registry.resolve(executionContext.provider()); + final var request = + new ChatRequest( + runtimeMemory.filteredMessages(), + agentContext.toolDefinitions(), + Optional.ofNullable(executionContext.response()) + .map(ResponseConfiguration::format) + .orElse(null)); + final var options = new ChatOptions(null, null, null, Map.of()); + + final var chatResponse = + joinChat( + api.complete(request, options, listener != null ? listener : ChatStreamListener.NOOP)); + + final var assistantMessage = chatResponse.assistantMessage(); + final var usage = + assistantMessage.usage() != null + ? assistantMessage.usage() + : AgentMetrics.TokenUsage.empty(); + final var updatedAgentContext = + agentContext.withMetrics( + agentContext.metrics().incrementModelCalls(1).incrementTokenUsage(usage)); + + return new ChatClientResult(updatedAgentContext, assistantMessage); } - private static @Nullable ResponseFormat toResponseFormat( - @Nullable ResponseConfiguration response) { - if (response == null || response.format() == null) { - return null; - } - if (response.format() instanceof JsonResponseFormatConfiguration json) { - return new ResponseFormat.Json(json.schemaName(), json.schema()); + private static io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse joinChat( + java.util.concurrent.CompletableFuture< + io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse> + future) { + try { + return future.join(); + } catch (CompletionException e) { + // unwrap so callers see the original ConnectorException (or other RuntimeException) rather + // than the CompletableFuture wrapper + if (e.getCause() instanceof RuntimeException re) { + throw re; + } + throw e; } - // TextResponseFormatConfiguration → null preserves "no explicit format on the wire" - // behaviour. Implementations choose their default text mode. - return null; } } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImpl.java index ec30f2c5dd6..0b4f66b3e3d 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImpl.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImpl.java @@ -30,21 +30,20 @@ private static Map> indexByProviderType( List> factories) { final Map> index = new HashMap<>(); for (ChatModelApiFactory factory : factories) { - for (String providerType : factory.supportedProviderTypes()) { - final var existing = index.get(providerType); - if (existing != null) { - throw new IllegalStateException( - "Two chat model API factories claim provider type '%s': %s and %s. To override the default factory, exclude the built-in bean (e.g. via @ConditionalOnMissingBean) before contributing your own." - .formatted( - providerType, existing.getClass().getName(), factory.getClass().getName())); - } - index.put(providerType, factory); - LOGGER.debug( - "Registered chat model API factory for provider type '{}': {} (apiFamily={})", - providerType, - factory.getClass().getName(), - factory.apiFamily()); + final var providerType = factory.providerType(); + final var existing = index.get(providerType); + if (existing != null) { + throw new IllegalStateException( + "Two chat model API factories claim provider type '%s': %s and %s. To override the default factory, exclude the built-in bean (e.g. via @ConditionalOnMissingBean) before contributing your own." + .formatted( + providerType, existing.getClass().getName(), factory.getClass().getName())); } + index.put(providerType, factory); + LOGGER.debug( + "Registered chat model API factory for provider type '{}': {} (apiFamily={})", + providerType, + factory.getClass().getName(), + factory.apiFamily()); } return Map.copyOf(index); } @@ -58,6 +57,15 @@ public ChatModelApi resolve(ProviderConfiguration configuration) { throw new IllegalStateException( "No chat model API factory registered for provider type '%s'".formatted(providerType)); } + if (!factory.configurationType().isInstance(configuration)) { + throw new IllegalStateException( + "Chat model API factory %s claims provider type '%s' but expects configurationType %s, got %s" + .formatted( + factory.getClass().getName(), + providerType, + factory.configurationType().getName(), + configuration.getClass().getName())); + } return ((ChatModelApiFactory) factory).create(configuration); } } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClient.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClient.java index 51a3253759f..42710259a9d 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClient.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClient.java @@ -9,23 +9,24 @@ import io.camunda.connector.agenticai.aiagent.memory.runtime.RuntimeMemory; import io.camunda.connector.agenticai.aiagent.model.AgentContext; import io.camunda.connector.agenticai.aiagent.model.AgentExecutionContext; -import java.util.concurrent.CompletableFuture; /** - * High-level facade called by {@code BaseAgentRequestHandler} once Phase 1 lands. Resolves the - * model and capabilities via {@link ChatModelApiRegistry}, applies the tool-result strategy, - * assembles the {@link ChatRequest} from the runtime memory, and dispatches to {@link - * ChatModelApi#complete}. + * High-level facade called by {@code BaseAgentRequestHandler}. Resolves the {@link ChatModelApi} + * for the request, applies the tool-result strategy, assembles a {@link ChatRequest} + {@link + * ChatOptions} from the runtime memory and the execution / agent context, dispatches to {@link + * ChatModelApi#complete}, joins the resulting future, increments {@link AgentContext#metrics()} + * based on the assistant message, and wraps everything in a {@link ChatClientResult}. * - *

    Replaces today's {@code AiFrameworkAdapter} call site. Mirroring its signature keeps the - * cutover diff small. + *

    The asynchronous nature of {@link ChatModelApi#complete} is an implementation detail — callers + * see a synchronous facade matching the previous {@code AiFrameworkAdapter} contract. In-process + * observability hooks attach via {@link ChatStreamListener}; the public surface is blocking. * *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via * ChatModelApiRegistry. */ public interface ChatClient { - CompletableFuture chat( + ChatClientResult chat( AgentExecutionContext executionContext, AgentContext agentContext, RuntimeMemory runtimeMemory, diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClientResult.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClientResult.java new file mode 100644 index 00000000000..8f0fae783de --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClientResult.java @@ -0,0 +1,21 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.api; + +import io.camunda.connector.agenticai.aiagent.model.AgentContext; +import io.camunda.connector.agenticai.model.message.AssistantMessage; + +/** + * Result of {@link ChatClient#chat}. Carries the agent context with model-call metrics and token + * usage already incremented, plus the assistant message produced by the underlying {@link + * ChatModelApi}. Replaces {@code AiFrameworkChatResponse} at the {@code BaseAgentRequestHandler} + * call site so the cutover stays a 1:1 swap. + * + *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + * ChatModelApiRegistry. + */ +public record ChatClientResult(AgentContext agentContext, AssistantMessage assistantMessage) {} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java index e4531731f9e..4b297439dae 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java @@ -7,16 +7,21 @@ package io.camunda.connector.agenticai.aiagent.framework.api; import io.camunda.connector.agenticai.aiagent.model.request.provider.ProviderConfiguration; -import java.util.Set; /** - * Stateless factory that produces per-job {@link ChatModelApi} instances for a single wire-protocol - * family ({@code anthropic-messages}, {@code bedrock-converse}, {@code openai-responses}, {@code - * openai-completions}, {@code google-genai}). + * Stateless factory that produces per-job {@link ChatModelApi} instances for a single {@link + * ProviderConfiguration#providerType()} discriminator. The bridge for a multi-provider framework + * (e.g. LangChain4j) is registered as one factory bean per discriminator it handles. * - *

    The {@link ChatModelApiRegistry} indexes factories by the {@link ProviderConfiguration#type - * providerType} strings each factory claims via {@link #supportedProviderTypes} and dispatches by - * exact match. {@link #apiFamily} is informational — used in logs and stream events. + *

    The {@link ChatModelApiRegistry} indexes factories by {@link #providerType()} and dispatches + * by exact match. Two factories claiming the same discriminator fail at startup; user overrides + * happen via {@code @ConditionalOnMissingBean} on the built-in bean rather than silent shadowing. + * + *

    {@link #apiFamily()} is informational telemetry (logs, {@link + * io.camunda.connector.agenticai.aiagent.framework.api.event.ChatModelEvent.StartEvent}) — not used + * for routing. {@link #configurationType()} acts as a defensive runtime check before the registry's + * unchecked cast — a friendlier error than {@link ClassCastException} when a factory is + * accidentally registered against the wrong discriminator. * *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via * ChatModelApiRegistry. @@ -25,9 +30,11 @@ */ public interface ChatModelApiFactory { + String providerType(); + String apiFamily(); - Set supportedProviderTypes(); + Class configurationType(); ChatModelApi create(C configuration); } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java index 0eb1915d7d8..041d5f06552 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java @@ -10,9 +10,8 @@ /** * Resolves a {@link ChatModelApi} for a given {@link ProviderConfiguration}. Composed of all - * registered {@link ChatModelApiFactory} beans, indexed by the {@link ProviderConfiguration#type - * providerType} strings each factory advertises via {@link - * ChatModelApiFactory#supportedProviderTypes}; lookup is exact-match on the configuration's + * registered {@link ChatModelApiFactory} beans, indexed by the strings each factory advertises via + * {@link ChatModelApiFactory#providerTypes()}; lookup is exact-match on the configuration's * provider type. * *

    Unknown provider types fail fast — there is no factory to resolve, so requests can never diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java index ca3c03226a8..92a46e6710c 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java @@ -28,12 +28,6 @@ * caller-supplied value wins, otherwise the resolved {@link ModelCapabilities#maxOutputTokens()} is * used as a fallback, otherwise the implementation supplies its own per-API default. * - *

    Note on {@link #responseFormat}: {@code null} leaves the provider default in - * place (matches today's behaviour, where no explicit format is sent for the text case). - * Implementations translate the value onto the provider's native shape; providers without a native - * structured-output mode (Anthropic Messages today) treat {@link ResponseFormat.Json} as - * best-effort and rely on the system prompt to constrain output. - * *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via * ChatModelApiRegistry. */ @@ -41,5 +35,4 @@ public record ChatOptions( @Nullable Integer maxOutputTokens, @Nullable ReasoningConfig reasoning, @Nullable CacheRetention cacheRetention, - @Nullable ResponseFormat responseFormat, Map providerOptions) {} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java index d992e605fca..878954e0759 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java @@ -6,20 +6,27 @@ */ package io.camunda.connector.agenticai.aiagent.framework.api; +import io.camunda.connector.agenticai.aiagent.model.request.ResponseFormatConfiguration; import io.camunda.connector.agenticai.model.message.Message; import io.camunda.connector.agenticai.model.tool.ToolDefinition; import java.util.List; import org.springframework.lang.Nullable; /** - * Inputs to a {@link ChatModelApi#complete} call assembled by {@code ChatClient}. Carries the - * conversation messages, an optional system prompt, and the resolved tool definitions; per-call - * tunables (max output tokens, stop sequences, reasoning, cache retention, vendor escape hatches) - * live on {@link ChatOptions} so a request can be reused while options vary. + * Inputs to a {@link ChatModelApi#complete} call assembled by {@code ChatClient}: the conversation + * messages (system message inline at the head when present), resolved tool definitions, and the + * requested response format. Per-call tunables (max output tokens, reasoning, cache retention, + * vendor escape hatches) live on {@link ChatOptions} so a request can be reused while options vary. + * + *

    {@code responseFormat} reuses the connector-config type {@link ResponseFormatConfiguration} + * directly. Implementations translate the {@code Json}/{@code Text} variants onto the provider's + * native shape; providers without a native structured-output mode (Anthropic Messages today) treat + * the JSON variant as best-effort and rely on the system prompt to constrain output. * *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via - * ChatModelApiRegistry — concrete fields will be finalised when the first native {@code - * ChatModelApi} implementation lands. + * ChatModelApiRegistry. */ public record ChatRequest( - List messages, @Nullable String systemPrompt, List tools) {} + List messages, + List toolDefinitions, + @Nullable ResponseFormatConfiguration responseFormat) {} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatResponse.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatResponse.java index 730374e57b9..eec24bfa9ee 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatResponse.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatResponse.java @@ -6,23 +6,16 @@ */ package io.camunda.connector.agenticai.aiagent.framework.api; -import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.agenticai.model.message.AssistantMessage; -import io.camunda.connector.agenticai.model.message.StopReason; -import org.springframework.lang.Nullable; /** - * Output of {@link ChatModelApi#complete}. Carries the assembled assistant message plus a - * normalised {@link StopReason} and per-call {@link AgentMetrics.TokenUsage}. {@code errorMessage} - * is populated only for model-side terminal failures (refusal, content filter, malformed tool-use) - * where {@code stopReason == ERROR}; transport / SDK / auth failures complete the future - * exceptionally instead. + * Output of {@link ChatModelApi#complete}. Carries the assembled assistant message; {@link + * AssistantMessage#stopReason()} and {@link AssistantMessage#usage()} convey the normalized stop + * reason and per-call token usage. Model-side terminal failures (refusal, content filter, malformed + * tool-use) populate the message with {@code stopReason = ERROR}; transport / SDK / auth failures + * complete the future exceptionally instead. * *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via * ChatModelApiRegistry. */ -public record ChatResponse( - AssistantMessage assistantMessage, - @Nullable StopReason stopReason, - @Nullable AgentMetrics.TokenUsage usage, - @Nullable String errorMessage) {} +public record ChatResponse(AssistantMessage assistantMessage) {} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ResponseFormat.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ResponseFormat.java deleted file mode 100644 index 8c62c7ae949..00000000000 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ResponseFormat.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH - * under one or more contributor license agreements. Licensed under a proprietary license. - * See the License.txt file for more information. You may not use this file - * except in compliance with the proprietary license. - */ -package io.camunda.connector.agenticai.aiagent.framework.api; - -import java.util.Map; -import org.springframework.lang.Nullable; - -/** - * Universal response-format hint passed via {@link ChatOptions#responseFormat()}. {@link Text} - * requests free-form text output; {@link Json} requests JSON, optionally constrained by a JSON - * Schema. {@code null} on {@link ChatOptions} leaves the provider default in place — this matches - * today's behaviour where no explicit format is sent for the text case. - * - *

    The {@link Json#schema()} payload is the JSON Schema wire format ({@code Map}) - * as supplied by the connector configuration; implementations translate it onto the provider's - * native shape (e.g. OpenAI {@code response_format.json_schema}, Google {@code responseSchema}). - * Providers without a native structured-output mode (Anthropic Messages today) treat {@link Json} - * as best-effort and rely on the system prompt to constrain output. - * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via - * ChatModelApiRegistry. - */ -public sealed interface ResponseFormat permits ResponseFormat.Text, ResponseFormat.Json { - - record Text() implements ResponseFormat {} - - record Json(@Nullable String schemaName, @Nullable Map schema) - implements ResponseFormat {} -} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApi.java index 968428f931f..33dfac1c29b 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApi.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApi.java @@ -10,6 +10,7 @@ import dev.langchain4j.model.chat.ChatModel; import dev.langchain4j.model.chat.request.ChatRequest; +import dev.langchain4j.model.chat.request.ResponseFormat; import dev.langchain4j.model.chat.request.ResponseFormatType; import dev.langchain4j.model.chat.request.json.JsonSchema; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; @@ -18,9 +19,10 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; -import io.camunda.connector.agenticai.aiagent.framework.api.ResponseFormat; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.jsonschema.JsonSchemaConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverter; +import io.camunda.connector.agenticai.aiagent.model.request.ResponseFormatConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.ResponseFormatConfiguration.JsonResponseFormatConfiguration; import io.camunda.connector.api.error.ConnectorException; import java.util.List; import java.util.Optional; @@ -80,12 +82,12 @@ public CompletableFuture complete( try { final var l4jMessages = chatMessageConverter.map(request.messages()); final var toolSpecifications = - toolSpecificationConverter.asToolSpecifications(request.tools()); + toolSpecificationConverter.asToolSpecifications(request.toolDefinitions()); final var l4jRequestBuilder = ChatRequest.builder().messages(l4jMessages).toolSpecifications(toolSpecifications); - final var l4jResponseFormat = toL4jResponseFormat(options.responseFormat()); + final var l4jResponseFormat = toL4jResponseFormat(request.responseFormat()); if (l4jResponseFormat != null) { l4jRequestBuilder.responseFormat(l4jResponseFormat); } @@ -93,9 +95,7 @@ public CompletableFuture complete( final var l4jResponse = chatModel.chat(l4jRequestBuilder.build()); final var assistantMessage = chatMessageConverter.toAssistantMessage(l4jResponse); - return CompletableFuture.completedFuture( - new ChatResponse( - assistantMessage, assistantMessage.stopReason(), assistantMessage.usage(), null)); + return CompletableFuture.completedFuture(new ChatResponse(assistantMessage)); } catch (Exception e) { final var message = Optional.ofNullable(e.getMessage()) @@ -107,16 +107,15 @@ public CompletableFuture complete( } } - private @Nullable dev.langchain4j.model.chat.request.ResponseFormat toL4jResponseFormat( - @Nullable ResponseFormat responseFormat) { - if (!(responseFormat instanceof ResponseFormat.Json json)) { - // Both null and ResponseFormat.Text leave the format unset — matches the previous adapter, - // which avoided sending TEXT explicitly because some models reject it. + private @Nullable ResponseFormat toL4jResponseFormat( + @Nullable ResponseFormatConfiguration responseFormat) { + // Do not explicitly configure response format to TEXT — depending on the model this can lead + // to exceptions. Leaving the format unset preserves the previous adapter's behaviour. + if (!(responseFormat instanceof JsonResponseFormatConfiguration json)) { return null; } - final var builder = - dev.langchain4j.model.chat.request.ResponseFormat.builder().type(ResponseFormatType.JSON); + final var builder = ResponseFormat.builder().type(ResponseFormatType.JSON); if (json.schema() != null) { final var name = StringUtils.isNotBlank(json.schemaName()) ? json.schemaName() : "Response"; diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApiFactory.java index 9eeeae77157..8024c56c283 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApiFactory.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApiFactory.java @@ -9,63 +9,66 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.jsonschema.JsonSchemaConverter; +import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProvider; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverter; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.ProviderConfiguration; -import java.util.Set; /** - * Bridge factory that produces a {@link Langchain4JChatModelApi} for every built-in {@link - * ProviderConfiguration} subtype. Until per-family native implementations ship, every chat call - * lands here. + * LangChain4j bridge factory: produces a {@link Langchain4JChatModelApi} for one specific {@link + * ProviderConfiguration} subtype using the matching {@link ChatModelProvider} bean. One factory + * bean per discriminator — the provider type, configuration class, and {@link ChatModelProvider} + * are wired explicitly per bean in {@code AgenticAiLangchain4JFrameworkConfiguration}, so there is + * no {@code switch} on provider type inside the factory. + * + *

    Public so customers can compose it from their own {@link ChatModelApiFactory} bean — e.g. to + * wire a LangChain4j-supported provider we don't ship by passing in their own {@link + * ChatModelProvider} alongside the framework's converter beans. */ -public class Langchain4JChatModelApiFactory implements ChatModelApiFactory { +public class Langchain4JChatModelApiFactory + implements ChatModelApiFactory { public static final String API_FAMILY = "langchain4j"; - private static final Set SUPPORTED_PROVIDER_TYPES = - Set.of( - AnthropicProviderConfiguration.ANTHROPIC_ID, - BedrockProviderConfiguration.BEDROCK_ID, - AzureOpenAiProviderConfiguration.AZURE_OPENAI_ID, - GoogleVertexAiProviderConfiguration.GOOGLE_VERTEX_AI_ID, - OpenAiProviderConfiguration.OPENAI_ID, - OpenAiCompatibleProviderConfiguration.OPENAI_COMPATIBLE_ID); - - private final ChatModelFactory chatModelFactory; + private final String providerType; + private final Class configurationType; + private final ChatModelProvider chatModelProvider; private final ChatMessageConverter chatMessageConverter; private final ToolSpecificationConverter toolSpecificationConverter; private final JsonSchemaConverter jsonSchemaConverter; public Langchain4JChatModelApiFactory( - ChatModelFactory chatModelFactory, + String providerType, + Class configurationType, + ChatModelProvider chatModelProvider, ChatMessageConverter chatMessageConverter, ToolSpecificationConverter toolSpecificationConverter, JsonSchemaConverter jsonSchemaConverter) { - this.chatModelFactory = chatModelFactory; + this.providerType = providerType; + this.configurationType = configurationType; + this.chatModelProvider = chatModelProvider; this.chatMessageConverter = chatMessageConverter; this.toolSpecificationConverter = toolSpecificationConverter; this.jsonSchemaConverter = jsonSchemaConverter; } + @Override + public String providerType() { + return providerType; + } + @Override public String apiFamily() { return API_FAMILY; } @Override - public Set supportedProviderTypes() { - return SUPPORTED_PROVIDER_TYPES; + public Class configurationType() { + return configurationType; } @Override - public ChatModelApi create(ProviderConfiguration configuration) { - final var chatModel = chatModelFactory.createChatModel(configuration); + public ChatModelApi create(C configuration) { + final var chatModel = chatModelProvider.createChatModel(configuration); return new Langchain4JChatModelApi( chatModel, chatMessageConverter, toolSpecificationConverter, jsonSchemaConverter); } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java index 6bc9515990f..e3139903c78 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java @@ -7,19 +7,26 @@ package io.camunda.connector.agenticai.aiagent.framework.langchain4j.configuration; import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatMessageConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatMessageConverterImpl; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelFactory; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ContentConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ContentConverterImpl; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.Langchain4JChatModelApiFactory; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.document.DocumentToContentConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.document.DocumentToContentConverterImpl; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.jsonschema.JsonSchemaConverter; +import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProvider; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolCallConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolCallConverterImpl; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverterImpl; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; import io.camunda.connector.runtime.annotation.ConnectorsObjectMapper; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -78,13 +85,102 @@ public ChatMessageConverter langchain4JChatMessageConverter( } @Bean - @ConditionalOnMissingBean - public Langchain4JChatModelApiFactory langchain4JChatModelApiFactory( - ChatModelFactory chatModelFactory, + @ConditionalOnMissingBean(name = "langchain4JAnthropicChatModelApiFactory") + public ChatModelApiFactory + langchain4JAnthropicChatModelApiFactory( + ChatModelProvider provider, + ChatMessageConverter chatMessageConverter, + ToolSpecificationConverter toolSpecificationConverter, + JsonSchemaConverter jsonSchemaConverter) { + return new Langchain4JChatModelApiFactory<>( + AnthropicProviderConfiguration.ANTHROPIC_ID, + AnthropicProviderConfiguration.class, + provider, + chatMessageConverter, + toolSpecificationConverter, + jsonSchemaConverter); + } + + @Bean + @ConditionalOnMissingBean(name = "langchain4JBedrockChatModelApiFactory") + public ChatModelApiFactory langchain4JBedrockChatModelApiFactory( + ChatModelProvider provider, ChatMessageConverter chatMessageConverter, ToolSpecificationConverter toolSpecificationConverter, JsonSchemaConverter jsonSchemaConverter) { - return new Langchain4JChatModelApiFactory( - chatModelFactory, chatMessageConverter, toolSpecificationConverter, jsonSchemaConverter); + return new Langchain4JChatModelApiFactory<>( + BedrockProviderConfiguration.BEDROCK_ID, + BedrockProviderConfiguration.class, + provider, + chatMessageConverter, + toolSpecificationConverter, + jsonSchemaConverter); + } + + @Bean + @ConditionalOnMissingBean(name = "langchain4JAzureOpenAiChatModelApiFactory") + public ChatModelApiFactory + langchain4JAzureOpenAiChatModelApiFactory( + ChatModelProvider provider, + ChatMessageConverter chatMessageConverter, + ToolSpecificationConverter toolSpecificationConverter, + JsonSchemaConverter jsonSchemaConverter) { + return new Langchain4JChatModelApiFactory<>( + AzureOpenAiProviderConfiguration.AZURE_OPENAI_ID, + AzureOpenAiProviderConfiguration.class, + provider, + chatMessageConverter, + toolSpecificationConverter, + jsonSchemaConverter); + } + + @Bean + @ConditionalOnMissingBean(name = "langchain4JGoogleVertexAiChatModelApiFactory") + public ChatModelApiFactory + langchain4JGoogleVertexAiChatModelApiFactory( + ChatModelProvider provider, + ChatMessageConverter chatMessageConverter, + ToolSpecificationConverter toolSpecificationConverter, + JsonSchemaConverter jsonSchemaConverter) { + return new Langchain4JChatModelApiFactory<>( + GoogleVertexAiProviderConfiguration.GOOGLE_VERTEX_AI_ID, + GoogleVertexAiProviderConfiguration.class, + provider, + chatMessageConverter, + toolSpecificationConverter, + jsonSchemaConverter); + } + + @Bean + @ConditionalOnMissingBean(name = "langchain4JOpenAiChatModelApiFactory") + public ChatModelApiFactory langchain4JOpenAiChatModelApiFactory( + ChatModelProvider provider, + ChatMessageConverter chatMessageConverter, + ToolSpecificationConverter toolSpecificationConverter, + JsonSchemaConverter jsonSchemaConverter) { + return new Langchain4JChatModelApiFactory<>( + OpenAiProviderConfiguration.OPENAI_ID, + OpenAiProviderConfiguration.class, + provider, + chatMessageConverter, + toolSpecificationConverter, + jsonSchemaConverter); + } + + @Bean + @ConditionalOnMissingBean(name = "langchain4JOpenAiCompatibleChatModelApiFactory") + public ChatModelApiFactory + langchain4JOpenAiCompatibleChatModelApiFactory( + ChatModelProvider provider, + ChatMessageConverter chatMessageConverter, + ToolSpecificationConverter toolSpecificationConverter, + JsonSchemaConverter jsonSchemaConverter) { + return new Langchain4JChatModelApiFactory<>( + OpenAiCompatibleProviderConfiguration.OPENAI_COMPATIBLE_ID, + OpenAiCompatibleProviderConfiguration.class, + provider, + chatMessageConverter, + toolSpecificationConverter, + jsonSchemaConverter); } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/JobWorkerAgentRequestHandlerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/JobWorkerAgentRequestHandlerTest.java index be5c762f152..b8321293c6d 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/JobWorkerAgentRequestHandlerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/JobWorkerAgentRequestHandlerTest.java @@ -30,7 +30,7 @@ import io.camunda.connector.agenticai.aiagent.agent.AgentInitializationResult.AgentContextInitializationResult; import io.camunda.connector.agenticai.aiagent.agent.AgentInitializationResult.AgentResponseInitializationResult; import io.camunda.connector.agenticai.aiagent.framework.api.ChatClient; -import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatClientResult; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStore; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStoreRegistry; import io.camunda.connector.agenticai.aiagent.memory.conversation.inprocess.InProcessConversationContext; @@ -55,7 +55,6 @@ import io.camunda.connector.agenticai.model.tool.ToolCallResult; import java.util.List; import java.util.Map; -import java.util.concurrent.CompletableFuture; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -531,12 +530,19 @@ private void mockFrameworkExecution(AssistantMessage assistantMessage) { runtimeMemoryCaptor.capture(), any())) .thenAnswer( - i -> - CompletableFuture.completedFuture( - new ChatResponse( - assistantMessage, - assistantMessage.stopReason(), - TokenUsage.builder().inputTokenCount(10).outputTokenCount(20).build(), - null))); + i -> { + final var agentContext = i.getArgument(1, AgentContext.class); + return new ChatClientResult( + agentContext.withMetrics( + agentContext + .metrics() + .incrementModelCalls(1) + .incrementTokenUsage( + TokenUsage.builder() + .inputTokenCount(10) + .outputTokenCount(20) + .build())), + assistantMessage); + }); } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/OutboundConnectorAgentRequestHandlerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/OutboundConnectorAgentRequestHandlerTest.java index d1811736c25..9a1090dc49f 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/OutboundConnectorAgentRequestHandlerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/OutboundConnectorAgentRequestHandlerTest.java @@ -26,7 +26,7 @@ import io.camunda.connector.agenticai.aiagent.agent.AgentInitializationResult.AgentContextInitializationResult; import io.camunda.connector.agenticai.aiagent.agent.AgentInitializationResult.AgentResponseInitializationResult; import io.camunda.connector.agenticai.aiagent.framework.api.ChatClient; -import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatClientResult; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStoreRegistry; import io.camunda.connector.agenticai.aiagent.memory.conversation.inprocess.InProcessConversationContext; import io.camunda.connector.agenticai.aiagent.memory.conversation.inprocess.InProcessConversationStore; @@ -50,7 +50,6 @@ import io.camunda.connector.api.error.ConnectorException; import java.util.List; import java.util.Map; -import java.util.concurrent.CompletableFuture; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -391,12 +390,19 @@ private void mockFrameworkExecution(AssistantMessage assistantMessage) { runtimeMemoryCaptor.capture(), any())) .thenAnswer( - i -> - CompletableFuture.completedFuture( - new ChatResponse( - assistantMessage, - assistantMessage.stopReason(), - TokenUsage.builder().inputTokenCount(10).outputTokenCount(20).build(), - null))); + i -> { + final var agentContext = i.getArgument(1, AgentContext.class); + return new ChatClientResult( + agentContext.withMetrics( + agentContext + .metrics() + .incrementModelCalls(1) + .incrementTokenUsage( + TokenUsage.builder() + .inputTokenCount(10) + .outputTokenCount(20) + .build())), + assistantMessage); + }); } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImplTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImplTest.java index 73b82a94094..191ef611710 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImplTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImplTest.java @@ -6,9 +6,11 @@ */ package io.camunda.connector.agenticai.aiagent.framework; +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.assistantMessage; import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.systemMessage; import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.userMessage; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; @@ -18,11 +20,12 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; -import io.camunda.connector.agenticai.aiagent.framework.api.ResponseFormat; import io.camunda.connector.agenticai.aiagent.memory.runtime.DefaultRuntimeMemory; import io.camunda.connector.agenticai.aiagent.memory.runtime.RuntimeMemory; import io.camunda.connector.agenticai.aiagent.model.AgentContext; import io.camunda.connector.agenticai.aiagent.model.AgentExecutionContext; +import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; +import io.camunda.connector.agenticai.aiagent.model.AgentMetrics.TokenUsage; import io.camunda.connector.agenticai.aiagent.model.AgentState; import io.camunda.connector.agenticai.aiagent.model.request.OutboundConnectorResponseConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.ResponseConfiguration; @@ -32,7 +35,9 @@ import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicConnection; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicModel; +import io.camunda.connector.agenticai.model.message.AssistantMessage; import io.camunda.connector.agenticai.model.tool.ToolDefinition; +import io.camunda.connector.api.error.ConnectorException; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -50,9 +55,6 @@ class ChatClientImplTest { private static final List TOOL_DEFINITIONS = List.of(ToolDefinition.builder().name("Tool").description("desc").build()); - private static final AgentContext AGENT_CONTEXT = - AgentContext.empty().withState(AgentState.READY).withToolDefinitions(TOOL_DEFINITIONS); - private static final AnthropicProviderConfiguration PROVIDER_CONFIG = new AnthropicProviderConfiguration( new AnthropicConnection( @@ -61,10 +63,13 @@ class ChatClientImplTest { null, new AnthropicModel("claude", null))); + private static final AssistantMessage ASSISTANT_MESSAGE = + assistantMessage("hello world") + .withUsage(TokenUsage.builder().inputTokenCount(10).outputTokenCount(20).build()); + @Mock private ChatModelApiRegistry registry; @Mock private ChatModelApi chatModelApi; @Mock private AgentExecutionContext executionContext; - @Mock private ChatResponse chatResponse; @Captor private ArgumentCaptor requestCaptor; @Captor private ArgumentCaptor optionsCaptor; @@ -80,21 +85,43 @@ void setUp() { when(executionContext.provider()).thenReturn(PROVIDER_CONFIG); when(registry.resolve(PROVIDER_CONFIG)).thenReturn(chatModelApi); when(chatModelApi.complete(requestCaptor.capture(), optionsCaptor.capture(), any())) - .thenReturn(CompletableFuture.completedFuture(chatResponse)); + .thenReturn(CompletableFuture.completedFuture(new ChatResponse(ASSISTANT_MESSAGE))); chatClient = new ChatClientImpl(registry); } @Test void buildsRequestFromRuntimeMemoryAndAgentContext() { - final var future = chatClient.chat(executionContext, AGENT_CONTEXT, runtimeMemory, null); + final var agentContext = + AgentContext.empty().withState(AgentState.READY).withToolDefinitions(TOOL_DEFINITIONS); - assertThat(future).isCompleted(); - assertThat(future.join()).isSameAs(chatResponse); + final var result = chatClient.chat(executionContext, agentContext, runtimeMemory, null); + + assertThat(result.assistantMessage()).isEqualTo(ASSISTANT_MESSAGE); assertThat(requestCaptor.getValue().messages()) .containsExactlyElementsOf(runtimeMemory.filteredMessages()); - assertThat(requestCaptor.getValue().tools()).containsExactlyElementsOf(TOOL_DEFINITIONS); - assertThat(requestCaptor.getValue().systemPrompt()).isNull(); + assertThat(requestCaptor.getValue().toolDefinitions()) + .containsExactlyElementsOf(TOOL_DEFINITIONS); + assertThat(requestCaptor.getValue().responseFormat()).isNull(); + } + + @Test + void incrementsAgentContextMetrics() { + final var agentContext = + AgentContext.empty() + .withState(AgentState.READY) + .withToolDefinitions(TOOL_DEFINITIONS) + .withMetrics( + AgentMetrics.empty() + .withModelCalls(2) + .withTokenUsage( + TokenUsage.builder().inputTokenCount(5).outputTokenCount(7).build())); + + final var result = chatClient.chat(executionContext, agentContext, runtimeMemory, null); + + assertThat(result.agentContext().metrics().modelCalls()).isEqualTo(3); + assertThat(result.agentContext().metrics().tokenUsage()) + .isEqualTo(TokenUsage.builder().inputTokenCount(15).outputTokenCount(27).build()); } @Test @@ -104,40 +131,57 @@ void translatesJsonResponseFormatConfiguration() { new OutboundConnectorResponseConfiguration( new JsonResponseFormatConfiguration(Map.of("type", "object"), "MySchema"), false)); - chatClient.chat(executionContext, AGENT_CONTEXT, runtimeMemory, ChatStreamListener.NOOP); + chatClient.chat( + executionContext, agentContextWithTools(), runtimeMemory, ChatStreamListener.NOOP); - final var responseFormat = optionsCaptor.getValue().responseFormat(); - assertThat(responseFormat).isInstanceOf(ResponseFormat.Json.class); - final var json = (ResponseFormat.Json) responseFormat; + final var responseFormat = requestCaptor.getValue().responseFormat(); + assertThat(responseFormat).isInstanceOf(JsonResponseFormatConfiguration.class); + final var json = (JsonResponseFormatConfiguration) responseFormat; assertThat(json.schemaName()).isEqualTo("MySchema"); assertThat(json.schema()).containsEntry("type", "object"); } @Test - void leavesResponseFormatNullForTextConfiguration() { + void passesThroughTextResponseFormatConfiguration() { when(executionContext.response()) .thenReturn( new OutboundConnectorResponseConfiguration( new TextResponseFormatConfiguration(false), false)); - chatClient.chat(executionContext, AGENT_CONTEXT, runtimeMemory, ChatStreamListener.NOOP); + chatClient.chat( + executionContext, agentContextWithTools(), runtimeMemory, ChatStreamListener.NOOP); - assertThat(optionsCaptor.getValue().responseFormat()).isNull(); + assertThat(requestCaptor.getValue().responseFormat()) + .isInstanceOf(TextResponseFormatConfiguration.class); } @Test void leavesResponseFormatNullWhenResponseConfigurationMissing() { when(executionContext.response()).thenReturn((ResponseConfiguration) null); - chatClient.chat(executionContext, AGENT_CONTEXT, runtimeMemory, ChatStreamListener.NOOP); + chatClient.chat( + executionContext, agentContextWithTools(), runtimeMemory, ChatStreamListener.NOOP); - assertThat(optionsCaptor.getValue().responseFormat()).isNull(); + assertThat(requestCaptor.getValue().responseFormat()).isNull(); } @Test - void usesNoopListenerWhenCallerPassesNull() { - chatClient.chat(executionContext, AGENT_CONTEXT, runtimeMemory, null); + void unwrapsCompletionExceptionFromUnderlyingApi() { + final var cause = new ConnectorException("MODEL_ERROR", "boom"); + when(chatModelApi.complete(any(), any(), any())) + .thenReturn(CompletableFuture.failedFuture(cause)); + + assertThatThrownBy( + () -> + chatClient.chat( + executionContext, + agentContextWithTools(), + runtimeMemory, + ChatStreamListener.NOOP)) + .isSameAs(cause); + } - // no NPE on the underlying ChatModelApi means the null listener was substituted with NOOP + private static AgentContext agentContextWithTools() { + return AgentContext.empty().withState(AgentState.READY).withToolDefinitions(TOOL_DEFINITIONS); } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImplTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImplTest.java index a2500aa150f..7374ee62652 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImplTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImplTest.java @@ -20,7 +20,6 @@ import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicModel; import io.camunda.connector.agenticai.aiagent.model.request.provider.ProviderConfiguration; import java.util.List; -import java.util.Set; import org.junit.jupiter.api.Test; class ChatModelApiRegistryImplTest { @@ -38,18 +37,6 @@ void resolvesFactoryByProviderType() { assertThat(registry.resolve(validAnthropicConfig())).isSameAs(resolvedApi); } - @Test - void resolvesFactoryClaimingMultipleProviderTypes() { - final var multiTypeFactory = - factoryFor(AnthropicProviderConfiguration.ANTHROPIC_ID, "openai", "bedrock"); - final var resolvedApi = mock(ChatModelApi.class); - when(multiTypeFactory.create(any())).thenReturn(resolvedApi); - - final var registry = new ChatModelApiRegistryImpl(List.of(multiTypeFactory)); - - assertThat(registry.resolve(validAnthropicConfig())).isSameAs(resolvedApi); - } - @Test void throwsWhenNoFactoryRegisteredForType() { final var registry = new ChatModelApiRegistryImpl(List.of()); @@ -71,11 +58,12 @@ void throwsWhenTwoFactoriesClaimSameProviderType() { .hasMessageContaining("Two chat model API factories claim provider type 'duplicate'"); } - @SuppressWarnings("unchecked") - private static ChatModelApiFactory factoryFor(String... providerTypes) { + @SuppressWarnings({"unchecked", "rawtypes"}) + private static ChatModelApiFactory factoryFor(String providerType) { final ChatModelApiFactory factory = mock(ChatModelApiFactory.class); - when(factory.supportedProviderTypes()).thenReturn(Set.of(providerTypes)); + when(factory.providerType()).thenReturn(providerType); when(factory.apiFamily()).thenReturn("test"); + when(factory.configurationType()).thenReturn((Class) ProviderConfiguration.class); return factory; } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApiTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApiTest.java index 48cb7552780..e8d77c399e4 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApiTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/Langchain4JChatModelApiTest.java @@ -27,10 +27,12 @@ import dev.langchain4j.model.chat.response.ChatResponse; import io.camunda.connector.agenticai.aiagent.framework.api.ChatOptions; import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; -import io.camunda.connector.agenticai.aiagent.framework.api.ResponseFormat; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.jsonschema.JsonSchemaConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverter; import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; +import io.camunda.connector.agenticai.aiagent.model.request.ResponseFormatConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.ResponseFormatConfiguration.JsonResponseFormatConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.ResponseFormatConfiguration.TextResponseFormatConfiguration; import io.camunda.connector.agenticai.model.message.AssistantMessage; import io.camunda.connector.agenticai.model.message.Message; import io.camunda.connector.agenticai.model.tool.ToolDefinition; @@ -107,7 +109,7 @@ void setUp() { @Test void modelRequestContainsMessagesAndToolSpecifications() { - complete(textOptions()); + complete(null); final var chatRequest = chatRequestCaptor.getValue(); assertThat(chatRequest.messages()).containsExactlyElementsOf(L4J_MESSAGES); @@ -122,7 +124,7 @@ void wrapsUnderlyingExceptionsInConnectorException() { final var cause = new ModelNotFoundException("Model 'dummy' was not found"); doThrow(cause).when(chatModel).chat(any(ChatRequest.class)); - assertThatThrownBy(() -> complete(textOptions()).join()) + assertThatThrownBy(() -> complete(null).join()) .isInstanceOfSatisfying( CompletionException.class, wrapper -> @@ -145,7 +147,7 @@ void usesExceptionClassIfNoMessageIncludedInException() { final var cause = new UnresolvedModelServerException((String) null); doThrow(cause).when(chatModel).chat(any(ChatRequest.class)); - assertThatThrownBy(() -> complete(textOptions()).join()) + assertThatThrownBy(() -> complete(null).join()) .isInstanceOfSatisfying( CompletionException.class, wrapper -> @@ -161,22 +163,22 @@ void usesExceptionClassIfNoMessageIncludedInException() { } @Test - void doesNotExplicitlyConfigureResponseFormatForText() { - complete(textOptions()); + void doesNotExplicitlyConfigureResponseFormatWhenNull() { + complete(null); assertThat(chatRequestCaptor.getValue().responseFormat()).isNull(); } @Test - void doesNotExplicitlyConfigureResponseFormatWhenNull() { - complete(options(null)); + void doesNotExplicitlyConfigureResponseFormatForText() { + complete(new TextResponseFormatConfiguration(false)); assertThat(chatRequestCaptor.getValue().responseFormat()).isNull(); } @Test void requestsJsonResponseWhenConfigured() { - complete(options(new ResponseFormat.Json(null, null))); + complete(new JsonResponseFormatConfiguration(null, null)); final var format = chatRequestCaptor.getValue().responseFormat(); assertThat(format.type()).isEqualTo(ResponseFormatType.JSON); @@ -191,7 +193,7 @@ void requestsJsonResponseWithSchemaWhenConfigured() { final var jsonObjectSchema = JsonObjectSchema.builder().description("My schema").build(); when(jsonSchemaConverter.mapToSchema(schema)).thenReturn(jsonObjectSchema); - complete(options(new ResponseFormat.Json(schemaName, schema))); + complete(new JsonResponseFormatConfiguration(schema, schemaName)); final var format = chatRequestCaptor.getValue().responseFormat(); assertThat(format.type()).isEqualTo(ResponseFormatType.JSON); @@ -209,7 +211,7 @@ void usesDefaultSchemaNameIfNullOrBlank(String schemaName) { final var jsonObjectSchema = JsonObjectSchema.builder().description("My schema").build(); when(jsonSchemaConverter.mapToSchema(schema)).thenReturn(jsonObjectSchema); - complete(options(new ResponseFormat.Json(schemaName, schema))); + complete(new JsonResponseFormatConfiguration(schema, schemaName)); final var format = chatRequestCaptor.getValue().responseFormat(); assertThat(format.type()).isEqualTo(ResponseFormatType.JSON); @@ -219,31 +221,22 @@ void usesDefaultSchemaNameIfNullOrBlank(String schemaName) { } @Test - void responseCarriesUsageAndStopReasonFromAssistantMessage() { - final var response = complete(textOptions()).join(); + void responseCarriesAssistantMessageWithUsage() { + final var response = complete(null).join(); assertThat(response.assistantMessage()).isNotNull(); - assertThat(response.usage()) + assertThat(response.assistantMessage().usage()) .isEqualTo( AgentMetrics.TokenUsage.builder().inputTokenCount(5).outputTokenCount(6).build()); - assertThat(response.errorMessage()).isNull(); } private java.util.concurrent.CompletableFuture< io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse> - complete(ChatOptions options) { + complete(ResponseFormatConfiguration responseFormat) { return api.complete( new io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest( - INPUT_MESSAGES, null, TOOL_DEFINITIONS), - options, + INPUT_MESSAGES, TOOL_DEFINITIONS, responseFormat), + new ChatOptions(null, null, null, Map.of()), ChatStreamListener.NOOP); } - - private static ChatOptions textOptions() { - return options(new ResponseFormat.Text()); - } - - private static ChatOptions options(ResponseFormat responseFormat) { - return new ChatOptions(null, null, null, responseFormat, Map.of()); - } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java index b842ede2f44..a50e2c4e179 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java @@ -34,7 +34,6 @@ import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelFactory; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelHttpProxySupport; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ContentConverter; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.Langchain4JChatModelApiFactory; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.document.DocumentToContentConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.jsonschema.JsonSchemaConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.AnthropicChatModelProvider; @@ -126,8 +125,7 @@ class AgenticAiConnectorsAutoConfigurationTest { ToolCallConverter.class, JsonSchemaConverter.class, ToolSpecificationConverter.class, - ChatMessageConverter.class, - Langchain4JChatModelApiFactory.class); + ChatMessageConverter.class); // this will need to be updated in case we support different frameworks private static final List> ALL_BEANS = From f19eec0a43ce0e73daf38a229a763327f3065b5c Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 14:20:14 +0200 Subject: [PATCH 50/81] docs(agentic-ai): align AGENTS.md and ADR-004 plan with as-built SPI AGENTS.md now reflects the framework/api/ + ChatClientImpl layout instead of the removed AiFrameworkAdapter. The ADR-004 plan picks up the as-built bridge wiring (one factory bean per discriminator), the ChatOptions surface (incl. maxOutputTokens), and notes that the legacy AiFramework* types were already removed in Phase A. --- connectors/agentic-ai/AGENTS.md | 10 +- .../docs/adr-004-implementation-plan.md | 335 ++++++++++++++++++ 2 files changed, 343 insertions(+), 2 deletions(-) create mode 100644 connectors/agentic-ai/docs/adr-004-implementation-plan.md diff --git a/connectors/agentic-ai/AGENTS.md b/connectors/agentic-ai/AGENTS.md index f76b6b225eb..662a00d2a23 100644 --- a/connectors/agentic-ai/AGENTS.md +++ b/connectors/agentic-ai/AGENTS.md @@ -69,8 +69,14 @@ agent/ └── AgentLimitsValidatorImpl # Safety limits (max model calls) framework/ -├── AiFrameworkAdapter # Abstract LLM interface (RuntimeMemory → response) -└── langchain4j/ # LangChain4J implementation +├── api/ # Provider-neutral SPI +│ ├── ChatClient # Facade called by BaseAgentRequestHandler +│ ├── ChatModelApi # Per-job model client: capabilities() + complete(...) +│ ├── ChatModelApiFactory # One bean per provider discriminator +│ └── ChatModelApiRegistry # Resolves provider config → factory +├── ChatClientImpl # Assembles ChatRequest/ChatOptions, dispatches, accumulates metrics +├── ChatModelApiRegistryImpl # Indexes factories by providerType() +└── langchain4j/ # LangChain4j-backed ChatModelApi (one factory bean per provider) memory/ ├── conversation/ diff --git a/connectors/agentic-ai/docs/adr-004-implementation-plan.md b/connectors/agentic-ai/docs/adr-004-implementation-plan.md new file mode 100644 index 00000000000..9a84e3bb9da --- /dev/null +++ b/connectors/agentic-ai/docs/adr-004-implementation-plan.md @@ -0,0 +1,335 @@ +# ADR-004 Phase 1 — Incremental Implementation Plan + +## Context + +[ADR-004](connectors/agentic-ai/docs/adr/004-replace-langchain4j-framework.md) (on branch +`origin/agentic-ai/custom-llm-layer`) replaces the LangChain4j-backed `AiFrameworkAdapter` with +a native provider layer over official vendor SDKs. The ADR names this "Phase 1 — one shipping +unit," but it is large enough to review and merge in chunks. This plan breaks it into six +green-build checkpoints (Phases A–F) on the working branch. + +**Actual starting state** (`origin/agentic-ai/custom-llm-layer`, 4 commits ahead of `main`): +- Phase 0 done: `AssistantMessage` gains `modelId`/`apiId`/`stopReason`/`usage`; `TokenUsage` + gains cache and reasoning fields; `ReasoningContent` added to `Content` hierarchy; + `contentBlocks` added to `ToolCallResult`; `AiFrameworkChatResponse#rawChatResponse()` dropped. + LangChain4j adapter populates the new fields from `ChatResponseMetadata`. +- Wire-format e2e regression tests added for Anthropic Messages API and OpenAI Responses API + (WireMock-based, currently exercising LangChain4j). +- ADR-004 document committed. +- **SPI under `framework/api/` does NOT exist yet.** The plan starts here. + +**First action**: merge `origin/agentic-ai/custom-llm-layer` into +`claude/refine-local-plan-e5Asz` to carry Phase 0 forward, then implement Phases A–F on top. + +**End state**: `BaseAgentRequestHandler` calls `ChatClient`, not `AiFrameworkAdapter`. Five native +`ChatModelApiFactory` families ship. LangChain4j survives only as an opt-in bridge. + +--- + +## Architecture overview + +``` +BaseAgentRequestHandler + │ + ▼ + ChatClient (facade) resolves factory, applies ToolCallResultStrategy + │ updates agentContext metrics, returns ChatClientResult + ▼ +ChatModelApiRegistry Map + │ + ▼ +ChatModelApiFactory singleton bean per wire-protocol family (or bridge) + │ create(config) → ChatModelApi (per-job) + ▼ +ChatModelApi capabilities() + complete(request, options, listener) + │ + ▼ + Vendor SDK Anthropic / AWS Bedrock / OpenAI / Google GenAI +``` + +**Data flow through ChatClient.chat():** +1. Resolve factory from `executionContext.provider()` via registry → `ChatModelApi` +2. Query `ChatModelApi.capabilities()` → `ModelCapabilities` +3. Apply `ToolCallResultStrategy` to route `toolCallResults.contentBlocks` (native vs fallback) +4. Build `ChatRequest` (messages + tools + response format) + `ChatOptions` (cache, reasoning) +5. `chatModelApi.complete(request, options, NOOP_LISTENER)` → `CompletableFuture` +6. Wrap into `ChatClientResult` (updated `agentContext` + `assistantMessage`) + +--- + +## SPI types (all in `framework/api/`) + +Defined in Phase A; referenced by all subsequent phases: + +| Type | Role | +|------|------| +| `ChatRequest` | `List messages`, `List toolDefinitions`, `ResponseFormatConfiguration responseFormat` | +| `ChatResponse` | `AssistantMessage assistantMessage` (carries `stopReason`, `usage` etc. per Phase 0) | +| `ChatOptions` | `@Nullable Integer maxOutputTokens`, `@Nullable ReasoningConfig reasoning`, `@Nullable CacheRetention cacheRetention`, `Map providerOptions` | +| `CacheRetention` | enum `NONE \| SHORT \| LONG` | +| `ReasoningConfig` | sealed: `ReasoningEffort(Effort)`, `ReasoningBudget(int)`, `ReasoningDisabled` | +| `ModelCapabilities` | modality lists per location, `supports_*` flags, context window, max output tokens | +| `ChatModelEvent` | sealed hierarchy per ADR §"Stream event hierarchy" | +| `ChatStreamListener` | single method per event type; static `NOOP` constant | +| `ChatModelApi` | `ModelCapabilities capabilities()`, `CompletableFuture complete(ChatRequest, ChatOptions, ChatStreamListener)` | +| `ChatModelApiFactory` | `String providerType()`, `Class configurationType()`, `ChatModelApi create(C config)` | +| `ChatModelApiRegistry` | `Map>` keyed by `providerType()` string; mirrors `ChatModelProviderRegistry` exactly | +| `ChatClient` | `ChatClientResult chat(AgentExecutionContext, AgentContext, RuntimeMemory)` | +| `ChatClientResult` | `AgentContext agentContext()`, `AssistantMessage assistantMessage()` — replaces `AiFrameworkChatResponse` in call sites | + +**Registry dispatch** mirrors `ChatModelProviderRegistry`: +`providerConfiguration.providerType()` → factory lookup → `factory.create(config)`. +The bridge `Langchain4JChatModelApiFactory` is parameterised on a single discriminator (`providerType()` returns one string) and registered as **one Spring bean per discriminator** in `AgenticAiLangchain4JFrameworkConfiguration` — six beans total, each named `langchain4JChatModelApiFactory` and gated with `@ConditionalOnMissingBean(name = ...)` so a customer or a native impl can replace any individual one. The registry collects all `ChatModelApiFactory` beans and indexes by `providerType()`; duplicate discriminators fail at startup. Phase E swaps each bridge bean out by registering a native factory under the same name. + +--- + +## Phase A — Bridge cutover (behavior-identical) + +**Goal**: every chat call routes through the new SPI. LangChain4j is still the only +implementation. Proves SPI shape is right before any native provider lands. + +**Files to create** (`framework/api/`): +- All SPI types listed above (interfaces + records, no impls) + +**Files to create** (`framework/langchain4j/`): +- `Langchain4JChatModelApi` + `Langchain4JChatModelApiFactory` — + the API wraps a pre-resolved L4J `ChatModel` and translates `ChatRequest`/`ChatOptions` to/from + the L4J shape; the factory is parameterised on a single discriminator and produced as one bean + per provider in `AgenticAiLangchain4JFrameworkConfiguration` (six beans total). Returns + conservative `ModelCapabilities` (text-only, no caching/reasoning, parallel tool calls true, + null context window) from `capabilities()`. +- `ChatModelApiRegistryImpl` — identical structure to `ChatModelProviderRegistry` (lines 16–57). +- `ChatClientImpl` — steps 1–6 from the data-flow above; wraps the future with `.get()` (sync for + now, async for Phase C+); extracts `assistantMessage` → updates `agentContext.metrics`. + +**Files to modify**: +- `BaseAgentRequestHandler`: field `AiFrameworkAdapter framework` → `ChatClient chatClient`; + lines 171–172: `framework.executeChatRequest(...)` → `chatClient.chat(...)`. Return type changes + from `AiFrameworkChatResponse` to `ChatClientResult` — update 3 usages on lines 173–178. +- `AgenticAiConnectorsAutoConfiguration`: inject `ChatClient` instead of `AiFrameworkAdapter` + into the two request handler beans; add beans for `ChatModelApiRegistry` and `ChatClientImpl`. + The LangChain4j config still produces the `Langchain4JChatModelApiFactory` bean. +- `OutboundConnectorAgentRequestHandlerTest` + `JobWorkerAgentRequestHandlerTest`: change + `@Mock AiFrameworkAdapter` → `@Mock ChatClient`; update stub return type. + +**Tests to add**: +- `ChatModelApiRegistryImplTest` — mirrors `ChatModelProviderRegistryTest` (resolution, duplicate + detection, missing-key error). +- `ChatClientImplTest` — verifies request assembly (messages from `RuntimeMemory`, tools from + `AgentContext`), dispatches to the mock `ChatModelApi`, updates `agentContext.metrics`. + +**Verification**: +```bash +mvn clean test -pl connectors/agentic-ai +mvn test -pl connectors-e2e-test/connectors-e2e-test-agentic-ai \ + -Dtest="AnthropicMessagesApiAiAgentJobWorkerTests,OpenAiResponsesApiAiAgentJobWorkerTests" +``` + +--- + +## Phase B — Capability matrix + tool-result strategy + +**Goal**: infrastructure native providers will depend on. Bridge returns conservative defaults; +`ToolCallResultStrategy` always falls back (current behavior unchanged). + +**Files to create**: +- `resources/capabilities/model-capabilities.yaml` — empty entries initially; schema per ADR + §"Capability Matrix". +- `ModelCapabilitiesResolver` — resolution chain: connector override → exact id/alias → glob + (longest match) → conservative defaults. INFO log on pattern or default use. +- `ToolCallResultStrategy` — pure function `apply(ToolCallResult, ModelCapabilities)` returning + per-block routing decision (inline `contentBlocks` vs. synthetic `UserMessage` fallback via + existing `AgentMessagesHandlerImpl` path from PR #6999). + +**ChatOptions** completes here: `CacheRetention` defaults to `SHORT` in `ChatClientImpl`; bridge +always uses `NONE` (no cache support), resolved from `ModelCapabilities.supportsPromptCaching()`. + +**Tests to add**: +- `ModelCapabilitiesResolverTest` — all 4 resolution steps, alias match, glob longest-match. +- `ToolCallResultStrategyTest` — table-driven: every modality × every capability combo. +- YAML round-trip test (Jackson reads the resource, validates required fields). + +**Verification**: `mvn clean test -pl connectors/agentic-ai` green; e2e wire-format tests still +pass. + +--- + +## Phase C — First native: `anthropic-messages` (direct backend only) + +**Goal**: prove the streaming-first internal pattern, signed reasoning roundtrip, and cache +breakpoints with one provider end-to-end. + +**Files to create** (`framework/anthropic/`): +- `AnthropicMessagesChatModelApi` — drives `anthropic-java` SDK streaming endpoint. Emits + `ChatModelEvent`s, accumulates content blocks by index, materialises `AssistantMessage` + (including `ReasoningContent` with `signature`), maps `stop_reason` → `StopReason`, populates + `TokenUsage` with cache/reasoning fields. Error mapping per ADR §"Error semantics": model-side + terminal → `ChatResponse{stopReason=ERROR}`; transport/auth → exceptional future with + `ConnectorException(ERROR_CODE_FAILED_MODEL_CALL, ...)`. +- `AnthropicMessagesChatModelApiFactory` — `providerType() = AnthropicProviderConfiguration.ANTHROPIC_ID`; + `configurationType() = AnthropicProviderConfiguration.class`. Phase C: `backend = direct` only + (other backends added in Phase E). +- `AnthropicMessagesApiConfiguration` — `@ConditionalOnClass(AnthropicClient.class)` Spring config + bean. + +**Files to modify**: +- `pom.xml`: add `com.anthropic:anthropic-java` SDK. +- `model-capabilities.yaml`: populate Anthropic models (Opus, Sonnet, Haiku families). +- `AgenticAiConnectorsAutoConfiguration`: import `AnthropicMessagesApiConfiguration`; the bridge + factory falls back only for providers not covered natively (`@ConditionalOnMissingBean` scoped to + `ChatModelApiFactory` bean named for each discriminator, or registry registration order with a + `@Order` lower on the bridge). +- `ChatClientImpl`: switch from `.get()` to async completion (the `CompletableFuture` from Phase A + was already wired, just blocked synchronously — make it truly async or keep sync if call-site + doesn't need it). + +**Tests to add**: +- `AnthropicMessagesChatModelApiTest` — mocked SDK client; verify event ordering, content-block + accumulation, signature roundtrip, usage accounting, error path. +- Extend `AnthropicMessagesApiAiAgentJobWorkerTests` with a streaming variant and reasoning + roundtrip case (WireMock stubs still in HTTP, now exercising the native implementation). + +**Verification**: unit tests green; both wire-format e2e tests still green (Anthropic via native, +OpenAI still via bridge). + +--- + +## Phase D — ProviderConfiguration restructure + Jackson migration + +**Goal**: introduce new discriminators without breaking saved process state. Element template +version bump to 11. + +**Files to modify**: +- `AnthropicProviderConfiguration`: add `AnthropicBackend { DIRECT, BEDROCK, VERTEX, FOUNDRY }`; + conditional auth fields per backend. +- `OpenAiProviderConfiguration` / `AzureOpenAiProviderConfiguration` / + `OpenAiCompatibleProviderConfiguration`: add `ApiFamily { RESPONSES, COMPLETIONS }` with + provider-appropriate defaults. +- `GoogleVertexAiProviderConfiguration` → `GoogleGenAiProviderConfiguration`: add + `GoogleBackend { DEVELOPER_API, VERTEX }`; discriminator `googleVertexAi` → `googleGenAi`. + Update `ProviderConfiguration` sealed type + `@JsonSubTypes`. +- `BedrockProviderConfiguration`: validate at construction that model ID is non-Anthropic. +- New `ProviderConfigurationDeserializer extends StdDeserializer`: pre- + processes JSON node before delegating. Migration rules per ADR table (7 rows). Pattern: + `JsonSchemaElementDeserializer.java:52` (tree-walking dispatch). Register via Jackson `Module`. +- `element-templates/agenticai-aiagent-outbound-connector.json`: bump `version` 10 → 11; add + conditional UI groups for `backend` / `apiFamily`. Maven generates versioned snapshot and + job-worker template automatically via existing `gmavenplus` step. +- `element-templates/README.md`: same Camunda minor (8.10) → replace top row per AGENTS.md + §"Version index README" rule. + +**Tests to add**: +- `ProviderConfigurationDeserializerTest` — every row of the migration table round-trips to new + shape; forward serialization writes new shape. +- `AgentContextTest` round-trip with stored agent context (no provider config inside — sanity). + +**Verification**: +```bash +mvn clean install -pl connectors/agentic-ai # triggers element-template generation +``` +Manual inspect: `versioned/agenticai-aiagent-outbound-connector-10.json` created correctly. + +--- + +## Phase E — Remaining native implementations + +**Goal**: four remaining `ChatModelApiFactory` families. Each follows the Phase C pattern. +Structured as five sub-steps (note: `anthropic-messages` cloud backends reuse the Phase C +factory, so it's not a full new impl): + +1. **`anthropic-messages` cloud backends** — extend `AnthropicMessagesChatModelApiFactory` to + honour `backend = bedrock | vertex | foundry` (SDK modules: `anthropic-java-bedrock`, + `anthropic-java-vertex`, `anthropic-java-foundry`). Same wire format; only client construction + differs. + +2. **`openai-responses`** — `openai-java` SDK, Responses API. Encrypted reasoning item; cache via + `prompt_cache_key`. Covers OpenAI direct + Azure when `apiFamily = responses`. + +3. **`openai-completions`** — same SDK, Chat Completions endpoint. Covers legacy Chat models, + OpenAI-compatible gateways (Ollama, vLLM). Default for `OpenAiCompatibleProviderConfiguration`. + +4. **`google-genai`** — `google-genai-java` SDK; backend toggle (`developer-api` / `vertex`). + Reasoning via `thoughtSignature`; thinking budget via `ThinkingConfig.thinkingBudget`. + +5. **`bedrock-converse`** — AWS SDK v2 `bedrockruntime`. Non-Anthropic models only. Multimodal + tool results via `ToolResultContentBlock`; cache via `cachePoint` blocks. + +**Per-impl checklist** (same as Phase C): +- SDK dependency with `@ConditionalOnClass` +- Streaming-first internal driver +- Full `ChatModelEvent` emission +- Content + usage accumulation +- Error classification per ADR table +- `model-capabilities.yaml` entries +- Unit tests (mocked SDK client) +- Wire-format e2e regression test added under `connectors-e2e-test-agentic-ai/.../wireformat/` + +**Verification**: after each impl, corresponding wire-format e2e test passes; full unit suite +green. + +--- + +## Phase F — Cleanup + LangChain4j demotion + +**Goal**: remove the legacy contract; bridge stays on shelf as opt-in. + +**Files to delete**: already removed during Phase A +(`AiFrameworkAdapter`, `AiFrameworkChatResponse`, `Langchain4JAiFrameworkChatResponse` — +no leftover references in `connectors/agentic-ai/src`). + +**Files to modify**: +- `Langchain4JChatModelApiFactory`: drop default Spring registration; gate behind + `camunda.connector.agenticai.framework.langchain4j.bridge.enabled=false` by default. Document + in release notes / `docs/reference/ai-agent.md`. +- `AgenticAiConnectorsAutoConfiguration`: drop `AgenticAiLangchain4JFrameworkConfiguration` import; + add per-provider `@ConditionalOnClass` configurations (built in Phase C–E). +- ADR-004 status: Proposed → Implemented, dated 2026-05-07. +- `docs/reference/ai-agent.md` + `CLAUDE.md` (agentic-ai): describe `ChatModelApi` as the + framework; LangChain4j bridge as legacy opt-in. + +**Tests to add/update**: +- `AgenticAiConnectorsAutoConfigurationTest` updated for new wiring. +- Smoke test: bridge re-enabled via property → handler still resolves all provider types. + +**Verification**: +```bash +mvn clean install -pl connectors/agentic-ai +mvn test -pl connectors-e2e-test/connectors-e2e-test-agentic-ai -Dtest="*Wireformat*" +mvn test -pl connectors-e2e-test/connectors-e2e-test-agentic-ai # full suite (slow — final gate) +``` + +--- + +## Critical files + +| File | Phase | +|------|-------| +| `BaseAgentRequestHandler.java:171–172` | A (cutover site) | +| `AgenticAiConnectorsAutoConfiguration.java` | A, C–F (Spring wiring) | +| `framework/api/` (new SPI package) | A | +| `framework/langchain4j/Langchain4JAiFrameworkAdapter.java` | A (wrapped by bridge) | +| `framework/langchain4j/provider/ChatModelProviderRegistry.java` | A (registry pattern to mirror) | +| `model/request/provider/*ProviderConfiguration.java` | D | +| `element-templates/agenticai-aiagent-outbound-connector.json` | D | +| `element-templates/README.md` | D | +| `docs/adr/004-replace-langchain4j-framework.md` | F (status update) | + +## Reusable existing code + +| Code | Used in | +|------|---------| +| `ChatModelProviderRegistry.java:16–57` | registry pattern for `ChatModelApiRegistryImpl` (Phase A) | +| `JsonSchemaElementDeserializer.java:52` | tree-walking deserializer pattern for `ProviderConfigurationDeserializer` (Phase D) | +| `AgentErrorCodes.ERROR_CODE_FAILED_MODEL_CALL` | transport/auth error wrapping in each native impl | +| `AgentMessagesHandlerImpl` tool-result fallback path (PR #6999) | `ToolCallResultStrategy` fallback (Phase B) | + +## End-to-end verification (after Phase F) + +1. `mvn clean install -pl connectors/agentic-ai` — all unit tests green, element templates + regenerate to v11, `AI_AGENT.md` regenerates. +2. `mvn test -pl connectors-e2e-test/connectors-e2e-test-agentic-ai -Dtest=*Wireformat*` — both + wire-format tests pass via native Anthropic and OpenAI implementations. +3. `mvn test -pl connectors-e2e-test/connectors-e2e-test-agentic-ai` — full suite green. +4. Manual: `git diff element-templates/` shows v11 source + v10 in `versioned/` + README updated. +5. Stale-process smoke: deserialize a saved `agentContext` from a v10 instance with the new + `ProviderConfigurationDeserializer` — all 7 ADR migration table rows covered by tests. From 304442a84aec343cf071e85d6816472013ff381b Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 14:38:01 +0200 Subject: [PATCH 51/81] docs(agentic-ai): re-order ADR-004 plan to ship native Anthropic+OpenAI first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase B–D become native impls (Anthropic Messages, OpenAI Chat Completions, OpenAI Responses) before the capability matrix and multimodality work, which moves to Phase E. ProviderConfiguration restructure shifts to F, remaining native impls (Azure OpenAI, Anthropic cloud backends, Google GenAI, Bedrock-Converse) consolidate into G, cleanup is now H. Reasoning, prompt caching, multimodal content, and Azure OpenAI are explicitly deferred from the first native cut so we can validate the SPI shape against two divergent wire formats before generalising. --- .../docs/adr-004-implementation-plan.md | 375 +++++++++++------- 1 file changed, 240 insertions(+), 135 deletions(-) diff --git a/connectors/agentic-ai/docs/adr-004-implementation-plan.md b/connectors/agentic-ai/docs/adr-004-implementation-plan.md index 9a84e3bb9da..4c287765d5b 100644 --- a/connectors/agentic-ai/docs/adr-004-implementation-plan.md +++ b/connectors/agentic-ai/docs/adr-004-implementation-plan.md @@ -2,27 +2,32 @@ ## Context -[ADR-004](connectors/agentic-ai/docs/adr/004-replace-langchain4j-framework.md) (on branch -`origin/agentic-ai/custom-llm-layer`) replaces the LangChain4j-backed `AiFrameworkAdapter` with -a native provider layer over official vendor SDKs. The ADR names this "Phase 1 — one shipping -unit," but it is large enough to review and merge in chunks. This plan breaks it into six -green-build checkpoints (Phases A–F) on the working branch. - -**Actual starting state** (`origin/agentic-ai/custom-llm-layer`, 4 commits ahead of `main`): +[ADR-004](connectors/agentic-ai/docs/adr/004-replace-langchain4j-framework.md) replaces the +LangChain4j-backed `AiFrameworkAdapter` with a native provider layer over official vendor SDKs. +The ADR names this "Phase 1 — one shipping unit," but it is large enough to review and merge in +chunks. This plan breaks it into eight green-build checkpoints (Phases A–H) on the working branch. + +**Plan revision (2026-05-07)** — re-ordered to ship two real native providers ahead of the +capability/multimodality infrastructure. Validating the SPI against two divergent wire formats +before generalising leads to a smaller, better-shaped abstraction in Phase E. Reasoning, prompt +caching, multimodal user/tool-result content, Azure OpenAI, Anthropic cloud backends, Google GenAI +and Bedrock-Converse are all explicitly **deferred** out of the first native cut. + +**Actual starting state** (`agentic-ai/custom-llm-layer`): - Phase 0 done: `AssistantMessage` gains `modelId`/`apiId`/`stopReason`/`usage`; `TokenUsage` gains cache and reasoning fields; `ReasoningContent` added to `Content` hierarchy; `contentBlocks` added to `ToolCallResult`; `AiFrameworkChatResponse#rawChatResponse()` dropped. - LangChain4j adapter populates the new fields from `ChatResponseMetadata`. +- **Phase A done**: SPI under `framework/api/` shipped; `BaseAgentRequestHandler` routes through + `ChatClient`; LangChain4j wired as the bridge `ChatModelApi` for all six provider discriminators + (one factory bean per discriminator). `AiFrameworkAdapter` and `AiFrameworkChatResponse` already + removed from the source tree (ahead of the original Phase F schedule). - Wire-format e2e regression tests added for Anthropic Messages API and OpenAI Responses API - (WireMock-based, currently exercising LangChain4j). + (WireMock-based, currently exercising the bridge). - ADR-004 document committed. -- **SPI under `framework/api/` does NOT exist yet.** The plan starts here. - -**First action**: merge `origin/agentic-ai/custom-llm-layer` into -`claude/refine-local-plan-e5Asz` to carry Phase 0 forward, then implement Phases A–F on top. -**End state**: `BaseAgentRequestHandler` calls `ChatClient`, not `AiFrameworkAdapter`. Five native -`ChatModelApiFactory` families ship. LangChain4j survives only as an opt-in bridge. +**End state**: `BaseAgentRequestHandler` calls `ChatClient`. Native `ChatModelApi` impls ship for +Anthropic Messages (direct), OpenAI Chat Completions, OpenAI Responses, Azure OpenAI, Anthropic +cloud backends, Google GenAI and Bedrock-Converse. LangChain4j survives only as an opt-in bridge. --- @@ -127,134 +132,233 @@ mvn test -pl connectors-e2e-test/connectors-e2e-test-agentic-ai \ --- -## Phase B — Capability matrix + tool-result strategy +## Scope deferred out of native cut (Phases B–D) + +The first native cut deliberately ignores these — they re-enter in Phase E or G: + +| Topic | Re-enters | +|-------|-----------| +| Multimodal user-message / tool-result content (image, PDF, audio, video) | E (capabilities + strategy) | +| Reasoning content (signed thinking blocks, encrypted reasoning items) | E | +| Prompt caching (`cache_control`, `prompt_cache_key`) | E | +| Capability matrix YAML + resolver | E | +| `ToolCallResultStrategy` (always inline-text in B–D) | E | +| Azure OpenAI native impl | G | +| Anthropic cloud backends (Bedrock / Vertex / Foundry) | G | +| Google GenAI native impl | G | +| Bedrock-Converse native impl (non-Anthropic models) | G | +| `ProviderConfiguration` discriminator restructure + Jackson migration | F | + +Under this scope each native impl returns a hardcoded `ModelCapabilities` (text-only, no +reasoning, no caching, parallel tool calls true). `ChatOptions.cacheRetention` and +`ChatOptions.reasoning` are accepted but ignored. `ToolCallResult.contentBlocks` carries text +parts only — image / PDF / audio / video parts get rejected (or pass through unchanged where the +existing handler already drops them) until Phase E lands the strategy. + +--- -**Goal**: infrastructure native providers will depend on. Bridge returns conservative defaults; -`ToolCallResultStrategy` always falls back (current behavior unchanged). +## Phase B — Native `anthropic-messages` (direct backend, text-only) -**Files to create**: -- `resources/capabilities/model-capabilities.yaml` — empty entries initially; schema per ADR - §"Capability Matrix". -- `ModelCapabilitiesResolver` — resolution chain: connector override → exact id/alias → glob - (longest match) → conservative defaults. INFO log on pattern or default use. -- `ToolCallResultStrategy` — pure function `apply(ToolCallResult, ModelCapabilities)` returning - per-block routing decision (inline `contentBlocks` vs. synthetic `UserMessage` fallback via - existing `AgentMessagesHandlerImpl` path from PR #6999). +**Goal**: replace the L4J bridge for the `anthropic` discriminator with a direct vendor-SDK impl. +Validates streaming-first internal pattern + content-block accumulation against one wire format. -**ChatOptions** completes here: `CacheRetention` defaults to `SHORT` in `ChatClientImpl`; bridge -always uses `NONE` (no cache support), resolved from `ModelCapabilities.supportsPromptCaching()`. +**Files to create** (`framework/anthropic/`): +- `AnthropicMessagesChatModelApi` — drives `anthropic-java` SDK streaming endpoint. Accumulates + content blocks by index, materialises `AssistantMessage`, maps `stop_reason` → `StopReason`, + populates `TokenUsage` (cache / reasoning fields left at zero). Emits `ChatModelEvent`s as the + stream progresses. Reasoning blocks parsed structurally but **dropped** in this phase (or kept + as opaque text content per ADR — we'll decide during impl). Error mapping: model-side terminal + → `ChatResponse{stopReason=ERROR}`; transport/auth → exceptional future with + `ConnectorException(ERROR_CODE_FAILED_MODEL_CALL, ...)`. +- `AnthropicMessagesChatModelApiFactory` — `providerType() = AnthropicProviderConfiguration.ANTHROPIC_ID`; + `configurationType() = AnthropicProviderConfiguration.class`. Direct backend only. +- `AnthropicMessagesApiConfiguration` — `@ConditionalOnClass(AnthropicClient.class)` Spring config + bean. Registers the factory under bean name `langchain4JAnthropicChatModelApiFactory` (same name + as the bridge bean), so `@ConditionalOnMissingBean(name = ...)` in + `AgenticAiLangchain4JFrameworkConfiguration` skips registering the bridge for this discriminator. + +**Files to modify**: +- `connectors/agentic-ai/pom.xml`: add `com.anthropic:anthropic-java` SDK. +- `AgenticAiConnectorsAutoConfiguration`: `@Import(AnthropicMessagesApiConfiguration.class)`. **Tests to add**: -- `ModelCapabilitiesResolverTest` — all 4 resolution steps, alias match, glob longest-match. -- `ToolCallResultStrategyTest` — table-driven: every modality × every capability combo. -- YAML round-trip test (Jackson reads the resource, validates required fields). +- `AnthropicMessagesChatModelApiTest` — mocked SDK client; verify message conversion, tool + conversion, content-block accumulation, stop-reason mapping, usage accounting, error path. +- `AnthropicMessagesChatModelApiFactoryTest` — providerType/configurationType wiring. -**Verification**: `mvn clean test -pl connectors/agentic-ai` green; e2e wire-format tests still -pass. +**Verification**: `mvn clean test -pl connectors/agentic-ai`; the existing +`AnthropicMessagesApiAiAgentJobWorkerTests` wire-format e2e should pass against the native impl +(skip if it asserts on bridge-specific behaviour — flag for later). --- -## Phase C — First native: `anthropic-messages` (direct backend only) +## Phase C — Native OpenAI Chat Completions (`openai` + `openaiCompatible`) + +**Goal**: replace the L4J bridge for `openai` and `openaiCompatible` discriminators. Azure +deferred to G. + +**Files to create** (`framework/openai/`): +- `OpenAiChatCompletionsChatModelApi` — drives `openai-java` SDK chat-completions endpoint + (streaming-first internal). Same accumulation / stop-reason / error patterns as Phase B's + Anthropic impl. +- `OpenAiToolConverter` — small shared converter: `ToolDefinition` → openai-java `ChatTool` + (JSON schema). Lives in `framework/openai/` — reused by Phase D's Responses impl. +- `OpenAiChatModelApiFactory` — registered under bean name + `langchain4JOpenAiChatModelApiFactory`. Builds an `OpenAIClient` from + `OpenAiProviderConfiguration` and instantiates `OpenAiChatCompletionsChatModelApi`. + (Phase D wraps an apiFamily branch around the impl-class choice.) +- `OpenAiCompatibleChatModelApiFactory` — registered under bean name + `langchain4JOpenAiCompatibleChatModelApiFactory`. Builds an `OpenAIClient` with custom baseUrl + and optional auth, hands to `OpenAiChatCompletionsChatModelApi`. +- `OpenAiChatModelApiConfiguration` — `@ConditionalOnClass(OpenAIClient.class)` Spring config bean + registering both factories. -**Goal**: prove the streaming-first internal pattern, signed reasoning roundtrip, and cache -breakpoints with one provider end-to-end. +**Files to modify**: +- `connectors/agentic-ai/pom.xml`: add `com.openai:openai-java` SDK. +- `AgenticAiConnectorsAutoConfiguration`: `@Import(OpenAiChatModelApiConfiguration.class)`. -**Files to create** (`framework/anthropic/`): -- `AnthropicMessagesChatModelApi` — drives `anthropic-java` SDK streaming endpoint. Emits - `ChatModelEvent`s, accumulates content blocks by index, materialises `AssistantMessage` - (including `ReasoningContent` with `signature`), maps `stop_reason` → `StopReason`, populates - `TokenUsage` with cache/reasoning fields. Error mapping per ADR §"Error semantics": model-side - terminal → `ChatResponse{stopReason=ERROR}`; transport/auth → exceptional future with - `ConnectorException(ERROR_CODE_FAILED_MODEL_CALL, ...)`. -- `AnthropicMessagesChatModelApiFactory` — `providerType() = AnthropicProviderConfiguration.ANTHROPIC_ID`; - `configurationType() = AnthropicProviderConfiguration.class`. Phase C: `backend = direct` only - (other backends added in Phase E). -- `AnthropicMessagesApiConfiguration` — `@ConditionalOnClass(AnthropicClient.class)` Spring config - bean. +**Tests to add**: +- `OpenAiChatCompletionsChatModelApiTest` — mocked SDK client; conversion + accumulation + + errors. +- `OpenAiChatModelApiFactoryTest` / `OpenAiCompatibleChatModelApiFactoryTest`. +- `OpenAiToolConverterTest` — JSON schema fidelity. + +**Verification**: `mvn clean test -pl connectors/agentic-ai`. The existing +`OpenAiChatCompletionsApiAiAgentJobWorkerTests` (or analogous wire-format e2e) should pass. + +--- + +## Phase D — Add OpenAI Responses (`apiFamily` switch on `openai`) + +**Goal**: ship native `openai-responses` without doing the Phase F config restructure. +Azure stays bridged; openaiCompatible stays Completions-only. + +**Files to create** (`framework/openai/`): +- `OpenAiResponsesChatModelApi` — drives `openai-java` SDK responses endpoint. Handles input-item + shape, output-item shape, and the different streaming format. Reuses `OpenAiToolConverter` for + the `tools[]` JSON schema. Encrypted reasoning items parsed structurally but dropped in this + phase (Phase E re-enables roundtripping). **Files to modify**: -- `pom.xml`: add `com.anthropic:anthropic-java` SDK. -- `model-capabilities.yaml`: populate Anthropic models (Opus, Sonnet, Haiku families). -- `AgenticAiConnectorsAutoConfiguration`: import `AnthropicMessagesApiConfiguration`; the bridge - factory falls back only for providers not covered natively (`@ConditionalOnMissingBean` scoped to - `ChatModelApiFactory` bean named for each discriminator, or registry registration order with a - `@Order` lower on the bridge). -- `ChatClientImpl`: switch from `.get()` to async completion (the `CompletableFuture` from Phase A - was already wired, just blocked synchronously — make it truly async or keep sync if call-site - doesn't need it). +- `OpenAiProviderConfiguration`: + - Add `ApiFamily { COMPLETIONS, RESPONSES }`. + - Add `apiFamily` field, defaulting to `COMPLETIONS` so saved process state from v10 element + template instances deserializes unchanged. **No Jackson migration deserializer** needed for + this — the canonical restructure (with multi-row migration table) is Phase F. +- `OpenAiChatModelApiFactory` (from Phase C): branch on `config.apiFamily()`. Same client built + from auth/baseUrl is handed to either impl class. +- `element-templates/agenticai-aiagent-outbound-connector.json`: bump `version` 10 → 11; add + `apiFamily` dropdown (Completions / Responses) under the OpenAI provider section. Maven's + `gmavenplus` step regenerates the versioned snapshot and the job-worker template automatically. +- `element-templates/README.md`: update top row for AI Agent (Task + Sub-process tables, both + reflect v11) per AGENTS.md §"Version index README" rule. **Tests to add**: -- `AnthropicMessagesChatModelApiTest` — mocked SDK client; verify event ordering, content-block - accumulation, signature roundtrip, usage accounting, error path. -- Extend `AnthropicMessagesApiAiAgentJobWorkerTests` with a streaming variant and reasoning - roundtrip case (WireMock stubs still in HTTP, now exercising the native implementation). +- `OpenAiResponsesChatModelApiTest` — mocked SDK client; input-item / output-item conversion; + tool-call extraction; streaming accumulation; errors. +- `OpenAiChatModelApiFactoryTest`: extend with `apiFamily=RESPONSES` branch. +- `OpenAiProviderConfigurationTest`: round-trip a config without `apiFamily` (defaults applied). -**Verification**: unit tests green; both wire-format e2e tests still green (Anthropic via native, -OpenAI still via bridge). +**Verification**: `mvn clean install -pl connectors/agentic-ai` (regenerates element templates); +verify `versioned/agenticai-aiagent-outbound-connector-10.json` exists alongside the new v11 file +in the main folder. The existing `OpenAiResponsesApiAiAgentJobWorkerTests` wire-format e2e should +pass against the native impl. --- -## Phase D — ProviderConfiguration restructure + Jackson migration +## Phase E — Capability matrix + tool-result strategy + multimodality + reasoning + caching + +**Goal**: re-enable the deferred features now that we have three real native impls and one bridge +to generalise from. -**Goal**: introduce new discriminators without breaking saved process state. Element template -version bump to 11. +**Files to create**: +- `resources/capabilities/model-capabilities.yaml` — populated entries for Anthropic Claude + families, OpenAI Chat Completions models, OpenAI Responses models. Schema per ADR §"Capability + Matrix". +- `ModelCapabilitiesResolver` — 4-step resolution chain: connector override → exact id/alias → + glob (longest match) → conservative defaults. INFO log on pattern or default use. +- `ToolCallResultStrategy` — pure function `apply(ToolCallResult, ModelCapabilities)` returning + per-block routing decision (inline `contentBlocks` vs. synthetic `UserMessage` fallback through + the existing `AgentMessagesHandlerImpl` path from PR #6999). + +**Files to modify**: +- Each native `ChatModelApi`: replace the hardcoded `ModelCapabilities` with the resolved one; + start emitting/consuming the deferred features: + - **Anthropic**: extended thinking blocks (signed reasoning roundtrip), `cache_control` markers + on the last system / message / tool-definition block, image/PDF user content, image tool-result + content. + - **OpenAI Responses**: encrypted reasoning items roundtrip, `prompt_cache_key`, image input. + - **OpenAI Completions**: `prompt_cache_key`, image input. + - **L4J bridge**: keeps conservative defaults; cache → NONE. +- `ChatClientImpl`: default `ChatOptions.cacheRetention` to `SHORT` (clamped to `NONE` when + `!supportsPromptCaching()`); query `capabilities()` once per call; apply + `ToolCallResultStrategy`. + +**Tests to add**: +- `ModelCapabilitiesResolverTest` — all 4 steps, alias match, glob longest-match. +- `ToolCallResultStrategyTest` — table-driven across modality × capability combos. +- YAML round-trip test (Jackson reads the resource, validates required fields). +- Per-impl: extend tests with multimodal / reasoning / cache cases. + +**Verification**: `mvn clean test -pl connectors/agentic-ai`; both wire-format e2e tests still +green; new multimodal / reasoning / cache cases added under `wireformat/` exercise the new code +paths. + +--- + +## Phase F — `ProviderConfiguration` restructure + Jackson migration + +**Goal**: introduce the canonical discriminator scheme without breaking saved process state. +Element template version bump 11 → 12 (after D's bump). **Files to modify**: - `AnthropicProviderConfiguration`: add `AnthropicBackend { DIRECT, BEDROCK, VERTEX, FOUNDRY }`; conditional auth fields per backend. -- `OpenAiProviderConfiguration` / `AzureOpenAiProviderConfiguration` / - `OpenAiCompatibleProviderConfiguration`: add `ApiFamily { RESPONSES, COMPLETIONS }` with - provider-appropriate defaults. +- `OpenAiProviderConfiguration`: `apiFamily` already added in Phase D — extend handling here as + needed. +- `AzureOpenAiProviderConfiguration`: add `apiFamily` matching the OpenAI shape. +- `OpenAiCompatibleProviderConfiguration`: stays Completions-only (no `apiFamily` field). - `GoogleVertexAiProviderConfiguration` → `GoogleGenAiProviderConfiguration`: add - `GoogleBackend { DEVELOPER_API, VERTEX }`; discriminator `googleVertexAi` → `googleGenAi`. - Update `ProviderConfiguration` sealed type + `@JsonSubTypes`. -- `BedrockProviderConfiguration`: validate at construction that model ID is non-Anthropic. -- New `ProviderConfigurationDeserializer extends StdDeserializer`: pre- - processes JSON node before delegating. Migration rules per ADR table (7 rows). Pattern: + `GoogleBackend { DEVELOPER_API, VERTEX }`; rename discriminator `googleVertexAi` → `googleGenAi`. +- `BedrockProviderConfiguration`: validate at construction that model ID is non-Anthropic + (forwarding to the Anthropic factory if it is). +- New `ProviderConfigurationDeserializer extends StdDeserializer`: + pre-processes JSON node before delegating. Migration rules per ADR table. Pattern: `JsonSchemaElementDeserializer.java:52` (tree-walking dispatch). Register via Jackson `Module`. -- `element-templates/agenticai-aiagent-outbound-connector.json`: bump `version` 10 → 11; add - conditional UI groups for `backend` / `apiFamily`. Maven generates versioned snapshot and - job-worker template automatically via existing `gmavenplus` step. -- `element-templates/README.md`: same Camunda minor (8.10) → replace top row per AGENTS.md - §"Version index README" rule. +- `element-templates/agenticai-aiagent-outbound-connector.json`: bump 11 → 12; add conditional UI + groups for `backend`. Maven regenerates the versioned snapshot + job-worker template. +- `element-templates/README.md`: replace top row again (or insert new row if Camunda min version + changes). **Tests to add**: - `ProviderConfigurationDeserializerTest` — every row of the migration table round-trips to new shape; forward serialization writes new shape. -- `AgentContextTest` round-trip with stored agent context (no provider config inside — sanity). -**Verification**: -```bash -mvn clean install -pl connectors/agentic-ai # triggers element-template generation -``` -Manual inspect: `versioned/agenticai-aiagent-outbound-connector-10.json` created correctly. +**Verification**: `mvn clean install -pl connectors/agentic-ai`; manual inspect +`versioned/agenticai-aiagent-outbound-connector-11.json` created; deserialization smoke test +covers a saved `agentContext` from a v10/v11 instance. --- -## Phase E — Remaining native implementations +## Phase G — Remaining native impls -**Goal**: four remaining `ChatModelApiFactory` families. Each follows the Phase C pattern. -Structured as five sub-steps (note: `anthropic-messages` cloud backends reuse the Phase C -factory, so it's not a full new impl): +**Goal**: native impls for the last four families. Each follows the Phase B pattern. -1. **`anthropic-messages` cloud backends** — extend `AnthropicMessagesChatModelApiFactory` to - honour `backend = bedrock | vertex | foundry` (SDK modules: `anthropic-java-bedrock`, +1. **Anthropic cloud backends** — extend `AnthropicMessagesChatModelApiFactory` to honour + `backend = bedrock | vertex | foundry` (SDK modules: `anthropic-java-bedrock`, `anthropic-java-vertex`, `anthropic-java-foundry`). Same wire format; only client construction differs. - -2. **`openai-responses`** — `openai-java` SDK, Responses API. Encrypted reasoning item; cache via - `prompt_cache_key`. Covers OpenAI direct + Azure when `apiFamily = responses`. - -3. **`openai-completions`** — same SDK, Chat Completions endpoint. Covers legacy Chat models, - OpenAI-compatible gateways (Ollama, vLLM). Default for `OpenAiCompatibleProviderConfiguration`. - -4. **`google-genai`** — `google-genai-java` SDK; backend toggle (`developer-api` / `vertex`). +2. **Native Azure OpenAI** — `AzureOpenAiChatModelApiFactory` builds the Azure-flavored + `OpenAIClient` (API key or Entra/AAD auth) and hands to the existing `OpenAiChatCompletions` + / `OpenAiResponses` impl classes. Branches on `config.apiFamily()` exactly like the OpenAI + factory. (If openai-java's Azure variant lags Responses endpoint coverage, Azure stays + Completions-only for one release.) +3. **Native Google GenAI** — `google-genai-java` SDK; backend toggle (`developer-api` / `vertex`). Reasoning via `thoughtSignature`; thinking budget via `ThinkingConfig.thinkingBudget`. +4. **Native Bedrock-Converse** — AWS SDK v2 `bedrockruntime`. Non-Anthropic models only. + Multimodal tool results via `ToolResultContentBlock`; cache via `cachePoint` blocks. -5. **`bedrock-converse`** — AWS SDK v2 `bedrockruntime`. Non-Anthropic models only. Multimodal - tool results via `ToolResultContentBlock`; cache via `cachePoint` blocks. - -**Per-impl checklist** (same as Phase C): +**Per-impl checklist** (same as Phase B): - SDK dependency with `@ConditionalOnClass` - Streaming-first internal driver - Full `ChatModelEvent` emission @@ -262,30 +366,28 @@ factory, so it's not a full new impl): - Error classification per ADR table - `model-capabilities.yaml` entries - Unit tests (mocked SDK client) -- Wire-format e2e regression test added under `connectors-e2e-test-agentic-ai/.../wireformat/` +- Wire-format e2e regression test under `connectors-e2e-test-agentic-ai/.../wireformat/` **Verification**: after each impl, corresponding wire-format e2e test passes; full unit suite green. --- -## Phase F — Cleanup + LangChain4j demotion +## Phase H — Cleanup + LangChain4j demotion -**Goal**: remove the legacy contract; bridge stays on shelf as opt-in. +**Goal**: bridge stays as opt-in only. -**Files to delete**: already removed during Phase A -(`AiFrameworkAdapter`, `AiFrameworkChatResponse`, `Langchain4JAiFrameworkChatResponse` — -no leftover references in `connectors/agentic-ai/src`). +**Files to delete**: already removed during Phase A. **Files to modify**: -- `Langchain4JChatModelApiFactory`: drop default Spring registration; gate behind - `camunda.connector.agenticai.framework.langchain4j.bridge.enabled=false` by default. Document - in release notes / `docs/reference/ai-agent.md`. -- `AgenticAiConnectorsAutoConfiguration`: drop `AgenticAiLangchain4JFrameworkConfiguration` import; - add per-provider `@ConditionalOnClass` configurations (built in Phase C–E). -- ADR-004 status: Proposed → Implemented, dated 2026-05-07. -- `docs/reference/ai-agent.md` + `CLAUDE.md` (agentic-ai): describe `ChatModelApi` as the - framework; LangChain4j bridge as legacy opt-in. +- `Langchain4JChatModelApiFactory` / `AgenticAiLangchain4JFrameworkConfiguration`: drop default + Spring registration; gate behind `camunda.connector.agenticai.framework.langchain4j.bridge.enabled=false` + by default. Document in release notes / `docs/reference/ai-agent.md`. +- `AgenticAiConnectorsAutoConfiguration`: drop `AgenticAiLangchain4JFrameworkConfiguration` + default import. +- ADR-004 status: Proposed → Implemented (final date set when shipping). +- `docs/reference/ai-agent.md` + `AGENTS.md` (agentic-ai): `ChatModelApi` is the framework; + LangChain4j bridge is legacy opt-in. **Tests to add/update**: - `AgenticAiConnectorsAutoConfigurationTest` updated for new wiring. @@ -304,32 +406,35 @@ mvn test -pl connectors-e2e-test/connectors-e2e-test-agentic-ai # full suite ( | File | Phase | |------|-------| -| `BaseAgentRequestHandler.java:171–172` | A (cutover site) | -| `AgenticAiConnectorsAutoConfiguration.java` | A, C–F (Spring wiring) | -| `framework/api/` (new SPI package) | A | -| `framework/langchain4j/Langchain4JAiFrameworkAdapter.java` | A (wrapped by bridge) | -| `framework/langchain4j/provider/ChatModelProviderRegistry.java` | A (registry pattern to mirror) | -| `model/request/provider/*ProviderConfiguration.java` | D | -| `element-templates/agenticai-aiagent-outbound-connector.json` | D | -| `element-templates/README.md` | D | -| `docs/adr/004-replace-langchain4j-framework.md` | F (status update) | +| `BaseAgentRequestHandler.java:172–176` | A (cutover site, done) | +| `framework/api/` SPI package | A (done) | +| `framework/ChatClientImpl.java` | A (done), E (capability + strategy wiring) | +| `framework/anthropic/` (new) | B, G (cloud backends) | +| `framework/openai/` (new) | C (Completions + factories), D (Responses), G (Azure) | +| `OpenAiProviderConfiguration.java` | D (apiFamily field) | +| `model/request/provider/*ProviderConfiguration.java` | F (canonical restructure) | +| `element-templates/agenticai-aiagent-outbound-connector.json` | D (v11), F (v12) | +| `element-templates/README.md` | D, F | +| `AgenticAiConnectorsAutoConfiguration.java` | A (done), B–G (provider imports) | +| `docs/adr/004-replace-langchain4j-framework.md` | H (status update) | ## Reusable existing code | Code | Used in | |------|---------| -| `ChatModelProviderRegistry.java:16–57` | registry pattern for `ChatModelApiRegistryImpl` (Phase A) | -| `JsonSchemaElementDeserializer.java:52` | tree-walking deserializer pattern for `ProviderConfigurationDeserializer` (Phase D) | -| `AgentErrorCodes.ERROR_CODE_FAILED_MODEL_CALL` | transport/auth error wrapping in each native impl | -| `AgentMessagesHandlerImpl` tool-result fallback path (PR #6999) | `ToolCallResultStrategy` fallback (Phase B) | +| `ChatModelProviderRegistry.java:16–57` | registry pattern for `ChatModelApiRegistryImpl` (Phase A, done) | +| `JsonSchemaElementDeserializer.java:52` | tree-walking deserializer pattern for `ProviderConfigurationDeserializer` (Phase F) | +| `AgentErrorCodes.ERROR_CODE_FAILED_MODEL_CALL` | transport/auth error wrapping in each native impl (B onwards) | +| `AgentMessagesHandlerImpl` tool-result fallback path (PR #6999) | `ToolCallResultStrategy` fallback (Phase E) | -## End-to-end verification (after Phase F) +## End-to-end verification (after Phase H) 1. `mvn clean install -pl connectors/agentic-ai` — all unit tests green, element templates - regenerate to v11, `AI_AGENT.md` regenerates. + regenerate, `AI_AGENT.md` regenerates. 2. `mvn test -pl connectors-e2e-test/connectors-e2e-test-agentic-ai -Dtest=*Wireformat*` — both wire-format tests pass via native Anthropic and OpenAI implementations. 3. `mvn test -pl connectors-e2e-test/connectors-e2e-test-agentic-ai` — full suite green. -4. Manual: `git diff element-templates/` shows v11 source + v10 in `versioned/` + README updated. -5. Stale-process smoke: deserialize a saved `agentContext` from a v10 instance with the new - `ProviderConfigurationDeserializer` — all 7 ADR migration table rows covered by tests. +4. Manual: `git diff element-templates/` shows the latest source + previous in `versioned/` + + README updated. +5. Stale-process smoke: deserialize a saved `agentContext` from a pre-restructure instance with + the new `ProviderConfigurationDeserializer` — every migration table row covered by tests. From a979a1aab1c0120fc0a45865419dfc95eb0c242e Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 15:04:32 +0200 Subject: [PATCH 52/81] feat(agentic-ai): native Anthropic Messages ChatModelApi (text-only) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the LangChain4j bridge for the `anthropic` discriminator with a native impl driving the anthropic-java SDK's blocking Messages endpoint. The new `AnthropicMessagesApiConfiguration` registers the factory under the same Spring bean name as the bridge factory, so the bridge's `@ConditionalOnMissingBean(name = ...)` lets the native bean take over whenever the SDK is on the classpath. Phase B scope (per ADR-004 Phase 1 plan revision): - Direct backend only — Bedrock/Vertex/Foundry deferred to Phase G. - Text-only content for user / assistant / tool-result blocks. Multi- modal content, reasoning round-tripping and prompt caching all wait for Phase E (capability matrix + tool-result strategy). - Conservative hardcoded `ModelCapabilities` (text-only, no reasoning, no caching, parallel tool calls true) until the resolver lands. - Blocking SDK call; the OkHttp transport is the SDK default. Adding a JDK `java.net.http.HttpClient` adapter (replacing the OkHttp dependency) is a Phase E follow-up captured in the deferred-scope table of the impl plan. --- .../docs/adr-004-implementation-plan.md | 1 + connectors/agentic-ai/pom.xml | 13 + .../AnthropicMessagesApiConfiguration.java | 35 ++ .../AnthropicMessagesChatModelApi.java | 391 ++++++++++++++++++ .../AnthropicMessagesChatModelApiFactory.java | 91 ++++ .../AgenticAiConnectorsAutoConfiguration.java | 2 + .../AnthropicMessagesChatModelApiTest.java | 250 +++++++++++ 7 files changed, 783 insertions(+) create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesApiConfiguration.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java create mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java diff --git a/connectors/agentic-ai/docs/adr-004-implementation-plan.md b/connectors/agentic-ai/docs/adr-004-implementation-plan.md index 4c287765d5b..f7524346f5f 100644 --- a/connectors/agentic-ai/docs/adr-004-implementation-plan.md +++ b/connectors/agentic-ai/docs/adr-004-implementation-plan.md @@ -148,6 +148,7 @@ The first native cut deliberately ignores these — they re-enter in Phase E or | Google GenAI native impl | G | | Bedrock-Converse native impl (non-Anthropic models) | G | | `ProviderConfiguration` discriminator restructure + Jackson migration | F | +| JDK `java.net.http.HttpClient` adapter for the Anthropic / OpenAI SDKs (replaces OkHttp transport) | E (follow-up) | Under this scope each native impl returns a hardcoded `ModelCapabilities` (text-only, no reasoning, no caching, parallel tool calls true). `ChatOptions.cacheRetention` and diff --git a/connectors/agentic-ai/pom.xml b/connectors/agentic-ai/pom.xml index 4de3899f825..8870e363065 100644 --- a/connectors/agentic-ai/pom.xml +++ b/connectors/agentic-ai/pom.xml @@ -22,6 +22,7 @@ 5.0.5 4.3.1 0.3.3.Final + 2.16.1 @@ -132,6 +133,18 @@ langchain4j-http-client-jdk + + + com.anthropic + anthropic-java-core + ${version.anthropic-java} + + + com.anthropic + anthropic-java-client-okhttp + ${version.anthropic-java} + + io.modelcontextprotocol.sdk diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesApiConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesApiConfiguration.java new file mode 100644 index 00000000000..259e9535fc9 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesApiConfiguration.java @@ -0,0 +1,35 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.anthropic; + +import com.anthropic.client.AnthropicClient; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; +import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsConfigurationProperties; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Spring configuration that registers the native Anthropic Messages factory under the same bean + * name as the LangChain4j bridge factory ({@code langchain4JAnthropicChatModelApiFactory}). The + * bridge bean is gated on {@code @ConditionalOnMissingBean(name = ...)} so this native bean wins + * automatically whenever the SDK is on the classpath. + */ +@Configuration +@ConditionalOnClass(AnthropicClient.class) +public class AnthropicMessagesApiConfiguration { + + @Bean(name = "langchain4JAnthropicChatModelApiFactory") + @ConditionalOnMissingBean(name = "langchain4JAnthropicChatModelApiFactory") + public ChatModelApiFactory anthropicMessagesChatModelApiFactory( + AgenticAiConnectorsConfigurationProperties properties) { + return new AnthropicMessagesChatModelApiFactory( + properties.aiagent().chatModel().api().defaultTimeout()); + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java new file mode 100644 index 00000000000..319732231a5 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java @@ -0,0 +1,391 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.anthropic; + +import static io.camunda.connector.agenticai.aiagent.agent.AgentErrorCodes.ERROR_CODE_FAILED_MODEL_CALL; + +import com.anthropic.client.AnthropicClient; +import com.anthropic.core.JsonValue; +import com.anthropic.models.messages.ContentBlock; +import com.anthropic.models.messages.ContentBlockParam; +import com.anthropic.models.messages.Message; +import com.anthropic.models.messages.MessageCreateParams; +import com.anthropic.models.messages.MessageParam; +import com.anthropic.models.messages.TextBlockParam; +import com.anthropic.models.messages.Tool; +import com.anthropic.models.messages.ToolResultBlockParam; +import com.anthropic.models.messages.ToolUnion; +import com.anthropic.models.messages.ToolUseBlockParam; +import com.anthropic.models.messages.Usage; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatOptions; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; +import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; +import io.camunda.connector.agenticai.model.message.AssistantMessage; +import io.camunda.connector.agenticai.model.message.AssistantMessageBuilder; +import io.camunda.connector.agenticai.model.message.StopReason; +import io.camunda.connector.agenticai.model.message.SystemMessage; +import io.camunda.connector.agenticai.model.message.ToolCallResultMessage; +import io.camunda.connector.agenticai.model.message.UserMessage; +import io.camunda.connector.agenticai.model.message.content.Content; +import io.camunda.connector.agenticai.model.message.content.ObjectContent; +import io.camunda.connector.agenticai.model.message.content.TextContent; +import io.camunda.connector.agenticai.model.tool.ToolCall; +import io.camunda.connector.agenticai.model.tool.ToolCallResult; +import io.camunda.connector.agenticai.model.tool.ToolDefinition; +import io.camunda.connector.api.error.ConnectorException; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.commons.lang3.StringUtils; +import org.springframework.lang.Nullable; + +/** + * Native {@link ChatModelApi} for the Anthropic Messages API (direct backend), driving the {@code + * anthropic-java} SDK's blocking {@code messages().create(...)} endpoint. + * + *

    Phase B scope (text-only): user / assistant / tool-result content is restricted to text; + * multimodal content blocks, reasoning round-tripping and prompt caching are deferred to Phase E. + * {@link ChatOptions#reasoning()} and {@link ChatOptions#cacheRetention()} are accepted but ignored + * in this phase. Streaming is not yet wired in either — {@link ChatStreamListener} is accepted but + * no events are emitted. + */ +public class AnthropicMessagesChatModelApi implements ChatModelApi { + + private static final long DEFAULT_MAX_TOKENS = 4096L; + + private static final ModelCapabilities CAPABILITIES = + new ModelCapabilities( + List.of(Modality.TEXT), + List.of(Modality.TEXT), + List.of(Modality.TEXT), + false, + false, + false, + true, + null, + null); + + private final AnthropicClient client; + private final String model; + @Nullable private final Long configuredMaxTokens; + @Nullable private final Double temperature; + @Nullable private final Double topP; + @Nullable private final Long topK; + + public AnthropicMessagesChatModelApi( + AnthropicClient client, + String model, + @Nullable Long configuredMaxTokens, + @Nullable Double temperature, + @Nullable Double topP, + @Nullable Long topK) { + this.client = Objects.requireNonNull(client, "client"); + this.model = Objects.requireNonNull(model, "model"); + this.configuredMaxTokens = configuredMaxTokens; + this.temperature = temperature; + this.topP = topP; + this.topK = topK; + } + + @Override + public ModelCapabilities capabilities() { + return CAPABILITIES; + } + + @Override + public CompletableFuture complete( + ChatRequest request, ChatOptions options, ChatStreamListener listener) { + try { + final var params = buildParams(request, options); + final var response = client.messages().create(params); + return CompletableFuture.completedFuture(new ChatResponse(toAssistantMessage(response))); + } catch (RuntimeException e) { + return CompletableFuture.failedFuture(wrapModelCallFailure(e)); + } + } + + private MessageCreateParams buildParams(ChatRequest request, ChatOptions options) { + final var builder = + MessageCreateParams.builder().model(model).maxTokens(resolveMaxTokens(options)); + + Optional.ofNullable(temperature).ifPresent(builder::temperature); + Optional.ofNullable(topP).ifPresent(builder::topP); + Optional.ofNullable(topK).ifPresent(builder::topK); + + final var messages = request.messages(); + if (messages != null) { + for (var message : messages) { + switch (message) { + case SystemMessage system -> builder.system(systemPrompt(system)); + case UserMessage user -> builder.addMessage(toMessageParam(user)); + case AssistantMessage assistant -> builder.addMessage(toMessageParam(assistant)); + case ToolCallResultMessage toolResult -> builder.addMessage(toMessageParam(toolResult)); + default -> + throw new IllegalArgumentException( + "Unsupported message type: " + message.getClass().getSimpleName()); + } + } + } + + final var toolDefinitions = request.toolDefinitions(); + if (toolDefinitions != null && !toolDefinitions.isEmpty()) { + builder.tools(toolDefinitions.stream().map(this::toToolUnion).toList()); + } + + return builder.build(); + } + + private long resolveMaxTokens(ChatOptions options) { + if (options != null && options.maxOutputTokens() != null) { + return options.maxOutputTokens().longValue(); + } + return configuredMaxTokens != null ? configuredMaxTokens : DEFAULT_MAX_TOKENS; + } + + private static String systemPrompt(SystemMessage system) { + if (system.content() == null || system.content().isEmpty()) { + return ""; + } + if (system.content().size() == 1 && system.content().getFirst() instanceof TextContent t) { + return t.text(); + } + throw new IllegalArgumentException( + "SystemMessage currently only supports a single TextContent block."); + } + + private MessageParam toMessageParam(UserMessage message) { + final var blocks = textOnlyBlocks(message.content()); + return MessageParam.builder().role(MessageParam.Role.USER).contentOfBlockParams(blocks).build(); + } + + private MessageParam toMessageParam(AssistantMessage message) { + final var blocks = new ArrayList(); + if (message.content() != null) { + blocks.addAll(textOnlyBlocks(message.content())); + } + if (message.toolCalls() != null) { + for (var call : message.toolCalls()) { + blocks.add(toolUseBlock(call)); + } + } + return MessageParam.builder() + .role(MessageParam.Role.ASSISTANT) + .contentOfBlockParams(blocks) + .build(); + } + + private MessageParam toMessageParam(ToolCallResultMessage message) { + final var blocks = + message.results().stream().map(AnthropicMessagesChatModelApi::toolResultBlock).toList(); + return MessageParam.builder().role(MessageParam.Role.USER).contentOfBlockParams(blocks).build(); + } + + private static List textOnlyBlocks(List content) { + if (content == null) { + return List.of(); + } + final var blocks = new ArrayList(); + for (var c : content) { + blocks.add(textOnlyBlock(c)); + } + return blocks; + } + + private static ContentBlockParam textOnlyBlock(Content content) { + if (content instanceof TextContent text) { + return ContentBlockParam.ofText(TextBlockParam.builder().text(text.text()).build()); + } + if (content instanceof ObjectContent object) { + return ContentBlockParam.ofText( + TextBlockParam.builder().text(String.valueOf(object.content())).build()); + } + throw new IllegalArgumentException( + "Unsupported content block for text-only Anthropic Messages API: " + + content.getClass().getSimpleName()); + } + + private static ContentBlockParam toolUseBlock(ToolCall call) { + final var inputBuilder = ToolUseBlockParam.Input.builder(); + if (call.arguments() != null) { + call.arguments() + .forEach((key, value) -> inputBuilder.putAdditionalProperty(key, JsonValue.from(value))); + } + return ContentBlockParam.ofToolUse( + ToolUseBlockParam.builder() + .id(call.id()) + .name(call.name()) + .input(inputBuilder.build()) + .build()); + } + + private static ContentBlockParam toolResultBlock(ToolCallResult result) { + final var b = ToolResultBlockParam.builder().toolUseId(result.id()); + final var content = result.content(); + if (content == null) { + b.content(ToolCallResult.CONTENT_NO_RESULT); + } else if (content instanceof String s) { + b.content(s); + } else { + b.contentAsJson(content); + } + final var interrupted = + result.properties() != null + && Boolean.TRUE.equals(result.properties().get(ToolCallResult.PROPERTY_INTERRUPTED)); + if (interrupted) { + b.isError(true); + } + return ContentBlockParam.ofToolResult(b.build()); + } + + private ToolUnion toToolUnion(ToolDefinition definition) { + final var tool = + Tool.builder().name(definition.name()).inputSchema(toInputSchema(definition.inputSchema())); + if (definition.description() != null) { + tool.description(definition.description()); + } + return ToolUnion.ofTool(tool.build()); + } + + private static final Set KNOWN_SCHEMA_KEYS = Set.of("type", "properties", "required"); + + @SuppressWarnings("unchecked") + private static Tool.InputSchema toInputSchema(Map schemaMap) { + final var builder = Tool.InputSchema.builder(); + if (schemaMap == null || schemaMap.isEmpty()) { + return builder.build(); + } + + final var type = schemaMap.get("type"); + if (type != null) { + builder.type(JsonValue.from(type)); + } + + final var properties = schemaMap.get("properties"); + if (properties instanceof Map propsMap) { + final var pb = Tool.InputSchema.Properties.builder(); + ((Map) propsMap) + .forEach((key, value) -> pb.putAdditionalProperty(key, JsonValue.from(value))); + builder.properties(pb.build()); + } + + final var required = schemaMap.get("required"); + if (required instanceof List reqList) { + builder.required(reqList.stream().map(String::valueOf).toList()); + } + + schemaMap.forEach( + (key, value) -> { + if (!KNOWN_SCHEMA_KEYS.contains(key)) { + builder.putAdditionalProperty(key, JsonValue.from(value)); + } + }); + + return builder.build(); + } + + private AssistantMessage toAssistantMessage(Message message) { + final var builder = AssistantMessage.builder(); + builder.metadata(Map.of("timestamp", ZonedDateTime.now())); + + if (StringUtils.isNotBlank(message.id())) { + builder.messageId(message.id()); + } + final var modelId = message.model().asString(); + if (StringUtils.isNotBlank(modelId)) { + builder.modelId(modelId); + } + + final var contentBlocks = new ArrayList(); + final var toolCalls = new ArrayList(); + for (ContentBlock block : message.content()) { + if (block.isText()) { + final var text = block.asText().text(); + if (StringUtils.isNotBlank(text)) { + contentBlocks.add(TextContent.textContent(text)); + } + } else if (block.isToolUse()) { + final var toolUse = block.asToolUse(); + toolCalls.add(toToolCall(toolUse.id(), toolUse.name(), toolUse._input())); + } + // ignore other block types (thinking / server-tool-* / etc.) for the text-only first cut + } + if (!contentBlocks.isEmpty()) { + builder.content(contentBlocks); + } + builder.toolCalls(toolCalls); + + message + .stopReason() + .map(AnthropicMessagesChatModelApi::toStopReason) + .ifPresent(builder::stopReason); + builder.usage(toTokenUsage(message.usage())); + + return finalizeBuilder(builder); + } + + private static AssistantMessage finalizeBuilder(AssistantMessageBuilder builder) { + return builder.build(); + } + + @SuppressWarnings("unchecked") + private static ToolCall toToolCall(String id, String name, JsonValue input) { + final var arguments = + Optional.ofNullable(input.convert(Map.class)) + .map(map -> (Map) map) + .orElseGet(LinkedHashMap::new); + return ToolCall.builder().id(id).name(name).arguments(arguments).build(); + } + + private static StopReason toStopReason(com.anthropic.models.messages.StopReason stopReason) { + final var known = stopReason.known(); + if (known == null) { + return null; + } + return switch (known) { + case END_TURN, STOP_SEQUENCE -> StopReason.STOP; + case MAX_TOKENS -> StopReason.LENGTH; + case TOOL_USE -> StopReason.TOOL_USE; + case PAUSE_TURN -> StopReason.STOP; + case REFUSAL -> StopReason.CONTENT_FILTERED; + }; + } + + private static AgentMetrics.TokenUsage toTokenUsage(Usage usage) { + if (usage == null) { + return AgentMetrics.TokenUsage.empty(); + } + final var builder = + AgentMetrics.TokenUsage.builder() + .inputTokenCount((int) usage.inputTokens()) + .outputTokenCount((int) usage.outputTokens()); + usage.cacheReadInputTokens().ifPresent(v -> builder.cacheReadInputTokenCount(v.intValue())); + usage + .cacheCreationInputTokens() + .ifPresent(v -> builder.cacheCreationInputTokenCount(v.intValue())); + return builder.build(); + } + + private static ConnectorException wrapModelCallFailure(RuntimeException e) { + final var message = + Optional.ofNullable(e.getMessage()) + .filter(StringUtils::isNotBlank) + .orElseGet(() -> e.getClass().getSimpleName()); + return new ConnectorException( + ERROR_CODE_FAILED_MODEL_CALL, "Anthropic Messages call failed: %s".formatted(message), e); + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java new file mode 100644 index 00000000000..920af38a6f7 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java @@ -0,0 +1,91 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.anthropic; + +import com.anthropic.client.AnthropicClient; +import com.anthropic.client.okhttp.AnthropicOkHttpClient; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicConnection; +import java.time.Duration; +import java.util.Optional; +import org.apache.commons.lang3.StringUtils; +import org.springframework.lang.Nullable; + +/** + * Native Anthropic Messages factory for the {@code anthropic} discriminator (direct backend). + * Builds an {@link AnthropicClient} using the published OkHttp transport from a {@link + * AnthropicProviderConfiguration} and produces an {@link AnthropicMessagesChatModelApi} per + * invocation. + * + *

    Replaces the LangChain4j bridge factory at this discriminator. Cloud backends (Bedrock / + * Vertex / Foundry) will land in Phase G via additional factory variants. + */ +public class AnthropicMessagesChatModelApiFactory + implements ChatModelApiFactory { + + public static final String API_FAMILY = "anthropic-messages"; + + @Nullable private final Duration defaultTimeout; + + public AnthropicMessagesChatModelApiFactory(@Nullable Duration defaultTimeout) { + this.defaultTimeout = defaultTimeout; + } + + @Override + public String providerType() { + return AnthropicProviderConfiguration.ANTHROPIC_ID; + } + + @Override + public String apiFamily() { + return API_FAMILY; + } + + @Override + public Class configurationType() { + return AnthropicProviderConfiguration.class; + } + + @Override + public ChatModelApi create(AnthropicProviderConfiguration configuration) { + final var connection = configuration.anthropic(); + final var client = buildClient(connection); + final var parameters = connection.model().parameters(); + return new AnthropicMessagesChatModelApi( + client, + connection.model().model(), + parameters != null && parameters.maxTokens() != null + ? parameters.maxTokens().longValue() + : null, + parameters != null ? parameters.temperature() : null, + parameters != null ? parameters.topP() : null, + parameters != null && parameters.topK() != null ? parameters.topK().longValue() : null); + } + + private AnthropicClient buildClient(AnthropicConnection connection) { + final var builder = + AnthropicOkHttpClient.builder().apiKey(connection.authentication().apiKey()); + + if (StringUtils.isNotBlank(connection.endpoint())) { + builder.baseUrl(connection.endpoint()); + } + + final var timeout = resolveTimeout(connection); + if (timeout != null) { + builder.timeout(timeout); + } + + return builder.build(); + } + + @Nullable + private Duration resolveTimeout(AnthropicConnection connection) { + return Optional.ofNullable(connection.timeouts()).map(t -> t.timeout()).orElse(defaultTimeout); + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java index ec406f07002..ec607be877b 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java @@ -41,6 +41,7 @@ import io.camunda.connector.agenticai.aiagent.agent.ToolCallResultDocumentExtractor; import io.camunda.connector.agenticai.aiagent.framework.ChatClientImpl; import io.camunda.connector.agenticai.aiagent.framework.ChatModelApiRegistryImpl; +import io.camunda.connector.agenticai.aiagent.framework.anthropic.AnthropicMessagesApiConfiguration; import io.camunda.connector.agenticai.aiagent.framework.api.ChatClient; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiRegistry; @@ -82,6 +83,7 @@ @ConditionalOnBooleanProperty(value = "camunda.connector.agenticai.enabled", matchIfMissing = true) @EnableConfigurationProperties(AgenticAiConnectorsConfigurationProperties.class) @Import({ + AnthropicMessagesApiConfiguration.class, AgenticAiLangchain4JFrameworkConfiguration.class, McpDiscoveryConfiguration.class, McpClientConfiguration.class, diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java new file mode 100644 index 00000000000..94a69884108 --- /dev/null +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java @@ -0,0 +1,250 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.anthropic; + +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.assistantMessage; +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.systemMessage; +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.toolCallResultMessage; +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.userMessage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import com.anthropic.client.AnthropicClient; +import com.anthropic.core.JsonValue; +import com.anthropic.models.messages.ContentBlock; +import com.anthropic.models.messages.MessageCreateParams; +import com.anthropic.models.messages.MessageParam; +import com.anthropic.models.messages.Model; +import com.anthropic.models.messages.StopReason; +import com.anthropic.models.messages.TextBlock; +import com.anthropic.models.messages.TextCitation; +import com.anthropic.models.messages.ToolUseBlock; +import com.anthropic.models.messages.Usage; +import com.anthropic.services.blocking.MessageService; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatOptions; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; +import io.camunda.connector.agenticai.model.tool.ToolCall; +import io.camunda.connector.agenticai.model.tool.ToolCallResult; +import io.camunda.connector.agenticai.model.tool.ToolDefinition; +import io.camunda.connector.api.error.ConnectorException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletionException; +import org.assertj.core.groups.Tuple; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class AnthropicMessagesChatModelApiTest { + + private static final String MODEL_ID = "claude-sonnet-4-6"; + + @Mock private AnthropicClient client; + @Mock private MessageService messageService; + + @Captor private ArgumentCaptor paramsCaptor; + + private AnthropicMessagesChatModelApi api; + + @BeforeEach + void setUp() { + when(client.messages()).thenReturn(messageService); + api = new AnthropicMessagesChatModelApi(client, MODEL_ID, 1024L, null, null, null); + } + + @Test + void capabilitiesReturnsTextOnlyConservativeProfile() { + ModelCapabilities caps = api.capabilities(); + assertThat(caps.userMessageModalities()).containsExactly(Modality.TEXT); + assertThat(caps.toolResultModalities()).containsExactly(Modality.TEXT); + assertThat(caps.assistantMessageModalities()).containsExactly(Modality.TEXT); + assertThat(caps.supportsReasoning()).isFalse(); + assertThat(caps.supportsPromptCaching()).isFalse(); + assertThat(caps.supportsParallelToolCalls()).isTrue(); + } + + @Test + void buildsExpectedMessageCreateParamsForSimpleConversation() { + when(messageService.create((MessageCreateParams) paramsCaptor.capture())) + .thenReturn(textOnlyResponse("Hi there!")); + + var request = + new ChatRequest( + List.of(systemMessage("be helpful"), userMessage("hello")), List.of(), null); + + api.complete(request, defaultOptions(), ChatStreamListener.NOOP).join(); + + var params = paramsCaptor.getValue(); + assertThat(params.model().asString()).isEqualTo(MODEL_ID); + assertThat(params.maxTokens()).isEqualTo(1024L); + assertThat(params.system()).isPresent(); + assertThat(params.system().get().string()).hasValue("be helpful"); + assertThat(params.messages()).hasSize(1); + var first = params.messages().getFirst(); + assertThat(first.role()).isEqualTo(MessageParam.Role.USER); + } + + @Test + void appliesChatOptionsMaxOutputTokensOverride() { + when(messageService.create((MessageCreateParams) paramsCaptor.capture())) + .thenReturn(textOnlyResponse("ok")); + + var options = new ChatOptions(2048, null, null, Map.of()); + api.complete( + new ChatRequest(List.of(userMessage("hi")), List.of(), null), + options, + ChatStreamListener.NOOP) + .join(); + + assertThat(paramsCaptor.getValue().maxTokens()).isEqualTo(2048L); + } + + @Test + void mapsAssistantToolCallsBackToContentBlocks() { + when(messageService.create(any(MessageCreateParams.class))) + .thenReturn(toolUseResponse("getWeather", "abc", Map.of("location", "MUC"))); + + var response = + api.complete( + new ChatRequest(List.of(userMessage("weather?")), tools(), null), + defaultOptions(), + ChatStreamListener.NOOP) + .join(); + + var assistant = response.assistantMessage(); + assertThat(assistant.toolCalls()) + .extracting(ToolCall::id, ToolCall::name) + .containsExactly(Tuple.tuple("abc", "getWeather")); + assertThat(assistant.toolCalls().getFirst().arguments()).containsEntry("location", "MUC"); + assertThat(assistant.stopReason()) + .isEqualTo(io.camunda.connector.agenticai.model.message.StopReason.TOOL_USE); + } + + @Test + void priorAssistantToolCallsAndResultsRoundTripIntoParams() { + when(messageService.create((MessageCreateParams) paramsCaptor.capture())) + .thenReturn(textOnlyResponse("ack")); + + var prior = + assistantMessage( + "let me check", + List.of( + ToolCall.builder() + .id("abc") + .name("getWeather") + .arguments(Map.of("location", "MUC")) + .build())); + var results = + toolCallResultMessage( + List.of( + ToolCallResult.builder().id("abc").name("getWeather").content("Sunny").build())); + + api.complete( + new ChatRequest( + List.of(userMessage("weather?"), prior, results, userMessage("thanks")), + tools(), + null), + defaultOptions(), + ChatStreamListener.NOOP) + .join(); + + var params = paramsCaptor.getValue(); + assertThat(params.messages()).hasSize(4); + assertThat(params.messages().get(1).role()).isEqualTo(MessageParam.Role.ASSISTANT); + assertThat(params.messages().get(2).role()).isEqualTo(MessageParam.Role.USER); + } + + @Test + void wrapsSdkExceptionInConnectorException() { + when(messageService.create(any(MessageCreateParams.class))) + .thenThrow(new RuntimeException("boom")); + + var future = + api.complete( + new ChatRequest(List.of(userMessage("hi")), List.of(), null), + defaultOptions(), + ChatStreamListener.NOOP); + + assertThatThrownBy(future::join) + .isInstanceOf(CompletionException.class) + .cause() + .isInstanceOf(ConnectorException.class) + .hasMessageContaining("Anthropic Messages call failed"); + } + + private static ChatOptions defaultOptions() { + return new ChatOptions(null, null, null, Map.of()); + } + + private static List tools() { + return List.of( + ToolDefinition.builder() + .name("getWeather") + .description("Returns the current weather") + .inputSchema( + Map.of( + "type", "object", + "properties", Map.of("location", Map.of("type", "string")), + "required", List.of("location"))) + .build()); + } + + private static com.anthropic.models.messages.Message textOnlyResponse(String text) { + return baseResponseBuilder("msg_1", StopReason.END_TURN) + .addContent( + TextBlock.builder().text(text).citations((java.util.List) null).build()) + .usage(usage(10, 5)) + .build(); + } + + private static com.anthropic.models.messages.Message toolUseResponse( + String name, String id, Map input) { + var inputBuilder = ToolUseBlock.builder().id(id).name(name); + inputBuilder.input(JsonValue.from(input)); + inputBuilder.caller(com.anthropic.models.messages.DirectCaller.builder().build()); + return baseResponseBuilder("msg_2", StopReason.TOOL_USE) + .addContent(ContentBlock.ofToolUse(inputBuilder.build())) + .usage(usage(15, 8)) + .build(); + } + + private static com.anthropic.models.messages.Message.Builder baseResponseBuilder( + String id, StopReason stopReason) { + return com.anthropic.models.messages.Message.builder() + .id(id) + .model(Model.of(MODEL_ID)) + .container(java.util.Optional.empty()) + .stopReason(stopReason) + .stopSequence(java.util.Optional.empty()); + } + + private static Usage usage(long input, long output) { + return Usage.builder() + .inputTokens(input) + .outputTokens(output) + .cacheCreation(java.util.Optional.empty()) + .cacheCreationInputTokens(java.util.Optional.empty()) + .cacheReadInputTokens(java.util.Optional.empty()) + .inferenceGeo(java.util.Optional.empty()) + .serverToolUse(java.util.Optional.empty()) + .serviceTier(java.util.Optional.empty()) + .build(); + } +} From 5b940670ab0af0dac6c5dccf95af1ad939db0595 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 15:10:49 +0200 Subject: [PATCH 53/81] feat(agentic-ai): native OpenAI Chat Completions ChatModelApi (text-only) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the LangChain4j bridge for the `openai` and `openaiCompatible` discriminators with a native impl driving the openai-java SDK's blocking Chat Completions endpoint. Azure stays bridged until Phase G. The native factories register under the same Spring bean names as the bridge factories so the bridge's `@ConditionalOnMissingBean(name=...)` hands the discriminators over whenever the OpenAI SDK is on the classpath. `OpenAiChatCompletionsChatModelApi` is shared by both factories — they only differ in OkHttp client construction (direct auth/baseUrl vs custom endpoint, headers, query params, optional auth). Phase C scope (per ADR-004 plan revision): - Text-only content (TextContent + ObjectContent → text). Multimodal blocks rejected; revisited in Phase E. - Conservative hardcoded `ModelCapabilities` (text-only, no reasoning, no caching, parallel tool calls true). - `OpenAiToolConverter` is shared with the upcoming Responses impl in Phase D — both endpoints accept the same tools[] JSON shape. - Reasoning, prompt caching, streaming all deferred to Phase E. --- connectors/agentic-ai/pom.xml | 11 + .../OpenAiChatCompletionsChatModelApi.java | 358 ++++++++++++++++++ .../OpenAiChatModelApiConfiguration.java | 50 +++ .../openai/OpenAiChatModelApiFactory.java | 96 +++++ .../OpenAiCompatibleChatModelApiFactory.java | 98 +++++ .../framework/openai/OpenAiToolConverter.java | 53 +++ .../AgenticAiConnectorsAutoConfiguration.java | 2 + ...OpenAiChatCompletionsChatModelApiTest.java | 259 +++++++++++++ 8 files changed, 927 insertions(+) create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiConfiguration.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiCompatibleChatModelApiFactory.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiToolConverter.java create mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApiTest.java diff --git a/connectors/agentic-ai/pom.xml b/connectors/agentic-ai/pom.xml index 8870e363065..c5c5b0fabbe 100644 --- a/connectors/agentic-ai/pom.xml +++ b/connectors/agentic-ai/pom.xml @@ -23,6 +23,7 @@ 4.3.1 0.3.3.Final 2.16.1 + 4.17.0 @@ -144,6 +145,16 @@ anthropic-java-client-okhttp ${version.anthropic-java} + + com.openai + openai-java-core + ${version.openai-java} + + + com.openai + openai-java-client-okhttp + ${version.openai-java} + diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java new file mode 100644 index 00000000000..64590d3f791 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java @@ -0,0 +1,358 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.openai; + +import static io.camunda.connector.agenticai.aiagent.agent.AgentErrorCodes.ERROR_CODE_FAILED_MODEL_CALL; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.client.OpenAIClient; +import com.openai.models.chat.completions.ChatCompletion; +import com.openai.models.chat.completions.ChatCompletionAssistantMessageParam; +import com.openai.models.chat.completions.ChatCompletionCreateParams; +import com.openai.models.chat.completions.ChatCompletionMessageFunctionToolCall; +import com.openai.models.chat.completions.ChatCompletionMessageToolCall; +import com.openai.models.chat.completions.ChatCompletionToolMessageParam; +import com.openai.models.completions.CompletionUsage; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatOptions; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; +import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; +import io.camunda.connector.agenticai.model.message.AssistantMessage; +import io.camunda.connector.agenticai.model.message.StopReason; +import io.camunda.connector.agenticai.model.message.SystemMessage; +import io.camunda.connector.agenticai.model.message.ToolCallResultMessage; +import io.camunda.connector.agenticai.model.message.UserMessage; +import io.camunda.connector.agenticai.model.message.content.Content; +import io.camunda.connector.agenticai.model.message.content.ObjectContent; +import io.camunda.connector.agenticai.model.message.content.TextContent; +import io.camunda.connector.agenticai.model.tool.ToolCall; +import io.camunda.connector.agenticai.model.tool.ToolCallResult; +import io.camunda.connector.api.error.ConnectorException; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import org.apache.commons.lang3.StringUtils; +import org.springframework.lang.Nullable; + +/** + * Native {@link ChatModelApi} for the OpenAI Chat Completions endpoint, driving the {@code + * openai-java} SDK's blocking {@code chat().completions().create(...)} call. + * + *

    Phase C scope (text-only): user / assistant / tool-result content is restricted to text; + * multimodal content blocks, reasoning round-tripping (encrypted reasoning items don't apply to + * Chat Completions), and prompt caching are deferred to Phase E. {@link + * ChatOptions#cacheRetention()} is accepted but ignored. Streaming is not yet wired in either — + * {@link ChatStreamListener} is accepted but no events are emitted. + * + *

    Used by both the {@code openai} and {@code openaiCompatible} discriminators (the OkHttp {@link + * OpenAIClient} differs only in baseUrl / auth construction, which the factory handles). + */ +public class OpenAiChatCompletionsChatModelApi implements ChatModelApi { + + private static final ModelCapabilities CAPABILITIES = + new ModelCapabilities( + List.of(Modality.TEXT), + List.of(Modality.TEXT), + List.of(Modality.TEXT), + false, + false, + false, + true, + null, + null); + + private static final TypeReference> MAP_TYPE_REF = new TypeReference<>() {}; + + private final OpenAIClient client; + private final String model; + private final ObjectMapper objectMapper; + @Nullable private final Long configuredMaxCompletionTokens; + @Nullable private final Double temperature; + @Nullable private final Double topP; + + public OpenAiChatCompletionsChatModelApi( + OpenAIClient client, + String model, + ObjectMapper objectMapper, + @Nullable Long configuredMaxCompletionTokens, + @Nullable Double temperature, + @Nullable Double topP) { + this.client = Objects.requireNonNull(client, "client"); + this.model = Objects.requireNonNull(model, "model"); + this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper"); + this.configuredMaxCompletionTokens = configuredMaxCompletionTokens; + this.temperature = temperature; + this.topP = topP; + } + + @Override + public ModelCapabilities capabilities() { + return CAPABILITIES; + } + + @Override + public CompletableFuture complete( + ChatRequest request, ChatOptions options, ChatStreamListener listener) { + try { + final var params = buildParams(request, options); + final var completion = client.chat().completions().create(params); + return CompletableFuture.completedFuture(new ChatResponse(toAssistantMessage(completion))); + } catch (RuntimeException e) { + return CompletableFuture.failedFuture(wrapModelCallFailure(e)); + } + } + + private ChatCompletionCreateParams buildParams(ChatRequest request, ChatOptions options) { + final var builder = ChatCompletionCreateParams.builder().model(model); + + final var maxTokens = resolveMaxCompletionTokens(options); + if (maxTokens != null) { + builder.maxCompletionTokens(maxTokens); + } + Optional.ofNullable(temperature).ifPresent(builder::temperature); + Optional.ofNullable(topP).ifPresent(builder::topP); + + final var messages = request.messages(); + if (messages != null) { + for (var message : messages) { + switch (message) { + case SystemMessage system -> builder.addSystemMessage(systemPrompt(system)); + case UserMessage user -> builder.addUserMessage(extractText(user.content())); + case AssistantMessage assistant -> builder.addMessage(toAssistantParam(assistant)); + case ToolCallResultMessage toolResults -> addToolResultMessages(builder, toolResults); + default -> + throw new IllegalArgumentException( + "Unsupported message type: " + message.getClass().getSimpleName()); + } + } + } + + final var toolDefinitions = request.toolDefinitions(); + if (toolDefinitions != null && !toolDefinitions.isEmpty()) { + builder.tools(OpenAiToolConverter.toTools(toolDefinitions)); + } + + return builder.build(); + } + + @Nullable + private Long resolveMaxCompletionTokens(ChatOptions options) { + if (options != null && options.maxOutputTokens() != null) { + return options.maxOutputTokens().longValue(); + } + return configuredMaxCompletionTokens; + } + + private static String systemPrompt(SystemMessage system) { + return extractText(system.content()); + } + + private static String extractText(List content) { + if (content == null || content.isEmpty()) { + return ""; + } + final var sb = new StringBuilder(); + for (var c : content) { + sb.append(textOf(c)); + } + return sb.toString(); + } + + private static String textOf(Content content) { + if (content instanceof TextContent text) { + return text.text(); + } + if (content instanceof ObjectContent object) { + return String.valueOf(object.content()); + } + throw new IllegalArgumentException( + "Unsupported content block for text-only OpenAI Chat Completions API: " + + content.getClass().getSimpleName()); + } + + private ChatCompletionAssistantMessageParam toAssistantParam(AssistantMessage message) { + final var builder = ChatCompletionAssistantMessageParam.builder(); + final var text = message.content() != null ? extractText(message.content()) : ""; + if (StringUtils.isNotEmpty(text)) { + builder.content(text); + } + if (message.toolCalls() != null) { + for (var call : message.toolCalls()) { + builder.addToolCall(toFunctionToolCall(call)); + } + } + return builder.build(); + } + + private ChatCompletionMessageToolCall toFunctionToolCall(ToolCall call) { + final var args = call.arguments() != null ? toJsonString(call.arguments()) : "{}"; + return ChatCompletionMessageToolCall.ofFunction( + ChatCompletionMessageFunctionToolCall.builder() + .id(call.id()) + .function( + ChatCompletionMessageFunctionToolCall.Function.builder() + .name(call.name()) + .arguments(args) + .build()) + .build()); + } + + private void addToolResultMessages( + ChatCompletionCreateParams.Builder builder, ToolCallResultMessage message) { + for (var result : message.results()) { + builder.addMessage(toToolResultParam(result)); + } + } + + private ChatCompletionToolMessageParam toToolResultParam(ToolCallResult result) { + final var b = ChatCompletionToolMessageParam.builder(); + if (result.id() != null) { + b.toolCallId(result.id()); + } + final var content = result.content(); + if (content == null) { + b.content(ToolCallResult.CONTENT_NO_RESULT); + } else if (content instanceof String s) { + b.content(s); + } else { + b.content(toJsonString(content)); + } + return b.build(); + } + + private String toJsonString(Object value) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception e) { + throw new IllegalStateException("Failed to serialize tool argument value", e); + } + } + + private AssistantMessage toAssistantMessage(ChatCompletion completion) { + if (completion.choices().isEmpty()) { + throw new IllegalStateException("OpenAI Chat Completions returned no choices"); + } + + final var choice = completion.choices().getFirst(); + final var message = choice.message(); + final var builder = AssistantMessage.builder(); + builder.metadata(Map.of("timestamp", ZonedDateTime.now())); + + if (StringUtils.isNotBlank(completion.id())) { + builder.messageId(completion.id()); + } + if (StringUtils.isNotBlank(completion.model())) { + builder.modelId(completion.model()); + } + + final var content = new ArrayList(); + message + .content() + .filter(StringUtils::isNotBlank) + .map(TextContent::textContent) + .ifPresent(content::add); + if (!content.isEmpty()) { + builder.content(content); + } + + final var toolCalls = new ArrayList(); + message + .toolCalls() + .ifPresent( + calls -> { + for (var call : calls) { + if (call.isFunction()) { + toolCalls.add(toToolCall(call.asFunction())); + } + } + }); + builder.toolCalls(toolCalls); + + builder.stopReason(toStopReason(choice.finishReason())); + + completion + .usage() + .map(OpenAiChatCompletionsChatModelApi::toTokenUsage) + .ifPresent(builder::usage); + + return builder.build(); + } + + private ToolCall toToolCall(ChatCompletionMessageFunctionToolCall functionCall) { + final var function = functionCall.function(); + final var arguments = parseArguments(function.arguments()); + return ToolCall.builder() + .id(functionCall.id()) + .name(function.name()) + .arguments(arguments) + .build(); + } + + private Map parseArguments(String json) { + if (json == null || json.isBlank()) { + return new LinkedHashMap<>(); + } + try { + return objectMapper.readValue(json, MAP_TYPE_REF); + } catch (Exception e) { + throw new IllegalStateException("Failed to parse tool call arguments JSON", e); + } + } + + private static StopReason toStopReason(ChatCompletion.Choice.FinishReason finishReason) { + if (finishReason == null) { + return null; + } + final var known = finishReason.known(); + if (known == null) { + return null; + } + return switch (known) { + case STOP -> StopReason.STOP; + case LENGTH -> StopReason.LENGTH; + case TOOL_CALLS, FUNCTION_CALL -> StopReason.TOOL_USE; + case CONTENT_FILTER -> StopReason.CONTENT_FILTERED; + }; + } + + private static AgentMetrics.TokenUsage toTokenUsage(CompletionUsage usage) { + final var builder = + AgentMetrics.TokenUsage.builder() + .inputTokenCount((int) usage.promptTokens()) + .outputTokenCount((int) usage.completionTokens()); + usage + .promptTokensDetails() + .flatMap(CompletionUsage.PromptTokensDetails::cachedTokens) + .ifPresent(v -> builder.cacheReadInputTokenCount(v.intValue())); + usage + .completionTokensDetails() + .flatMap(CompletionUsage.CompletionTokensDetails::reasoningTokens) + .ifPresent(v -> builder.reasoningTokenCount(v.intValue())); + return builder.build(); + } + + private static ConnectorException wrapModelCallFailure(RuntimeException e) { + final var message = + Optional.ofNullable(e.getMessage()) + .filter(StringUtils::isNotBlank) + .orElseGet(() -> e.getClass().getSimpleName()); + return new ConnectorException( + ERROR_CODE_FAILED_MODEL_CALL, + "OpenAI Chat Completions call failed: %s".formatted(message), + e); + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiConfiguration.java new file mode 100644 index 00000000000..6a24d1faa4a --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiConfiguration.java @@ -0,0 +1,50 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.openai; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.client.OpenAIClient; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; +import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsConfigurationProperties; +import io.camunda.connector.runtime.annotation.ConnectorsObjectMapper; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Registers the native OpenAI factories under the same Spring bean names as the LangChain4j bridge + * factories ({@code langchain4JOpenAiChatModelApiFactory} and {@code + * langchain4JOpenAiCompatibleChatModelApiFactory}). The bridge configuration uses + * {@code @ConditionalOnMissingBean(name = ...)}, so these native beans take over whenever the + * OpenAI SDK is on the classpath. Azure OpenAI stays on the bridge for now (Phase G). + */ +@Configuration +@ConditionalOnClass(OpenAIClient.class) +public class OpenAiChatModelApiConfiguration { + + @Bean(name = "langchain4JOpenAiChatModelApiFactory") + @ConditionalOnMissingBean(name = "langchain4JOpenAiChatModelApiFactory") + public ChatModelApiFactory openAiChatModelApiFactory( + @ConnectorsObjectMapper ObjectMapper objectMapper, + AgenticAiConnectorsConfigurationProperties properties) { + return new OpenAiChatModelApiFactory( + objectMapper, properties.aiagent().chatModel().api().defaultTimeout()); + } + + @Bean(name = "langchain4JOpenAiCompatibleChatModelApiFactory") + @ConditionalOnMissingBean(name = "langchain4JOpenAiCompatibleChatModelApiFactory") + public ChatModelApiFactory + openAiCompatibleChatModelApiFactory( + @ConnectorsObjectMapper ObjectMapper objectMapper, + AgenticAiConnectorsConfigurationProperties properties) { + return new OpenAiCompatibleChatModelApiFactory( + objectMapper, properties.aiagent().chatModel().api().defaultTimeout()); + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java new file mode 100644 index 00000000000..570432096a9 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java @@ -0,0 +1,96 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.openai; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.client.OpenAIClient; +import com.openai.client.okhttp.OpenAIOkHttpClient; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiConnection; +import java.time.Duration; +import java.util.Optional; +import org.apache.commons.lang3.StringUtils; +import org.springframework.lang.Nullable; + +/** + * Native OpenAI factory for the {@code openai} discriminator. Builds an {@link OpenAIClient} + * (OkHttp transport) from {@link OpenAiProviderConfiguration} and instantiates an {@link + * OpenAiChatCompletionsChatModelApi}. + * + *

    Phase D will add the {@code apiFamily} branch: when the config selects {@code apiFamily = + * RESPONSES}, the factory will build {@code OpenAiResponsesChatModelApi} instead, sharing this + * client construction. + */ +public class OpenAiChatModelApiFactory implements ChatModelApiFactory { + + public static final String API_FAMILY = "openai-completions"; + + private final ObjectMapper objectMapper; + @Nullable private final Duration defaultTimeout; + + public OpenAiChatModelApiFactory(ObjectMapper objectMapper, @Nullable Duration defaultTimeout) { + this.objectMapper = objectMapper; + this.defaultTimeout = defaultTimeout; + } + + @Override + public String providerType() { + return OpenAiProviderConfiguration.OPENAI_ID; + } + + @Override + public String apiFamily() { + return API_FAMILY; + } + + @Override + public Class configurationType() { + return OpenAiProviderConfiguration.class; + } + + @Override + public ChatModelApi create(OpenAiProviderConfiguration configuration) { + final var connection = configuration.openai(); + final var client = buildClient(connection); + final var parameters = connection.model().parameters(); + return new OpenAiChatCompletionsChatModelApi( + client, + connection.model().model(), + objectMapper, + parameters != null && parameters.maxCompletionTokens() != null + ? parameters.maxCompletionTokens().longValue() + : null, + parameters != null ? parameters.temperature() : null, + parameters != null ? parameters.topP() : null); + } + + private OpenAIClient buildClient(OpenAiConnection connection) { + final var builder = OpenAIOkHttpClient.builder(); + builder.apiKey(connection.authentication().apiKey()); + + if (StringUtils.isNotBlank(connection.authentication().organizationId())) { + builder.organization(connection.authentication().organizationId()); + } + if (StringUtils.isNotBlank(connection.authentication().projectId())) { + builder.project(connection.authentication().projectId()); + } + + final var timeout = resolveTimeout(connection); + if (timeout != null) { + builder.timeout(timeout); + } + + return builder.build(); + } + + @Nullable + private Duration resolveTimeout(OpenAiConnection connection) { + return Optional.ofNullable(connection.timeouts()).map(t -> t.timeout()).orElse(defaultTimeout); + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiCompatibleChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiCompatibleChatModelApiFactory.java new file mode 100644 index 00000000000..305ccbe48d5 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiCompatibleChatModelApiFactory.java @@ -0,0 +1,98 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.openai; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.client.OpenAIClient; +import com.openai.client.okhttp.OpenAIOkHttpClient; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration.OpenAiCompatibleConnection; +import java.time.Duration; +import java.util.Optional; +import org.apache.commons.lang3.StringUtils; +import org.springframework.lang.Nullable; + +/** + * Native factory for the {@code openaiCompatible} discriminator. Same {@link + * OpenAiChatCompletionsChatModelApi} impl as the OpenAI-direct factory; only the OkHttp client + * construction differs (custom baseUrl, optional API key, custom headers / query params). + */ +public class OpenAiCompatibleChatModelApiFactory + implements ChatModelApiFactory { + + public static final String API_FAMILY = "openai-completions"; + + private final ObjectMapper objectMapper; + @Nullable private final Duration defaultTimeout; + + public OpenAiCompatibleChatModelApiFactory( + ObjectMapper objectMapper, @Nullable Duration defaultTimeout) { + this.objectMapper = objectMapper; + this.defaultTimeout = defaultTimeout; + } + + @Override + public String providerType() { + return OpenAiCompatibleProviderConfiguration.OPENAI_COMPATIBLE_ID; + } + + @Override + public String apiFamily() { + return API_FAMILY; + } + + @Override + public Class configurationType() { + return OpenAiCompatibleProviderConfiguration.class; + } + + @Override + public ChatModelApi create(OpenAiCompatibleProviderConfiguration configuration) { + final var connection = configuration.openaiCompatible(); + final var client = buildClient(connection); + final var parameters = connection.model().parameters(); + return new OpenAiChatCompletionsChatModelApi( + client, + connection.model().model(), + objectMapper, + parameters != null && parameters.maxCompletionTokens() != null + ? parameters.maxCompletionTokens().longValue() + : null, + parameters != null ? parameters.temperature() : null, + parameters != null ? parameters.topP() : null); + } + + private OpenAIClient buildClient(OpenAiCompatibleConnection connection) { + final var builder = OpenAIOkHttpClient.builder().baseUrl(connection.endpoint()); + + final var auth = connection.authentication(); + final var apiKey = + auth != null && StringUtils.isNotBlank(auth.apiKey()) ? auth.apiKey() : "no-key"; + builder.apiKey(apiKey); + + if (connection.headers() != null && !connection.headers().isEmpty()) { + connection.headers().forEach(builder::putHeader); + } + if (connection.queryParameters() != null && !connection.queryParameters().isEmpty()) { + connection.queryParameters().forEach(builder::putQueryParam); + } + + final var timeout = resolveTimeout(connection); + if (timeout != null) { + builder.timeout(timeout); + } + + return builder.build(); + } + + @Nullable + private Duration resolveTimeout(OpenAiCompatibleConnection connection) { + return Optional.ofNullable(connection.timeouts()).map(t -> t.timeout()).orElse(defaultTimeout); + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiToolConverter.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiToolConverter.java new file mode 100644 index 00000000000..10ddedfd9c3 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiToolConverter.java @@ -0,0 +1,53 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.openai; + +import com.openai.core.JsonValue; +import com.openai.models.FunctionDefinition; +import com.openai.models.FunctionParameters; +import com.openai.models.chat.completions.ChatCompletionFunctionTool; +import com.openai.models.chat.completions.ChatCompletionTool; +import io.camunda.connector.agenticai.model.tool.ToolDefinition; +import java.util.List; +import java.util.Map; + +/** + * Converts {@link ToolDefinition}s into OpenAI {@link ChatCompletionTool}s. Shared between the Chat + * Completions and Responses native impls — both endpoints accept the same {@code tools[]} JSON + * shape. + */ +public final class OpenAiToolConverter { + + private OpenAiToolConverter() {} + + public static List toTools(List definitions) { + if (definitions == null || definitions.isEmpty()) { + return List.of(); + } + return definitions.stream().map(OpenAiToolConverter::toTool).toList(); + } + + public static ChatCompletionTool toTool(ToolDefinition definition) { + final var function = FunctionDefinition.builder().name(definition.name()); + if (definition.description() != null) { + function.description(definition.description()); + } + function.parameters(toFunctionParameters(definition.inputSchema())); + + return ChatCompletionTool.ofFunction( + ChatCompletionFunctionTool.builder().function(function.build()).build()); + } + + private static FunctionParameters toFunctionParameters(Map schemaMap) { + final var builder = FunctionParameters.builder(); + if (schemaMap == null || schemaMap.isEmpty()) { + return builder.putAdditionalProperty("type", JsonValue.from("object")).build(); + } + schemaMap.forEach((key, value) -> builder.putAdditionalProperty(key, JsonValue.from(value))); + return builder.build(); + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java index ec607be877b..c0b2139790a 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java @@ -47,6 +47,7 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiRegistry; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelHttpProxySupport; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.configuration.AgenticAiLangchain4JFrameworkConfiguration; +import io.camunda.connector.agenticai.aiagent.framework.openai.OpenAiChatModelApiConfiguration; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStore; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStoreRegistry; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStoreRegistryImpl; @@ -84,6 +85,7 @@ @EnableConfigurationProperties(AgenticAiConnectorsConfigurationProperties.class) @Import({ AnthropicMessagesApiConfiguration.class, + OpenAiChatModelApiConfiguration.class, AgenticAiLangchain4JFrameworkConfiguration.class, McpDiscoveryConfiguration.class, McpClientConfiguration.class, diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApiTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApiTest.java new file mode 100644 index 00000000000..bbddc38c226 --- /dev/null +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApiTest.java @@ -0,0 +1,259 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.openai; + +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.assistantMessage; +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.systemMessage; +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.toolCallResultMessage; +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.userMessage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.client.OpenAIClient; +import com.openai.models.chat.completions.ChatCompletion; +import com.openai.models.chat.completions.ChatCompletionCreateParams; +import com.openai.models.chat.completions.ChatCompletionMessage; +import com.openai.models.chat.completions.ChatCompletionMessageFunctionToolCall; +import com.openai.models.chat.completions.ChatCompletionMessageToolCall; +import com.openai.models.completions.CompletionUsage; +import com.openai.services.blocking.ChatService; +import com.openai.services.blocking.chat.ChatCompletionService; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatOptions; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; +import io.camunda.connector.agenticai.model.tool.ToolCall; +import io.camunda.connector.agenticai.model.tool.ToolCallResult; +import io.camunda.connector.agenticai.model.tool.ToolDefinition; +import io.camunda.connector.api.error.ConnectorException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletionException; +import org.assertj.core.groups.Tuple; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class OpenAiChatCompletionsChatModelApiTest { + + private static final String MODEL_ID = "gpt-4o"; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @Mock private OpenAIClient client; + @Mock private ChatService chatService; + @Mock private ChatCompletionService chatCompletionService; + + @Captor private ArgumentCaptor paramsCaptor; + + private OpenAiChatCompletionsChatModelApi api; + + @BeforeEach + void setUp() { + when(client.chat()).thenReturn(chatService); + when(chatService.completions()).thenReturn(chatCompletionService); + api = new OpenAiChatCompletionsChatModelApi(client, MODEL_ID, OBJECT_MAPPER, 1024L, null, null); + } + + @Test + void capabilitiesAreTextOnly() { + var caps = api.capabilities(); + assertThat(caps.userMessageModalities()).containsExactly(Modality.TEXT); + assertThat(caps.toolResultModalities()).containsExactly(Modality.TEXT); + assertThat(caps.assistantMessageModalities()).containsExactly(Modality.TEXT); + assertThat(caps.supportsReasoning()).isFalse(); + assertThat(caps.supportsPromptCaching()).isFalse(); + assertThat(caps.supportsParallelToolCalls()).isTrue(); + } + + @Test + void buildsExpectedParamsForSimpleConversation() { + when(chatCompletionService.create((ChatCompletionCreateParams) paramsCaptor.capture())) + .thenReturn(textOnlyResponse("hello")); + + var request = + new ChatRequest(List.of(systemMessage("be helpful"), userMessage("hi")), List.of(), null); + api.complete(request, defaultOptions(), ChatStreamListener.NOOP).join(); + + var params = paramsCaptor.getValue(); + assertThat(params.model().asString()).isEqualTo(MODEL_ID); + assertThat(params.maxCompletionTokens()).hasValue(1024L); + assertThat(params.messages()).hasSize(2); + } + + @Test + void appliesMaxOutputTokensOverride() { + when(chatCompletionService.create((ChatCompletionCreateParams) paramsCaptor.capture())) + .thenReturn(textOnlyResponse("ok")); + + var options = new ChatOptions(2048, null, null, Map.of()); + api.complete( + new ChatRequest(List.of(userMessage("hi")), List.of(), null), + options, + ChatStreamListener.NOOP) + .join(); + + assertThat(paramsCaptor.getValue().maxCompletionTokens()).hasValue(2048L); + } + + @Test + void mapsAssistantToolCallsBackToContentBlocks() { + when(chatCompletionService.create(any(ChatCompletionCreateParams.class))) + .thenReturn(toolCallResponse("getWeather", "abc", "{\"location\":\"MUC\"}")); + + var response = + api.complete( + new ChatRequest(List.of(userMessage("weather?")), tools(), null), + defaultOptions(), + ChatStreamListener.NOOP) + .join(); + + var assistant = response.assistantMessage(); + assertThat(assistant.toolCalls()) + .extracting(ToolCall::id, ToolCall::name) + .containsExactly(Tuple.tuple("abc", "getWeather")); + assertThat(assistant.toolCalls().getFirst().arguments()).containsEntry("location", "MUC"); + assertThat(assistant.stopReason()) + .isEqualTo(io.camunda.connector.agenticai.model.message.StopReason.TOOL_USE); + } + + @Test + void priorAssistantToolCallsAndResultsRoundTripIntoParams() { + when(chatCompletionService.create((ChatCompletionCreateParams) paramsCaptor.capture())) + .thenReturn(textOnlyResponse("ack")); + + var prior = + assistantMessage( + "let me check", + List.of( + ToolCall.builder() + .id("abc") + .name("getWeather") + .arguments(Map.of("location", "MUC")) + .build())); + var results = + toolCallResultMessage( + List.of( + ToolCallResult.builder().id("abc").name("getWeather").content("Sunny").build())); + + api.complete( + new ChatRequest( + List.of(userMessage("weather?"), prior, results, userMessage("thanks")), + tools(), + null), + defaultOptions(), + ChatStreamListener.NOOP) + .join(); + + var params = paramsCaptor.getValue(); + // user, assistant (with tool call), tool result, user → 4 messages + assertThat(params.messages()).hasSize(4); + } + + @Test + void wrapsSdkExceptionInConnectorException() { + when(chatCompletionService.create(any(ChatCompletionCreateParams.class))) + .thenThrow(new RuntimeException("boom")); + + var future = + api.complete( + new ChatRequest(List.of(userMessage("hi")), List.of(), null), + defaultOptions(), + ChatStreamListener.NOOP); + + assertThatThrownBy(future::join) + .isInstanceOf(CompletionException.class) + .cause() + .isInstanceOf(ConnectorException.class) + .hasMessageContaining("OpenAI Chat Completions call failed"); + } + + private static ChatOptions defaultOptions() { + return new ChatOptions(null, null, null, Map.of()); + } + + private static List tools() { + return List.of( + ToolDefinition.builder() + .name("getWeather") + .description("Returns the current weather") + .inputSchema( + Map.of( + "type", "object", + "properties", Map.of("location", Map.of("type", "string")), + "required", List.of("location"))) + .build()); + } + + private static ChatCompletion textOnlyResponse(String text) { + return ChatCompletion.builder() + .id("chatcmpl_1") + .model(MODEL_ID) + .created(1700000000L) + .addChoice( + ChatCompletion.Choice.builder() + .index(0L) + .finishReason(ChatCompletion.Choice.FinishReason.STOP) + .message( + ChatCompletionMessage.builder().content(text).refusal(Optional.empty()).build()) + .logprobs(Optional.empty()) + .build()) + .usage(usage(10, 5)) + .build(); + } + + private static ChatCompletion toolCallResponse(String name, String id, String argumentsJson) { + var toolCall = + ChatCompletionMessageToolCall.ofFunction( + ChatCompletionMessageFunctionToolCall.builder() + .id(id) + .function( + ChatCompletionMessageFunctionToolCall.Function.builder() + .name(name) + .arguments(argumentsJson) + .build()) + .build()); + + return ChatCompletion.builder() + .id("chatcmpl_2") + .model(MODEL_ID) + .created(1700000000L) + .addChoice( + ChatCompletion.Choice.builder() + .index(0L) + .finishReason(ChatCompletion.Choice.FinishReason.TOOL_CALLS) + .message( + ChatCompletionMessage.builder() + .content(Optional.empty()) + .refusal(Optional.empty()) + .addToolCall(toolCall) + .build()) + .logprobs(Optional.empty()) + .build()) + .usage(usage(15, 8)) + .build(); + } + + private static CompletionUsage usage(long prompt, long completion) { + return CompletionUsage.builder() + .promptTokens(prompt) + .completionTokens(completion) + .totalTokens(prompt + completion) + .build(); + } +} From eba8a7162128183c1056bf573394bf311698d566 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 15:21:26 +0200 Subject: [PATCH 54/81] feat(agentic-ai): native OpenAI Responses ChatModelApi + apiFamily switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an `apiFamily: COMPLETIONS | RESPONSES` field on `OpenAiProviderConfiguration` (defaulting to COMPLETIONS so saved v10 process state deserializes unchanged) and a new `OpenAiResponsesChatModelApi` that drives the openai-java SDK's blocking responses().create(...) call. `OpenAiChatModelApiFactory` now branches on `apiFamily` and constructs either the Responses or Chat Completions impl class from the same OpenAIClient. `OpenAiToolConverter` learned `toResponsesTools(...)` since the Responses endpoint uses a parallel-but-distinct Tool class. `openaiCompatible` stays Completions-only. Element template version bumps 10 → 11 (auto-generated; v10 archived under `versioned/`). README index updated to reflect the new top template version for Camunda 8.10. Phase D scope (per ADR-004 plan revision): - Text-only content; encrypted reasoning items, prompt caching (`prompt_cache_key`) and multimodal input items deferred to Phase E. - The Jackson migration deserializer for the canonical discriminator restructure stays a Phase F concern; here we lean on the field default to keep saved process state deserializing. --- connectors/agentic-ai/AI_AGENT.md | 2 +- .../agentic-ai/element-templates/README.md | 6 +- .../agenticai-aiagent-job-worker.json | 28 +- .../agenticai-aiagent-outbound-connector.json | 28 +- .../agenticai-aiagent-job-worker-hybrid.json | 28 +- ...cai-aiagent-outbound-connector-hybrid.json | 28 +- .../agenticai-aiagent-job-worker-10.json | 1803 +++++++++++++++++ ...enticai-aiagent-outbound-connector-10.json | 1782 ++++++++++++++++ .../agenticai/aiagent/AiAgentFunction.java | 2 +- .../openai/OpenAiChatModelApiFactory.java | 24 +- .../openai/OpenAiResponsesChatModelApi.java | 378 ++++ .../framework/openai/OpenAiToolConverter.java | 31 + .../provider/OpenAiProviderConfiguration.java | 39 +- .../OpenAiResponsesChatModelApiTest.java | 246 +++ 14 files changed, 4402 insertions(+), 23 deletions(-) create mode 100644 connectors/agentic-ai/element-templates/versioned/agenticai-aiagent-job-worker-10.json create mode 100644 connectors/agentic-ai/element-templates/versioned/agenticai-aiagent-outbound-connector-10.json create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApi.java create mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApiTest.java diff --git a/connectors/agentic-ai/AI_AGENT.md b/connectors/agentic-ai/AI_AGENT.md index cae54ae7af1..f143e824789 100644 --- a/connectors/agentic-ai/AI_AGENT.md +++ b/connectors/agentic-ai/AI_AGENT.md @@ -229,6 +229,6 @@ leading to the following result | Connector Info | | | --- | --- | | Type | io.camunda.agenticai:aiagent:1 | -| Version | 10 | +| Version | 11 | | Supported element types | | diff --git a/connectors/agentic-ai/element-templates/README.md b/connectors/agentic-ai/element-templates/README.md index 5d20bda5b76..39b1ad31766 100644 --- a/connectors/agentic-ai/element-templates/README.md +++ b/connectors/agentic-ai/element-templates/README.md @@ -15,7 +15,7 @@ field in each JSON file captures the same information (e.g. `^8.9` means "requires Camunda 8.9 or later"). For example, if you are on Camunda 8.9, use the AI Agent template version `7`; -if you are on Camunda 8.10, use version `10`. +if you are on Camunda 8.10, use version `11`. ## AI Agent connectors @@ -26,7 +26,7 @@ The AI Agent ships in two flavors that share the same versioning scheme. | Connector | Minimum Camunda version | Template version | File | | --- | --- | --- | --- | -| AI Agent Task | 8.10 | 10 | [`agenticai-aiagent-outbound-connector.json`](./agenticai-aiagent-outbound-connector.json) | +| AI Agent Task | 8.10 | 11 | [`agenticai-aiagent-outbound-connector.json`](./agenticai-aiagent-outbound-connector.json) | | AI Agent Task | 8.9 | 7 | [`versioned/agenticai-aiagent-outbound-connector-7.json`](./versioned/agenticai-aiagent-outbound-connector-7.json) | | AI Agent Task | 8.8 | 5 | [`versioned/agenticai-aiagent-outbound-connector-5.json`](./versioned/agenticai-aiagent-outbound-connector-5.json) | @@ -34,7 +34,7 @@ The AI Agent ships in two flavors that share the same versioning scheme. | Connector | Minimum Camunda version | Template version | File | | --- | --- | --- | --- | -| AI Agent Sub-process | 8.10 | 10 | [`agenticai-aiagent-job-worker.json`](./agenticai-aiagent-job-worker.json) | +| AI Agent Sub-process | 8.10 | 11 | [`agenticai-aiagent-job-worker.json`](./agenticai-aiagent-job-worker.json) | | AI Agent Sub-process | 8.9 | 7 | [`versioned/agenticai-aiagent-job-worker-7.json`](./versioned/agenticai-aiagent-job-worker-7.json) | | AI Agent Sub-process | 8.8 | 5 | [`versioned/agenticai-aiagent-job-worker-5.json`](./versioned/agenticai-aiagent-job-worker-5.json) | diff --git a/connectors/agentic-ai/element-templates/agenticai-aiagent-job-worker.json b/connectors/agentic-ai/element-templates/agenticai-aiagent-job-worker.json index b93dc37fb41..b1a5f826352 100644 --- a/connectors/agentic-ai/element-templates/agenticai-aiagent-job-worker.json +++ b/connectors/agentic-ai/element-templates/agenticai-aiagent-job-worker.json @@ -5,7 +5,7 @@ "description" : "Run a multi-step AI reasoning loop with dynamic tool selection", "keywords" : [ "AI", "AI Agent", "agentic orchestration" ], "documentationRef" : "https://docs.camunda.io/docs/8.10/components/connectors/out-of-the-box-connectors/agentic-ai-aiagent-subprocess/", - "version" : 10, + "version" : 11, "category" : { "id" : "connectors", "name" : "Connectors" @@ -681,6 +681,30 @@ "type" : "simple" }, "type" : "String" + }, { + "id" : "provider.openai.apiFamily", + "label" : "API", + "description" : "Which OpenAI API family to use. Chat Completions is the default for backward compatibility; Responses is the newer endpoint with structured input/output items.", + "optional" : true, + "value" : "completions", + "group" : "provider", + "binding" : { + "name" : "provider.openai.apiFamily", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Chat Completions (default)", + "value" : "completions" + }, { + "name" : "Responses", + "value" : "responses" + } ] }, { "id" : "provider.openaiCompatible.endpoint", "label" : "API endpoint", @@ -1703,7 +1727,7 @@ "id" : "version", "label" : "Version", "description" : "Version of the element template", - "value" : "10", + "value" : "11", "group" : "connector", "binding" : { "key" : "elementTemplateVersion", diff --git a/connectors/agentic-ai/element-templates/agenticai-aiagent-outbound-connector.json b/connectors/agentic-ai/element-templates/agenticai-aiagent-outbound-connector.json index a07015eb55f..e28221c51be 100644 --- a/connectors/agentic-ai/element-templates/agenticai-aiagent-outbound-connector.json +++ b/connectors/agentic-ai/element-templates/agenticai-aiagent-outbound-connector.json @@ -5,7 +5,7 @@ "description" : "Execute a single AI-powered action with tool calling capabilities", "keywords" : [ "AI", "AI Agent", "agentic orchestration" ], "documentationRef" : "https://docs.camunda.io/docs/8.10/components/connectors/out-of-the-box-connectors/agentic-ai-aiagent-task/", - "version" : 10, + "version" : 11, "category" : { "id" : "connectors", "name" : "Connectors" @@ -660,6 +660,30 @@ "type" : "simple" }, "type" : "String" + }, { + "id" : "provider.openai.apiFamily", + "label" : "API", + "description" : "Which OpenAI API family to use. Chat Completions is the default for backward compatibility; Responses is the newer endpoint with structured input/output items.", + "optional" : true, + "value" : "completions", + "group" : "provider", + "binding" : { + "name" : "provider.openai.apiFamily", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Chat Completions (default)", + "value" : "completions" + }, { + "name" : "Responses", + "value" : "responses" + } ] }, { "id" : "provider.openaiCompatible.endpoint", "label" : "API endpoint", @@ -1677,7 +1701,7 @@ "id" : "version", "label" : "Version", "description" : "Version of the element template", - "value" : "10", + "value" : "11", "group" : "connector", "binding" : { "key" : "elementTemplateVersion", diff --git a/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-job-worker-hybrid.json b/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-job-worker-hybrid.json index 26deb07fad6..5b575ac6d6e 100644 --- a/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-job-worker-hybrid.json +++ b/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-job-worker-hybrid.json @@ -5,7 +5,7 @@ "description" : "Run a multi-step AI reasoning loop with dynamic tool selection", "keywords" : [ "AI", "AI Agent", "agentic orchestration" ], "documentationRef" : "https://docs.camunda.io/docs/8.10/components/connectors/out-of-the-box-connectors/agentic-ai-aiagent-subprocess/", - "version" : 10, + "version" : 11, "category" : { "id" : "connectors", "name" : "Connectors" @@ -686,6 +686,30 @@ "type" : "simple" }, "type" : "String" + }, { + "id" : "provider.openai.apiFamily", + "label" : "API", + "description" : "Which OpenAI API family to use. Chat Completions is the default for backward compatibility; Responses is the newer endpoint with structured input/output items.", + "optional" : true, + "value" : "completions", + "group" : "provider", + "binding" : { + "name" : "provider.openai.apiFamily", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Chat Completions (default)", + "value" : "completions" + }, { + "name" : "Responses", + "value" : "responses" + } ] }, { "id" : "provider.openaiCompatible.endpoint", "label" : "API endpoint", @@ -1708,7 +1732,7 @@ "id" : "version", "label" : "Version", "description" : "Version of the element template", - "value" : "10", + "value" : "11", "group" : "connector", "binding" : { "key" : "elementTemplateVersion", diff --git a/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-outbound-connector-hybrid.json b/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-outbound-connector-hybrid.json index 45ead89d618..ed8c9f282e0 100644 --- a/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-outbound-connector-hybrid.json +++ b/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-outbound-connector-hybrid.json @@ -5,7 +5,7 @@ "description" : "Execute a single AI-powered action with tool calling capabilities", "keywords" : [ "AI", "AI Agent", "agentic orchestration" ], "documentationRef" : "https://docs.camunda.io/docs/8.10/components/connectors/out-of-the-box-connectors/agentic-ai-aiagent-task/", - "version" : 10, + "version" : 11, "category" : { "id" : "connectors", "name" : "Connectors" @@ -665,6 +665,30 @@ "type" : "simple" }, "type" : "String" + }, { + "id" : "provider.openai.apiFamily", + "label" : "API", + "description" : "Which OpenAI API family to use. Chat Completions is the default for backward compatibility; Responses is the newer endpoint with structured input/output items.", + "optional" : true, + "value" : "completions", + "group" : "provider", + "binding" : { + "name" : "provider.openai.apiFamily", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Chat Completions (default)", + "value" : "completions" + }, { + "name" : "Responses", + "value" : "responses" + } ] }, { "id" : "provider.openaiCompatible.endpoint", "label" : "API endpoint", @@ -1682,7 +1706,7 @@ "id" : "version", "label" : "Version", "description" : "Version of the element template", - "value" : "10", + "value" : "11", "group" : "connector", "binding" : { "key" : "elementTemplateVersion", diff --git a/connectors/agentic-ai/element-templates/versioned/agenticai-aiagent-job-worker-10.json b/connectors/agentic-ai/element-templates/versioned/agenticai-aiagent-job-worker-10.json new file mode 100644 index 00000000000..77b7001b799 --- /dev/null +++ b/connectors/agentic-ai/element-templates/versioned/agenticai-aiagent-job-worker-10.json @@ -0,0 +1,1803 @@ +{ + "$schema" : "https://unpkg.com/@camunda/zeebe-element-templates-json-schema/resources/schema.json", + "name" : "AI Agent Sub-process", + "id" : "io.camunda.connectors.agenticai.aiagent.jobworker.v1", + "description" : "Run a multi-step AI reasoning loop with dynamic tool selection", + "keywords" : [ "AI", "AI Agent", "agentic orchestration" ], + "documentationRef" : "https://docs.camunda.io/docs/8.10/components/connectors/out-of-the-box-connectors/agentic-ai-aiagent-subprocess/", + "version" : 10, + "category" : { + "id" : "connectors", + "name" : "Connectors" + }, + "appliesTo" : [ "bpmn:SubProcess" ], + "elementType" : { + "value" : "bpmn:AdHocSubProcess" + }, + "engines" : { + "camunda" : "^8.10" + }, + "groups" : [ { + "id" : "provider", + "label" : "Model provider", + "openByDefault" : false + }, { + "id" : "model", + "label" : "Model", + "openByDefault" : false + }, { + "id" : "systemPrompt", + "label" : "System prompt", + "tooltip" : "A system prompt is a set of foundational instructions given to a model before any user interaction begins. It defines the AI agent’s role, behavior, tone, and communication style, ensuring that responses remain consistent and aligned with the AI agent’s intended purpose. These instructions help shape how the model interprets and responds to user input throughout the conversation.", + "openByDefault" : false + }, { + "id" : "userPrompt", + "label" : "User prompt", + "tooltip" : "A user prompt is the message or question you give to the AI to start or continue a conversation. It tells the AI what you need, whether it's information, help with a task, or just a chat. The AI uses your prompt to understand how to respond.", + "openByDefault" : false + }, { + "id" : "tools", + "label" : "Tools", + "tooltip" : "Tools are optional features the AI Agent can use to perform specific tasks. Configure this if the agent should participate in a tools feedback loop.", + "openByDefault" : false + }, { + "id" : "memory", + "label" : "Memory", + "tooltip" : "Configuration of the Agent's short-term/conversational memory.", + "openByDefault" : false + }, { + "id" : "limits", + "label" : "Limits", + "openByDefault" : false + }, { + "id" : "events", + "label" : "Event handling", + "tooltip" : "Configure how event sub-process results are handled. Results are added as user messages to the running agent.", + "openByDefault" : false + }, { + "id" : "response", + "label" : "Response", + "tooltip" : "Configuration of the model response format and how to map the model response to the connector result.

    Depending on the selection, the model response will be available as response.responseText or response.responseJson.

    See documentation for details.", + "openByDefault" : false + }, { + "id" : "connector", + "label" : "Connector" + }, { + "id" : "output", + "label" : "Output mapping" + }, { + "id" : "error", + "label" : "Error handling" + }, { + "id" : "retries", + "label" : "Retries" + } ], + "properties" : [ { + "value" : "io.camunda.agenticai:aiagent-job-worker:1", + "binding" : { + "property" : "type", + "type" : "zeebe:taskDefinition" + }, + "type" : "Hidden" + }, { + "id" : "outputCollection", + "binding" : { + "property" : "outputCollection", + "type" : "zeebe:adHoc" + }, + "value" : "toolCallResults", + "type" : "Hidden" + }, { + "id" : "outputElement", + "binding" : { + "property" : "outputElement", + "type" : "zeebe:adHoc" + }, + "value" : "={\n id: toolCall._meta.id,\n name: toolCall._meta.name,\n content: toolCallResult\n}", + "type" : "Hidden" + }, { + "id" : "provider.type", + "label" : "Provider", + "description" : "Specify the LLM provider to use.", + "value" : "anthropic", + "group" : "provider", + "binding" : { + "name" : "provider.type", + "type" : "zeebe:input" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Anthropic", + "value" : "anthropic" + }, { + "name" : "AWS Bedrock", + "value" : "bedrock" + }, { + "name" : "Azure OpenAI", + "value" : "azureOpenAi" + }, { + "name" : "Google Vertex AI", + "value" : "google-vertex-ai" + }, { + "name" : "OpenAI", + "value" : "openai" + }, { + "name" : "OpenAI Compatible", + "value" : "openaiCompatible" + } ] + }, { + "id" : "provider.anthropic.endpoint", + "label" : "Endpoint", + "description" : "Optional custom API endpoint", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.apiKey", + "label" : "Anthropic API key", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.apiKey", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.anthropic.timeouts.timeout", + "label" : "Timeout", + "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.timeouts.timeout", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.bedrock.region", + "label" : "Region", + "description" : "Specify the AWS region (example: eu-west-1)", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.region", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.bedrock.endpoint", + "label" : "Endpoint", + "description" : "Custom API endpoint for VPC/PrivateLink configurations, AWS GovCloud, or other non-standard deployments.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.bedrock.authentication.type", + "label" : "Authentication", + "description" : "Specify the AWS authentication strategy. Learn more at the documentation page", + "value" : "credentials", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Credentials", + "value" : "credentials" + }, { + "name" : "API Key", + "value" : "apiKey" + }, { + "name" : "Default Credentials Chain (Hybrid/Self-Managed only)", + "value" : "defaultCredentialsChain" + } ] + }, { + "id" : "provider.bedrock.authentication.accessKey", + "label" : "Access key", + "description" : "Provide an IAM access key tailored to a user, equipped with the necessary permissions", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.authentication.accessKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.bedrock.authentication.type", + "equals" : "credentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.bedrock.authentication.secretKey", + "label" : "Secret key", + "description" : "Provide the secret key for the IAM access key", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.authentication.secretKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.bedrock.authentication.type", + "equals" : "credentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.bedrock.authentication.apiKey", + "label" : "API Key", + "description" : "Provide an API Key with permissions to interact with your AWS Bedrock Instance", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.authentication.apiKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.bedrock.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.bedrock.timeouts.timeout", + "label" : "Timeout", + "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.timeouts.timeout", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.azureOpenAi.endpoint", + "label" : "Endpoint", + "description" : "Specify Azure OpenAI endpoint. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.azureOpenAi.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.azureOpenAi.authentication.type", + "label" : "Authentication", + "description" : "Specify the Azure OpenAI authentication strategy.", + "value" : "apiKey", + "group" : "provider", + "binding" : { + "name" : "provider.azureOpenAi.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "API key", + "value" : "apiKey" + }, { + "name" : "Client credentials", + "value" : "clientCredentials" + } ] + }, { + "id" : "provider.azureOpenAi.authentication.apiKey", + "label" : "API key", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.azureOpenAi.authentication.apiKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.azureOpenAi.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.azureOpenAi.authentication.clientId", + "label" : "Client ID", + "description" : "ID of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.azureOpenAi.authentication.clientId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.azureOpenAi.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.azureOpenAi.authentication.clientSecret", + "label" : "Client secret", + "description" : "Secret of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.azureOpenAi.authentication.clientSecret", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.azureOpenAi.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.azureOpenAi.authentication.tenantId", + "label" : "Tenant ID", + "description" : "ID of a Microsoft Entra tenant. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.azureOpenAi.authentication.tenantId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.azureOpenAi.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.azureOpenAi.authentication.authorityHost", + "label" : "Authority host", + "description" : "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.azureOpenAi.authentication.authorityHost", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.azureOpenAi.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.azureOpenAi.timeouts.timeout", + "label" : "Timeout", + "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.azureOpenAi.timeouts.timeout", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.googleVertexAi.projectId", + "label" : "Project ID", + "description" : "Specify Google Cloud project ID", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.googleVertexAi.projectId", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "google-vertex-ai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.googleVertexAi.region", + "label" : "Region", + "description" : "Specify the region where AI inference should take place", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.googleVertexAi.region", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "google-vertex-ai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.googleVertexAi.authentication.type", + "label" : "Authentication", + "description" : "Specify the Google Vertex AI authentication strategy.", + "value" : "serviceAccountCredentials", + "group" : "provider", + "binding" : { + "name" : "provider.googleVertexAi.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "google-vertex-ai", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Service account credentials", + "value" : "serviceAccountCredentials" + }, { + "name" : "Application default credentials (Hybrid/Self-Managed only)", + "value" : "applicationDefaultCredentials" + } ] + }, { + "id" : "provider.googleVertexAi.authentication.jsonKey", + "label" : "JSON key of the service account", + "description" : "This is the key of the service account in JSON format.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.googleVertexAi.authentication.jsonKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.googleVertexAi.authentication.type", + "equals" : "serviceAccountCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "google-vertex-ai", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.apiKey", + "label" : "OpenAI API key", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.apiKey", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.organizationId", + "label" : "Organization ID", + "description" : "For members of multiple organizations. Details in the documentation.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.organizationId", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.projectId", + "label" : "Project ID", + "description" : "For accounts with multiple projects. Details in the documentation.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.projectId", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openai.timeouts.timeout", + "label" : "Timeout", + "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.timeouts.timeout", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openai.apiFamily", + "label" : "API", + "description" : "Which OpenAI API family to use. Chat Completions is the default for backward compatibility; Responses is the newer endpoint with structured input/output items.", + "optional" : true, + "value" : "completions", + "group" : "provider", + "binding" : { + "name" : "provider.openai.apiFamily", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Chat Completions (default)", + "value" : "completions" + }, { + "name" : "Responses", + "value" : "responses" + } ] + }, { + "id" : "provider.openaiCompatible.endpoint", + "label" : "API endpoint", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openaiCompatible.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "tooltip" : "Specify an endpoint to use the connector with an OpenAI compatible API. ", + "type" : "String" + }, { + "id" : "provider.openaiCompatible.authentication.apiKey", + "label" : "API key", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openaiCompatible.authentication.apiKey", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "tooltip" : "Leave blank if using HTTP headers for authentication.
    If an Authorization header is specified in the headers, then the API key is ignored.", + "type" : "String" + }, { + "id" : "provider.openaiCompatible.headers", + "label" : "Headers", + "description" : "Map of HTTP headers to add to the request.", + "optional" : true, + "feel" : "required", + "group" : "provider", + "binding" : { + "name" : "provider.openaiCompatible.headers", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openaiCompatible.queryParameters", + "label" : "Query Parameters", + "description" : "Map of query parameters to add to the request URL.", + "optional" : true, + "feel" : "required", + "group" : "provider", + "binding" : { + "name" : "provider.openaiCompatible.queryParameters", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openaiCompatible.timeouts.timeout", + "label" : "Timeout", + "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openaiCompatible.timeouts.timeout", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.anthropic.model.model", + "label" : "Model", + "description" : "Specify the model ID. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.model", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "placeholder" : "claude-sonnet-4-6", + "type" : "String" + }, { + "id" : "provider.anthropic.model.parameters.maxTokens", + "label" : "Maximum tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.parameters.maxTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "tooltip" : "The maximum number of tokens per request to generate before stopping.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.anthropic.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. The higher the number, the more randomness will be injected into the response.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.anthropic.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.anthropic.model.parameters.topK", + "label" : "top K", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.parameters.topK", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "tooltip" : "Integer greater than 0. Recommended for advanced use cases only (you usually only need to use temperature).

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.bedrock.model.model", + "label" : "Model", + "description" : "Specify an inference profile ID. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.bedrock.model.model", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "placeholder" : "global.anthropic.claude-sonnet-4-6", + "type" : "String" + }, { + "id" : "provider.bedrock.model.parameters.maxTokens", + "label" : "Maximum tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.bedrock.model.parameters.maxTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "tooltip" : "The maximum number of tokens per request to allow in the generated response.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.bedrock.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.bedrock.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. The higher the number, the more randomness will be injected into the response.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.bedrock.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.bedrock.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.azureOpenAi.model.deploymentName", + "label" : "Model deployment name", + "description" : "Specify the model deployment name. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.azureOpenAi.model.deploymentName", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.azureOpenAi.model.parameters.maxTokens", + "label" : "Maximum tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.azureOpenAi.model.parameters.maxTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + }, + "tooltip" : "The maximum number of tokens per request to generate before stopping.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.azureOpenAi.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.azureOpenAi.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 2. The higher the number, the more randomness will be injected into the response.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.azureOpenAi.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.azureOpenAi.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + }, + "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.googleVertexAi.model.model", + "label" : "Model", + "description" : "Specify the model ID. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.googleVertexAi.model.model", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "google-vertex-ai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.googleVertexAi.model.parameters.maxOutputTokens", + "label" : "Maximum output tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.googleVertexAi.model.parameters.maxOutputTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "google-vertex-ai", + "type" : "simple" + }, + "tooltip" : "Maximum number of tokens that can be generated in the response.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.googleVertexAi.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.googleVertexAi.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "google-vertex-ai", + "type" : "simple" + }, + "tooltip" : "Controls the degree of randomness in token selection.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.googleVertexAi.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.googleVertexAi.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "google-vertex-ai", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.googleVertexAi.model.parameters.topK", + "label" : "top K", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.googleVertexAi.model.parameters.topK", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "google-vertex-ai", + "type" : "simple" + }, + "tooltip" : "Integer greater than 0. Recommended for advanced use cases only (you usually only need to use temperature).

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openai.model.model", + "label" : "Model", + "description" : "Specify the model ID. Details in the documentation.", + "optional" : false, + "value" : "gpt-4o", + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.openai.model.model", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openai.model.parameters.maxCompletionTokens", + "label" : "Maximum completion tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openai.model.parameters.maxCompletionTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "tooltip" : "The maximum number of tokens per request to generate before stopping.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openai.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openai.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 2. The higher the number, the more randomness will be injected into the response.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openai.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openai.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openaiCompatible.model.model", + "label" : "Model", + "description" : "Specify the model ID. Details in the documentation.", + "optional" : false, + "value" : "gpt-4o", + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.openaiCompatible.model.model", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openaiCompatible.model.parameters.maxCompletionTokens", + "label" : "Maximum completion tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openaiCompatible.model.parameters.maxCompletionTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "tooltip" : "The maximum number of tokens per request to generate before stopping.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openaiCompatible.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openaiCompatible.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 2. The higher the number, the more randomness will be injected into the response.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openaiCompatible.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openaiCompatible.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openaiCompatible.model.parameters.customParameters", + "label" : "Custom parameters", + "description" : "Map of additional request parameters to include.", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openaiCompatible.model.parameters.customParameters", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.systemPrompt.prompt", + "label" : "System prompt", + "optional" : false, + "value" : "=\"You are **TaskAgent**, a helpful, generic chat agent that can handle a wide variety of customer requests using your own domain knowledge **and** any tools explicitly provided to you at runtime.\n\nIf tools are provided, you should prefer them instead of guessing an answer. You can call the same tool multiple times by providing different input values. Don't guess any tools which were not explicitly configured. If no tool matches the request, try to generate an answer. If you're not able to find a good answer, return with a message stating why you're not able to.\n\nWrap minimal, inspectable reasoning in *exactly* this XML template:\n\n\n…briefly state the customer’s need and current state…\n…list candidate tools, justify which you will call next and why…\n\n\nReveal **no** additional private reasoning outside these tags.\"", + "constraints" : { + "notEmpty" : true + }, + "feel" : "required", + "group" : "systemPrompt", + "binding" : { + "name" : "data.systemPrompt.prompt", + "type" : "zeebe:input" + }, + "type" : "Text" + }, { + "id" : "data.userPrompt.prompt", + "label" : "User prompt", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "required", + "group" : "userPrompt", + "binding" : { + "name" : "data.userPrompt.prompt", + "type" : "zeebe:input" + }, + "type" : "Text" + }, { + "id" : "data.userPrompt.documents", + "label" : "Documents", + "description" : "Documents to be included in the user prompt.", + "optional" : true, + "feel" : "required", + "group" : "userPrompt", + "binding" : { + "name" : "data.userPrompt.documents", + "type" : "zeebe:input" + }, + "tooltip" : "Referenced documents will be automatically added to the user prompt. See documentation for details and supported file types.", + "type" : "String" + }, { + "id" : "agentContext", + "label" : "Agent context", + "description" : "Initial agent context from previous interactions. Avoid reusing context variables across agents to prevent issues with stale data or tool access.", + "optional" : false, + "feel" : "required", + "group" : "memory", + "binding" : { + "name" : "agentContext", + "type" : "zeebe:input" + }, + "tooltip" : "The agent context variable containing all relevant data for the agent to support the feedback loop between user requests, tool calls and LLM responses. Make sure this variable points to the context variable which is returned from the agent response. See documentation for details.", + "type" : "Text" + }, { + "id" : "data.memory.storage.type", + "label" : "Memory storage type", + "description" : "Specify how to store the conversation memory.", + "value" : "in-process", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.type", + "type" : "zeebe:input" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "In Process (part of agent context)", + "value" : "in-process" + }, { + "name" : "Camunda Document Storage", + "value" : "camunda-document" + }, { + "name" : "AWS AgentCore Memory", + "value" : "aws-agentcore" + }, { + "name" : "Custom Implementation (Hybrid/Self-Managed only)", + "value" : "custom" + } ] + }, { + "id" : "data.memory.storage.timeToLive", + "label" : "Document TTL", + "description" : "How long to retain the conversation document as ISO-8601 duration (example: P14D).", + "optional" : true, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.timeToLive", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "camunda-document", + "type" : "simple" + }, + "tooltip" : "Will use the cluster default TTL (time-to-live) if not specified. Make sure to set this value to a reasonable duration matching your process lifecycle.", + "type" : "String" + }, { + "id" : "data.memory.storage.customProperties", + "label" : "Custom document properties", + "description" : "An optional map of custom properties to be stored with the conversation document.", + "optional" : true, + "feel" : "required", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.customProperties", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "camunda-document", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.region", + "label" : "Region", + "description" : "Specify the AWS region (example: us-east-1)", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.region", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.endpoint", + "label" : "Endpoint", + "description" : "Custom API endpoint for VPC/PrivateLink configurations, AWS GovCloud, or other non-standard deployments.", + "optional" : true, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.authentication.type", + "label" : "Authentication", + "description" : "Specify the AWS authentication strategy for AgentCore Memory access.", + "value" : "credentials", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Credentials", + "value" : "credentials" + }, { + "name" : "Default Credentials Chain (Hybrid/Self-Managed only)", + "value" : "defaultCredentialsChain" + } ] + }, { + "id" : "data.memory.storage.authentication.accessKey", + "label" : "Access key", + "description" : "Provide an IAM access key with permissions for bedrock-agentcore:CreateEvent and bedrock-agentcore:ListEvents", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.authentication.accessKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "data.memory.storage.authentication.type", + "equals" : "credentials", + "type" : "simple" + }, { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "data.memory.storage.authentication.secretKey", + "label" : "Secret key", + "description" : "Provide the secret key for the IAM access key", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.authentication.secretKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "data.memory.storage.authentication.type", + "equals" : "credentials", + "type" : "simple" + }, { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "data.memory.storage.memoryId", + "label" : "Memory ID", + "description" : "The ID of the pre-provisioned AgentCore Memory resource.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.memoryId", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.actorId", + "label" : "Actor ID", + "description" : "Identifier of the actor associated with events (e.g., end-user or agent/user combination).", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.actorId", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.storeType", + "label" : "Implementation type", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.storeType", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "custom", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.parameters", + "label" : "Parameters", + "description" : "Parameters for the custom memory storage implementation.", + "optional" : true, + "feel" : "required", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.parameters", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "custom", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.contextWindowSize", + "label" : "Context window size", + "description" : "Maximum number of recent conversation messages which are passed to the model.", + "optional" : false, + "value" : 20, + "feel" : "static", + "group" : "memory", + "binding" : { + "name" : "data.memory.contextWindowSize", + "type" : "zeebe:input" + }, + "tooltip" : "Use this to limit the number of messages which are sent to the model. The agent will only send the most recent messages up to the configured limit to the LLM. Older messages will be kept in the conversation store, but not sent to the model. See documentation for details.", + "type" : "Number" + }, { + "id" : "data.limits.maxModelCalls", + "label" : "Maximum model calls", + "description" : "Maximum number of calls to the model as a safety limit to prevent infinite loops.", + "optional" : false, + "value" : 10, + "feel" : "static", + "group" : "limits", + "binding" : { + "name" : "data.limits.maxModelCalls", + "type" : "zeebe:input" + }, + "type" : "Number" + }, { + "id" : "data.events.behavior", + "label" : "Event handling behavior", + "description" : "Behavior on completing an event sub-process.", + "optional" : false, + "value" : "WAIT_FOR_TOOL_CALL_RESULTS", + "constraints" : { + "notEmpty" : true + }, + "group" : "events", + "binding" : { + "name" : "data.events.behavior", + "type" : "zeebe:input" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Wait for tool call results", + "value" : "WAIT_FOR_TOOL_CALL_RESULTS" + }, { + "name" : "Cancel tool calls", + "value" : "INTERRUPT_TOOL_CALLS" + } ] + }, { + "id" : "data.response.format.type", + "label" : "Response format", + "description" : "Specify the response format. Support for JSON mode varies by provider.", + "value" : "text", + "group" : "response", + "binding" : { + "name" : "data.response.format.type", + "type" : "zeebe:input" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Text", + "value" : "text" + }, { + "name" : "JSON", + "value" : "json" + } ] + }, { + "id" : "data.response.format.parseJson", + "label" : "Parse text as JSON", + "description" : "Tries to parse the LLM response text as JSON object.", + "optional" : true, + "feel" : "static", + "group" : "response", + "binding" : { + "name" : "data.response.format.parseJson", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.response.format.type", + "equals" : "text", + "type" : "simple" + }, + "tooltip" : "Use this option in combination with models which don't support native JSON mode/structured tool calling (e.g. Anthropic). Make sure to instruct the model to return valid JSON in the system prompt. The parsed JSON will be available as response.responseJson.

    If parsing fails, null will be returned as JSON response, but the text content will still be available as response.responseText.", + "type" : "Boolean" + }, { + "id" : "data.response.format.schema", + "label" : "Response JSON schema", + "description" : "An optional response JSON Schema to instruct the model how to structure the JSON output.", + "optional" : true, + "feel" : "required", + "group" : "response", + "binding" : { + "name" : "data.response.format.schema", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.response.format.type", + "equals" : "json", + "type" : "simple" + }, + "tooltip" : "If supported by the model, the response will be structured according to the provided schema. A parsed version of the response will be available as response.responseJson.", + "type" : "String" + }, { + "id" : "data.response.format.schemaName", + "label" : "Response JSON schema name", + "description" : "An optional name for the response JSON Schema to make the model aware of the expected output.", + "optional" : true, + "value" : "Response", + "feel" : "optional", + "group" : "response", + "binding" : { + "name" : "data.response.format.schemaName", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.response.format.type", + "equals" : "json", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.response.includeAssistantMessage", + "label" : "Include assistant message", + "description" : "Include the full assistant message as part of the result object.", + "optional" : true, + "feel" : "static", + "group" : "response", + "binding" : { + "name" : "data.response.includeAssistantMessage", + "type" : "zeebe:input" + }, + "tooltip" : "In addition to the text content, the assistant message may include multiple additional content blocks and metadata (such as token usage). The message will be available as response.responseMessage.", + "type" : "Boolean" + }, { + "id" : "data.response.includeAgentContext", + "label" : "Include agent context", + "description" : "Include the agent context as part of the result object.", + "optional" : true, + "feel" : "static", + "group" : "response", + "binding" : { + "name" : "data.response.includeAgentContext", + "type" : "zeebe:input" + }, + "tooltip" : "Use this option if you need to re-inject the previous agent context into a future agent execution, for example when modeling a user feedback loop between an agent and a user task.", + "type" : "Boolean" + }, { + "id" : "version", + "label" : "Version", + "description" : "Version of the element template", + "value" : "10", + "group" : "connector", + "binding" : { + "key" : "elementTemplateVersion", + "type" : "zeebe:taskHeader" + }, + "type" : "Hidden" + }, { + "id" : "id", + "label" : "ID", + "description" : "ID of the element template", + "value" : "io.camunda.connectors.agenticai.aiagent.jobworker.v1", + "group" : "connector", + "binding" : { + "key" : "elementTemplateId", + "type" : "zeebe:taskHeader" + }, + "type" : "Hidden" + }, { + "id" : "resultVariable", + "label" : "Result variable", + "description" : "Name of variable to store the response in. Details in the documentation.", + "value" : "agent", + "group" : "output", + "binding" : { + "source" : "=agent", + "type" : "zeebe:output" + }, + "type" : "String" + }, { + "id" : "errorExpression", + "label" : "Error expression", + "description" : "Expression to handle errors. Details in the documentation.", + "feel" : "required", + "group" : "error", + "binding" : { + "key" : "errorExpression", + "type" : "zeebe:taskHeader" + }, + "type" : "Text" + }, { + "id" : "retryCount", + "label" : "Retries", + "description" : "Number of retries", + "value" : "3", + "feel" : "optional", + "group" : "retries", + "binding" : { + "property" : "retries", + "type" : "zeebe:taskDefinition" + }, + "type" : "String" + }, { + "id" : "retryBackoff", + "label" : "Retry backoff", + "description" : "ISO-8601 duration to wait between retries", + "value" : "PT0S", + "group" : "retries", + "binding" : { + "key" : "retryBackoff", + "type" : "zeebe:taskHeader" + }, + "type" : "String" + }, { + "binding" : { + "name" : "agent", + "type" : "zeebe:input" + }, + "type" : "Hidden" + } ], + "icon" : { + "contents" : "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMzIiIGhlaWdodD0iMzIiIHZpZXdCb3g9IjAgMCAzMiAzMiIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPGNpcmNsZSBjeD0iMTYiIGN5PSIxNiIgcj0iMTYiIGZpbGw9IiNBNTZFRkYiLz4KPG1hc2sgaWQ9InBhdGgtMi1vdXRzaWRlLTFfMTg1XzYiIG1hc2tVbml0cz0idXNlclNwYWNlT25Vc2UiIHg9IjQiIHk9IjQiIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgZmlsbD0iYmxhY2siPgo8cmVjdCBmaWxsPSJ3aGl0ZSIgeD0iNCIgeT0iNCIgd2lkdGg9IjI0IiBoZWlnaHQ9IjI0Ii8+CjxwYXRoIGZpbGwtcnVsZT0iZXZlbm9kZCIgY2xpcC1ydWxlPSJldmVub2RkIiBkPSJNMjAuMDEwNSAxMi4wOTg3QzE4LjQ5IDEwLjU4OTQgMTcuMTU5NCA4LjEwODE0IDE2LjE3OTkgNi4wMTEwM0MxNi4xNTIgNi4wMDQ1MSAxNi4xMTc2IDYgMTYuMDc5NCA2QzE2LjA0MTEgNiAxNi4wMDY2IDYuMDA0NTEgMTUuOTc4OCA2LjAxMTA0QzE0Ljk5OTQgOC4xMDgxNCAxMy42Njk3IDEwLjU4ODkgMTIuMTQ4MSAxMi4wOTgxQzEwLjYyNjkgMTMuNjA3MSA4LjEyNTY4IDE0LjkyNjQgNi4wMTE1NyAxNS44OTgxQzYuMDA0NzQgMTUuOTI2MSA2IDE1Ljk2MTEgNiAxNkM2IDE2LjAzODcgNi4wMDQ2OCAxNi4wNzM2IDYuMDExNDQgMTYuMTAxNEM4LjEyNTE5IDE3LjA3MjkgMTAuNjI2MiAxOC4zOTE5IDEyLjE0NzcgMTkuOTAxNkMxMy42Njk3IDIxLjQxMDcgMTQuOTk5NiAyMy44OTIgMTUuOTc5MSAyNS45ODlDMTYuMDA2OCAyNS45OTU2IDE2LjA0MTEgMjYgMTYuMDc5MyAyNkMxNi4xMTc1IDI2IDE2LjE1MTkgMjUuOTk1NCAxNi4xNzk2IDI1Ljk4OUMxNy4xNTkxIDIzLjg5MiAxOC40ODg4IDIxLjQxMSAyMC4wMDk5IDE5LjkwMjFNMjAuMDA5OSAxOS45MDIxQzIxLjUyNTMgMTguMzk4NyAyMy45NDY1IDE3LjA2NjkgMjUuOTkxNSAxNi4wODI0QzI1Ljk5NjUgMTYuMDU5MyAyNiAxNi4wMzEgMjYgMTUuOTk5N0MyNiAxNS45Njg0IDI1Ljk5NjUgMTUuOTQwMyAyNS45OTE1IDE1LjkxNzFDMjMuOTQ3NCAxNC45MzI3IDIxLjUyNTkgMTMuNjAxIDIwLjAxMDUgMTIuMDk4NyIvPgo8L21hc2s+CjxwYXRoIGZpbGwtcnVsZT0iZXZlbm9kZCIgY2xpcC1ydWxlPSJldmVub2RkIiBkPSJNMjAuMDEwNSAxMi4wOTg3QzE4LjQ5IDEwLjU4OTQgMTcuMTU5NCA4LjEwODE0IDE2LjE3OTkgNi4wMTEwM0MxNi4xNTIgNi4wMDQ1MSAxNi4xMTc2IDYgMTYuMDc5NCA2QzE2LjA0MTEgNiAxNi4wMDY2IDYuMDA0NTEgMTUuOTc4OCA2LjAxMTA0QzE0Ljk5OTQgOC4xMDgxNCAxMy42Njk3IDEwLjU4ODkgMTIuMTQ4MSAxMi4wOTgxQzEwLjYyNjkgMTMuNjA3MSA4LjEyNTY4IDE0LjkyNjQgNi4wMTE1NyAxNS44OTgxQzYuMDA0NzQgMTUuOTI2MSA2IDE1Ljk2MTEgNiAxNkM2IDE2LjAzODcgNi4wMDQ2OCAxNi4wNzM2IDYuMDExNDQgMTYuMTAxNEM4LjEyNTE5IDE3LjA3MjkgMTAuNjI2MiAxOC4zOTE5IDEyLjE0NzcgMTkuOTAxNkMxMy42Njk3IDIxLjQxMDcgMTQuOTk5NiAyMy44OTIgMTUuOTc5MSAyNS45ODlDMTYuMDA2OCAyNS45OTU2IDE2LjA0MTEgMjYgMTYuMDc5MyAyNkMxNi4xMTc1IDI2IDE2LjE1MTkgMjUuOTk1NCAxNi4xNzk2IDI1Ljk4OUMxNy4xNTkxIDIzLjg5MiAxOC40ODg4IDIxLjQxMSAyMC4wMDk5IDE5LjkwMjFNMjAuMDA5OSAxOS45MDIxQzIxLjUyNTMgMTguMzk4NyAyMy45NDY1IDE3LjA2NjkgMjUuOTkxNSAxNi4wODI0QzI1Ljk5NjUgMTYuMDU5MyAyNiAxNi4wMzEgMjYgMTUuOTk5N0MyNiAxNS45Njg0IDI1Ljk5NjUgMTUuOTQwMyAyNS45OTE1IDE1LjkxNzFDMjMuOTQ3NCAxNC45MzI3IDIxLjUyNTkgMTMuNjAxIDIwLjAxMDUgMTIuMDk4NyIgZmlsbD0id2hpdGUiLz4KPHBhdGggZmlsbC1ydWxlPSJldmVub2RkIiBjbGlwLXJ1bGU9ImV2ZW5vZGQiIGQ9Ik0yMC4wMTA1IDEyLjA5ODdDMTguNDkgMTAuNTg5NCAxNy4xNTk0IDguMTA4MTQgMTYuMTc5OSA2LjAxMTAzQzE2LjE1MiA2LjAwNDUxIDE2LjExNzYgNiAxNi4wNzk0IDZDMTYuMDQxMSA2IDE2LjAwNjYgNi4wMDQ1MSAxNS45Nzg4IDYuMDExMDRDMTQuOTk5NCA4LjEwODE0IDEzLjY2OTcgMTAuNTg4OSAxMi4xNDgxIDEyLjA5ODFDMTAuNjI2OSAxMy42MDcxIDguMTI1NjggMTQuOTI2NCA2LjAxMTU3IDE1Ljg5ODFDNi4wMDQ3NCAxNS45MjYxIDYgMTUuOTYxMSA2IDE2QzYgMTYuMDM4NyA2LjAwNDY4IDE2LjA3MzYgNi4wMTE0NCAxNi4xMDE0QzguMTI1MTkgMTcuMDcyOSAxMC42MjYyIDE4LjM5MTkgMTIuMTQ3NyAxOS45MDE2QzEzLjY2OTcgMjEuNDEwNyAxNC45OTk2IDIzLjg5MiAxNS45NzkxIDI1Ljk4OUMxNi4wMDY4IDI1Ljk5NTYgMTYuMDQxMSAyNiAxNi4wNzkzIDI2QzE2LjExNzUgMjYgMTYuMTUxOSAyNS45OTU0IDE2LjE3OTYgMjUuOTg5QzE3LjE1OTEgMjMuODkyIDE4LjQ4ODggMjEuNDExIDIwLjAwOTkgMTkuOTAyMU0yMC4wMDk5IDE5LjkwMjFDMjEuNTI1MyAxOC4zOTg3IDIzLjk0NjUgMTcuMDY2OSAyNS45OTE1IDE2LjA4MjRDMjUuOTk2NSAxNi4wNTkzIDI2IDE2LjAzMSAyNiAxNS45OTk3QzI2IDE1Ljk2ODQgMjUuOTk2NSAxNS45NDAzIDI1Ljk5MTUgMTUuOTE3MUMyMy45NDc0IDE0LjkzMjcgMjEuNTI1OSAxMy42MDEgMjAuMDEwNSAxMi4wOTg3IiBzdHJva2U9IiM0OTFEOEIiIHN0cm9rZS13aWR0aD0iNCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIgbWFzaz0idXJsKCNwYXRoLTItb3V0c2lkZS0xXzE4NV82KSIvPgo8L3N2Zz4K" + } +} \ No newline at end of file diff --git a/connectors/agentic-ai/element-templates/versioned/agenticai-aiagent-outbound-connector-10.json b/connectors/agentic-ai/element-templates/versioned/agenticai-aiagent-outbound-connector-10.json new file mode 100644 index 00000000000..6ae10b471a5 --- /dev/null +++ b/connectors/agentic-ai/element-templates/versioned/agenticai-aiagent-outbound-connector-10.json @@ -0,0 +1,1782 @@ +{ + "$schema" : "https://unpkg.com/@camunda/zeebe-element-templates-json-schema/resources/schema.json", + "name" : "AI Agent Task", + "id" : "io.camunda.connectors.agenticai.aiagent.v1", + "description" : "Execute a single AI-powered action with tool calling capabilities", + "keywords" : [ "AI", "AI Agent", "agentic orchestration" ], + "documentationRef" : "https://docs.camunda.io/docs/8.10/components/connectors/out-of-the-box-connectors/agentic-ai-aiagent-task/", + "version" : 10, + "category" : { + "id" : "connectors", + "name" : "Connectors" + }, + "appliesTo" : [ "bpmn:Task" ], + "elementType" : { + "value" : "bpmn:ServiceTask" + }, + "engines" : { + "camunda" : "^8.10" + }, + "groups" : [ { + "id" : "provider", + "label" : "Model provider", + "openByDefault" : false + }, { + "id" : "model", + "label" : "Model", + "openByDefault" : false + }, { + "id" : "systemPrompt", + "label" : "System prompt", + "tooltip" : "A system prompt is a set of foundational instructions given to a model before any user interaction begins. It defines the AI agent’s role, behavior, tone, and communication style, ensuring that responses remain consistent and aligned with the AI agent’s intended purpose. These instructions help shape how the model interprets and responds to user input throughout the conversation.", + "openByDefault" : false + }, { + "id" : "userPrompt", + "label" : "User prompt", + "tooltip" : "A user prompt is the message or question you give to the AI to start or continue a conversation. It tells the AI what you need, whether it's information, help with a task, or just a chat. The AI uses your prompt to understand how to respond.", + "openByDefault" : false + }, { + "id" : "tools", + "label" : "Tools", + "tooltip" : "Tools are optional features the AI Agent can use to perform specific tasks. Configure this if the agent should participate in a tools feedback loop.", + "openByDefault" : false + }, { + "id" : "memory", + "label" : "Memory", + "tooltip" : "Configuration of the Agent's short-term/conversational memory.", + "openByDefault" : false + }, { + "id" : "limits", + "label" : "Limits", + "openByDefault" : false + }, { + "id" : "response", + "label" : "Response", + "tooltip" : "Configuration of the model response format and how to map the model response to the connector result.

    Depending on the selection, the model response will be available as response.responseText or response.responseJson.

    See documentation for details.", + "openByDefault" : false + }, { + "id" : "connector", + "label" : "Connector" + }, { + "id" : "output", + "label" : "Output mapping" + }, { + "id" : "error", + "label" : "Error handling" + }, { + "id" : "retries", + "label" : "Retries" + } ], + "properties" : [ { + "value" : "io.camunda.agenticai:aiagent:1", + "binding" : { + "property" : "type", + "type" : "zeebe:taskDefinition" + }, + "type" : "Hidden" + }, { + "id" : "provider.type", + "label" : "Provider", + "description" : "Specify the LLM provider to use.", + "value" : "anthropic", + "group" : "provider", + "binding" : { + "name" : "provider.type", + "type" : "zeebe:input" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Anthropic", + "value" : "anthropic" + }, { + "name" : "AWS Bedrock", + "value" : "bedrock" + }, { + "name" : "Azure OpenAI", + "value" : "azureOpenAi" + }, { + "name" : "Google Vertex AI", + "value" : "google-vertex-ai" + }, { + "name" : "OpenAI", + "value" : "openai" + }, { + "name" : "OpenAI Compatible", + "value" : "openaiCompatible" + } ] + }, { + "id" : "provider.anthropic.endpoint", + "label" : "Endpoint", + "description" : "Optional custom API endpoint", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.apiKey", + "label" : "Anthropic API key", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.apiKey", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.anthropic.timeouts.timeout", + "label" : "Timeout", + "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.timeouts.timeout", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.bedrock.region", + "label" : "Region", + "description" : "Specify the AWS region (example: eu-west-1)", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.region", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.bedrock.endpoint", + "label" : "Endpoint", + "description" : "Custom API endpoint for VPC/PrivateLink configurations, AWS GovCloud, or other non-standard deployments.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.bedrock.authentication.type", + "label" : "Authentication", + "description" : "Specify the AWS authentication strategy. Learn more at the documentation page", + "value" : "credentials", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Credentials", + "value" : "credentials" + }, { + "name" : "API Key", + "value" : "apiKey" + }, { + "name" : "Default Credentials Chain (Hybrid/Self-Managed only)", + "value" : "defaultCredentialsChain" + } ] + }, { + "id" : "provider.bedrock.authentication.accessKey", + "label" : "Access key", + "description" : "Provide an IAM access key tailored to a user, equipped with the necessary permissions", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.authentication.accessKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.bedrock.authentication.type", + "equals" : "credentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.bedrock.authentication.secretKey", + "label" : "Secret key", + "description" : "Provide the secret key for the IAM access key", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.authentication.secretKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.bedrock.authentication.type", + "equals" : "credentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.bedrock.authentication.apiKey", + "label" : "API Key", + "description" : "Provide an API Key with permissions to interact with your AWS Bedrock Instance", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.authentication.apiKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.bedrock.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.bedrock.timeouts.timeout", + "label" : "Timeout", + "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.timeouts.timeout", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.azureOpenAi.endpoint", + "label" : "Endpoint", + "description" : "Specify Azure OpenAI endpoint. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.azureOpenAi.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.azureOpenAi.authentication.type", + "label" : "Authentication", + "description" : "Specify the Azure OpenAI authentication strategy.", + "value" : "apiKey", + "group" : "provider", + "binding" : { + "name" : "provider.azureOpenAi.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "API key", + "value" : "apiKey" + }, { + "name" : "Client credentials", + "value" : "clientCredentials" + } ] + }, { + "id" : "provider.azureOpenAi.authentication.apiKey", + "label" : "API key", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.azureOpenAi.authentication.apiKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.azureOpenAi.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.azureOpenAi.authentication.clientId", + "label" : "Client ID", + "description" : "ID of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.azureOpenAi.authentication.clientId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.azureOpenAi.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.azureOpenAi.authentication.clientSecret", + "label" : "Client secret", + "description" : "Secret of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.azureOpenAi.authentication.clientSecret", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.azureOpenAi.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.azureOpenAi.authentication.tenantId", + "label" : "Tenant ID", + "description" : "ID of a Microsoft Entra tenant. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.azureOpenAi.authentication.tenantId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.azureOpenAi.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.azureOpenAi.authentication.authorityHost", + "label" : "Authority host", + "description" : "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.azureOpenAi.authentication.authorityHost", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.azureOpenAi.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.azureOpenAi.timeouts.timeout", + "label" : "Timeout", + "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.azureOpenAi.timeouts.timeout", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.googleVertexAi.projectId", + "label" : "Project ID", + "description" : "Specify Google Cloud project ID", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.googleVertexAi.projectId", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "google-vertex-ai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.googleVertexAi.region", + "label" : "Region", + "description" : "Specify the region where AI inference should take place", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.googleVertexAi.region", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "google-vertex-ai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.googleVertexAi.authentication.type", + "label" : "Authentication", + "description" : "Specify the Google Vertex AI authentication strategy.", + "value" : "serviceAccountCredentials", + "group" : "provider", + "binding" : { + "name" : "provider.googleVertexAi.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "google-vertex-ai", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Service account credentials", + "value" : "serviceAccountCredentials" + }, { + "name" : "Application default credentials (Hybrid/Self-Managed only)", + "value" : "applicationDefaultCredentials" + } ] + }, { + "id" : "provider.googleVertexAi.authentication.jsonKey", + "label" : "JSON key of the service account", + "description" : "This is the key of the service account in JSON format.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.googleVertexAi.authentication.jsonKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.googleVertexAi.authentication.type", + "equals" : "serviceAccountCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "google-vertex-ai", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.apiKey", + "label" : "OpenAI API key", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.apiKey", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.organizationId", + "label" : "Organization ID", + "description" : "For members of multiple organizations. Details in the documentation.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.organizationId", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.projectId", + "label" : "Project ID", + "description" : "For accounts with multiple projects. Details in the documentation.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.projectId", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openai.timeouts.timeout", + "label" : "Timeout", + "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.timeouts.timeout", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openai.apiFamily", + "label" : "API", + "description" : "Which OpenAI API family to use. Chat Completions is the default for backward compatibility; Responses is the newer endpoint with structured input/output items.", + "optional" : true, + "value" : "completions", + "group" : "provider", + "binding" : { + "name" : "provider.openai.apiFamily", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Chat Completions (default)", + "value" : "completions" + }, { + "name" : "Responses", + "value" : "responses" + } ] + }, { + "id" : "provider.openaiCompatible.endpoint", + "label" : "API endpoint", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openaiCompatible.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "tooltip" : "Specify an endpoint to use the connector with an OpenAI compatible API. ", + "type" : "String" + }, { + "id" : "provider.openaiCompatible.authentication.apiKey", + "label" : "API key", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openaiCompatible.authentication.apiKey", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "tooltip" : "Leave blank if using HTTP headers for authentication.
    If an Authorization header is specified in the headers, then the API key is ignored.", + "type" : "String" + }, { + "id" : "provider.openaiCompatible.headers", + "label" : "Headers", + "description" : "Map of HTTP headers to add to the request.", + "optional" : true, + "feel" : "required", + "group" : "provider", + "binding" : { + "name" : "provider.openaiCompatible.headers", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openaiCompatible.queryParameters", + "label" : "Query Parameters", + "description" : "Map of query parameters to add to the request URL.", + "optional" : true, + "feel" : "required", + "group" : "provider", + "binding" : { + "name" : "provider.openaiCompatible.queryParameters", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openaiCompatible.timeouts.timeout", + "label" : "Timeout", + "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openaiCompatible.timeouts.timeout", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.anthropic.model.model", + "label" : "Model", + "description" : "Specify the model ID. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.model", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "placeholder" : "claude-sonnet-4-6", + "type" : "String" + }, { + "id" : "provider.anthropic.model.parameters.maxTokens", + "label" : "Maximum tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.parameters.maxTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "tooltip" : "The maximum number of tokens per request to generate before stopping.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.anthropic.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. The higher the number, the more randomness will be injected into the response.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.anthropic.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.anthropic.model.parameters.topK", + "label" : "top K", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.parameters.topK", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "tooltip" : "Integer greater than 0. Recommended for advanced use cases only (you usually only need to use temperature).

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.bedrock.model.model", + "label" : "Model", + "description" : "Specify an inference profile ID. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.bedrock.model.model", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "placeholder" : "global.anthropic.claude-sonnet-4-6", + "type" : "String" + }, { + "id" : "provider.bedrock.model.parameters.maxTokens", + "label" : "Maximum tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.bedrock.model.parameters.maxTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "tooltip" : "The maximum number of tokens per request to allow in the generated response.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.bedrock.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.bedrock.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. The higher the number, the more randomness will be injected into the response.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.bedrock.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.bedrock.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.azureOpenAi.model.deploymentName", + "label" : "Model deployment name", + "description" : "Specify the model deployment name. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.azureOpenAi.model.deploymentName", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.azureOpenAi.model.parameters.maxTokens", + "label" : "Maximum tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.azureOpenAi.model.parameters.maxTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + }, + "tooltip" : "The maximum number of tokens per request to generate before stopping.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.azureOpenAi.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.azureOpenAi.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 2. The higher the number, the more randomness will be injected into the response.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.azureOpenAi.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.azureOpenAi.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "azureOpenAi", + "type" : "simple" + }, + "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.googleVertexAi.model.model", + "label" : "Model", + "description" : "Specify the model ID. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.googleVertexAi.model.model", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "google-vertex-ai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.googleVertexAi.model.parameters.maxOutputTokens", + "label" : "Maximum output tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.googleVertexAi.model.parameters.maxOutputTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "google-vertex-ai", + "type" : "simple" + }, + "tooltip" : "Maximum number of tokens that can be generated in the response.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.googleVertexAi.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.googleVertexAi.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "google-vertex-ai", + "type" : "simple" + }, + "tooltip" : "Controls the degree of randomness in token selection.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.googleVertexAi.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.googleVertexAi.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "google-vertex-ai", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.googleVertexAi.model.parameters.topK", + "label" : "top K", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.googleVertexAi.model.parameters.topK", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "google-vertex-ai", + "type" : "simple" + }, + "tooltip" : "Integer greater than 0. Recommended for advanced use cases only (you usually only need to use temperature).

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openai.model.model", + "label" : "Model", + "description" : "Specify the model ID. Details in the documentation.", + "optional" : false, + "value" : "gpt-4o", + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.openai.model.model", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openai.model.parameters.maxCompletionTokens", + "label" : "Maximum completion tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openai.model.parameters.maxCompletionTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "tooltip" : "The maximum number of tokens per request to generate before stopping.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openai.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openai.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 2. The higher the number, the more randomness will be injected into the response.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openai.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openai.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openaiCompatible.model.model", + "label" : "Model", + "description" : "Specify the model ID. Details in the documentation.", + "optional" : false, + "value" : "gpt-4o", + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.openaiCompatible.model.model", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openaiCompatible.model.parameters.maxCompletionTokens", + "label" : "Maximum completion tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openaiCompatible.model.parameters.maxCompletionTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "tooltip" : "The maximum number of tokens per request to generate before stopping.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openaiCompatible.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openaiCompatible.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 2. The higher the number, the more randomness will be injected into the response.

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openaiCompatible.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openaiCompatible.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

    Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openaiCompatible.model.parameters.customParameters", + "label" : "Custom parameters", + "description" : "Map of additional request parameters to include.", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openaiCompatible.model.parameters.customParameters", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openaiCompatible", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.systemPrompt.prompt", + "label" : "System prompt", + "optional" : false, + "value" : "=\"You are **TaskAgent**, a helpful, generic chat agent that can handle a wide variety of customer requests using your own domain knowledge **and** any tools explicitly provided to you at runtime.\n\nIf tools are provided, you should prefer them instead of guessing an answer. You can call the same tool multiple times by providing different input values. Don't guess any tools which were not explicitly configured. If no tool matches the request, try to generate an answer. If you're not able to find a good answer, return with a message stating why you're not able to.\n\nWrap minimal, inspectable reasoning in *exactly* this XML template:\n\n\n…briefly state the customer’s need and current state…\n…list candidate tools, justify which you will call next and why…\n\n\nReveal **no** additional private reasoning outside these tags.\"", + "constraints" : { + "notEmpty" : true + }, + "feel" : "required", + "group" : "systemPrompt", + "binding" : { + "name" : "data.systemPrompt.prompt", + "type" : "zeebe:input" + }, + "type" : "Text" + }, { + "id" : "data.userPrompt.prompt", + "label" : "User prompt", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "required", + "group" : "userPrompt", + "binding" : { + "name" : "data.userPrompt.prompt", + "type" : "zeebe:input" + }, + "type" : "Text" + }, { + "id" : "data.userPrompt.documents", + "label" : "Documents", + "description" : "Documents to be included in the user prompt.", + "optional" : true, + "feel" : "required", + "group" : "userPrompt", + "binding" : { + "name" : "data.userPrompt.documents", + "type" : "zeebe:input" + }, + "tooltip" : "Referenced documents will be automatically added to the user prompt. See documentation for details and supported file types.", + "type" : "String" + }, { + "id" : "data.tools.containerElementId", + "label" : "Ad-hoc sub-process ID", + "description" : "ID of the sub-process that contains the tools the AI agent can use.", + "optional" : true, + "feel" : "optional", + "group" : "tools", + "binding" : { + "name" : "data.tools.containerElementId", + "type" : "zeebe:input" + }, + "tooltip" : "Add an ad-hoc sub-process ID to attach the AI agent to the tools. Ensure your process includes a tools feedback loop routing into the ad-hoc sub-process and back to the AI agent connector. See documentation for details.", + "type" : "String" + }, { + "id" : "data.tools.toolCallResults", + "label" : "Tool call results", + "description" : "Tool call results as returned by the sub-process.", + "optional" : true, + "feel" : "required", + "group" : "tools", + "binding" : { + "name" : "data.tools.toolCallResults", + "type" : "zeebe:input" + }, + "tooltip" : "This defines where to handle tool call results returned by the ad-hoc sub-process. Model this as part of your process and route it into the tools feedback loop. See documentation for details.", + "type" : "Text" + }, { + "id" : "data.agentContext", + "label" : "Agent context", + "description" : "Avoid reusing context variables across agents to prevent issues with stale data or tool access.", + "optional" : false, + "value" : "=agent.context", + "constraints" : { + "notEmpty" : true + }, + "feel" : "required", + "group" : "memory", + "binding" : { + "name" : "data.context", + "type" : "zeebe:input" + }, + "tooltip" : "The agent context variable containing all relevant data for the agent to support the feedback loop between user requests, tool calls and LLM responses. Make sure this variable points to the context variable which is returned from the agent response. See documentation for details.", + "type" : "Text" + }, { + "id" : "data.memory.storage.type", + "label" : "Memory storage type", + "description" : "Specify how to store the conversation memory.", + "value" : "in-process", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.type", + "type" : "zeebe:input" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "In Process (part of agent context)", + "value" : "in-process" + }, { + "name" : "Camunda Document Storage", + "value" : "camunda-document" + }, { + "name" : "AWS AgentCore Memory", + "value" : "aws-agentcore" + }, { + "name" : "Custom Implementation (Hybrid/Self-Managed only)", + "value" : "custom" + } ] + }, { + "id" : "data.memory.storage.timeToLive", + "label" : "Document TTL", + "description" : "How long to retain the conversation document as ISO-8601 duration (example: P14D).", + "optional" : true, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.timeToLive", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "camunda-document", + "type" : "simple" + }, + "tooltip" : "Will use the cluster default TTL (time-to-live) if not specified. Make sure to set this value to a reasonable duration matching your process lifecycle.", + "type" : "String" + }, { + "id" : "data.memory.storage.customProperties", + "label" : "Custom document properties", + "description" : "An optional map of custom properties to be stored with the conversation document.", + "optional" : true, + "feel" : "required", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.customProperties", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "camunda-document", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.region", + "label" : "Region", + "description" : "Specify the AWS region (example: us-east-1)", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.region", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.endpoint", + "label" : "Endpoint", + "description" : "Custom API endpoint for VPC/PrivateLink configurations, AWS GovCloud, or other non-standard deployments.", + "optional" : true, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.authentication.type", + "label" : "Authentication", + "description" : "Specify the AWS authentication strategy for AgentCore Memory access.", + "value" : "credentials", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Credentials", + "value" : "credentials" + }, { + "name" : "Default Credentials Chain (Hybrid/Self-Managed only)", + "value" : "defaultCredentialsChain" + } ] + }, { + "id" : "data.memory.storage.authentication.accessKey", + "label" : "Access key", + "description" : "Provide an IAM access key with permissions for bedrock-agentcore:CreateEvent and bedrock-agentcore:ListEvents", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.authentication.accessKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "data.memory.storage.authentication.type", + "equals" : "credentials", + "type" : "simple" + }, { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "data.memory.storage.authentication.secretKey", + "label" : "Secret key", + "description" : "Provide the secret key for the IAM access key", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.authentication.secretKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "data.memory.storage.authentication.type", + "equals" : "credentials", + "type" : "simple" + }, { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "data.memory.storage.memoryId", + "label" : "Memory ID", + "description" : "The ID of the pre-provisioned AgentCore Memory resource.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.memoryId", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.actorId", + "label" : "Actor ID", + "description" : "Identifier of the actor associated with events (e.g., end-user or agent/user combination).", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.actorId", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.storeType", + "label" : "Implementation type", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.storeType", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "custom", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.parameters", + "label" : "Parameters", + "description" : "Parameters for the custom memory storage implementation.", + "optional" : true, + "feel" : "required", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.parameters", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "custom", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.contextWindowSize", + "label" : "Context window size", + "description" : "Maximum number of recent conversation messages which are passed to the model.", + "optional" : false, + "value" : 20, + "feel" : "static", + "group" : "memory", + "binding" : { + "name" : "data.memory.contextWindowSize", + "type" : "zeebe:input" + }, + "tooltip" : "Use this to limit the number of messages which are sent to the model. The agent will only send the most recent messages up to the configured limit to the LLM. Older messages will be kept in the conversation store, but not sent to the model. See documentation for details.", + "type" : "Number" + }, { + "id" : "data.limits.maxModelCalls", + "label" : "Maximum model calls", + "description" : "Maximum number of calls to the model as a safety limit to prevent infinite loops.", + "optional" : false, + "value" : 10, + "feel" : "static", + "group" : "limits", + "binding" : { + "name" : "data.limits.maxModelCalls", + "type" : "zeebe:input" + }, + "type" : "Number" + }, { + "id" : "data.response.format.type", + "label" : "Response format", + "description" : "Specify the response format. Support for JSON mode varies by provider.", + "value" : "text", + "group" : "response", + "binding" : { + "name" : "data.response.format.type", + "type" : "zeebe:input" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Text", + "value" : "text" + }, { + "name" : "JSON", + "value" : "json" + } ] + }, { + "id" : "data.response.format.parseJson", + "label" : "Parse text as JSON", + "description" : "Tries to parse the LLM response text as JSON object.", + "optional" : true, + "feel" : "static", + "group" : "response", + "binding" : { + "name" : "data.response.format.parseJson", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.response.format.type", + "equals" : "text", + "type" : "simple" + }, + "tooltip" : "Use this option in combination with models which don't support native JSON mode/structured tool calling (e.g. Anthropic). Make sure to instruct the model to return valid JSON in the system prompt. The parsed JSON will be available as response.responseJson.

    If parsing fails, null will be returned as JSON response, but the text content will still be available as response.responseText.", + "type" : "Boolean" + }, { + "id" : "data.response.format.schema", + "label" : "Response JSON schema", + "description" : "An optional response JSON Schema to instruct the model how to structure the JSON output.", + "optional" : true, + "feel" : "required", + "group" : "response", + "binding" : { + "name" : "data.response.format.schema", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.response.format.type", + "equals" : "json", + "type" : "simple" + }, + "tooltip" : "If supported by the model, the response will be structured according to the provided schema. A parsed version of the response will be available as response.responseJson.", + "type" : "String" + }, { + "id" : "data.response.format.schemaName", + "label" : "Response JSON schema name", + "description" : "An optional name for the response JSON Schema to make the model aware of the expected output.", + "optional" : true, + "value" : "Response", + "feel" : "optional", + "group" : "response", + "binding" : { + "name" : "data.response.format.schemaName", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.response.format.type", + "equals" : "json", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.response.includeAssistantMessage", + "label" : "Include assistant message", + "description" : "Include the full assistant message as part of the result object.", + "optional" : true, + "feel" : "static", + "group" : "response", + "binding" : { + "name" : "data.response.includeAssistantMessage", + "type" : "zeebe:input" + }, + "tooltip" : "In addition to the text content, the assistant message may include multiple additional content blocks and metadata (such as token usage). The message will be available as response.responseMessage.", + "type" : "Boolean" + }, { + "id" : "version", + "label" : "Version", + "description" : "Version of the element template", + "value" : "10", + "group" : "connector", + "binding" : { + "key" : "elementTemplateVersion", + "type" : "zeebe:taskHeader" + }, + "type" : "Hidden" + }, { + "id" : "id", + "label" : "ID", + "description" : "ID of the element template", + "value" : "io.camunda.connectors.agenticai.aiagent.v1", + "group" : "connector", + "binding" : { + "key" : "elementTemplateId", + "type" : "zeebe:taskHeader" + }, + "type" : "Hidden" + }, { + "id" : "resultVariable", + "label" : "Result variable", + "description" : "Name of variable to store the response in. Details in the documentation.", + "value" : "agent", + "group" : "output", + "binding" : { + "key" : "resultVariable", + "type" : "zeebe:taskHeader" + }, + "type" : "String" + }, { + "id" : "resultExpression", + "label" : "Result expression", + "description" : "Expression to map the response into process variables. Details in the documentation.", + "feel" : "required", + "group" : "output", + "binding" : { + "key" : "resultExpression", + "type" : "zeebe:taskHeader" + }, + "type" : "Text" + }, { + "id" : "errorExpression", + "label" : "Error expression", + "description" : "Expression to handle errors. Details in the documentation.", + "feel" : "required", + "group" : "error", + "binding" : { + "key" : "errorExpression", + "type" : "zeebe:taskHeader" + }, + "type" : "Text" + }, { + "id" : "retryCount", + "label" : "Retries", + "description" : "Number of retries", + "value" : "3", + "feel" : "optional", + "group" : "retries", + "binding" : { + "property" : "retries", + "type" : "zeebe:taskDefinition" + }, + "type" : "String" + }, { + "id" : "retryBackoff", + "label" : "Retry backoff", + "description" : "ISO-8601 duration to wait between retries", + "value" : "PT0S", + "group" : "retries", + "binding" : { + "key" : "retryBackoff", + "type" : "zeebe:taskHeader" + }, + "type" : "String" + } ], + "icon" : { + "contents" : "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMzIiIGhlaWdodD0iMzIiIHZpZXdCb3g9IjAgMCAzMiAzMiIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPGNpcmNsZSBjeD0iMTYiIGN5PSIxNiIgcj0iMTYiIGZpbGw9IiNBNTZFRkYiLz4KPG1hc2sgaWQ9InBhdGgtMi1vdXRzaWRlLTFfMTg1XzYiIG1hc2tVbml0cz0idXNlclNwYWNlT25Vc2UiIHg9IjQiIHk9IjQiIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgZmlsbD0iYmxhY2siPgo8cmVjdCBmaWxsPSJ3aGl0ZSIgeD0iNCIgeT0iNCIgd2lkdGg9IjI0IiBoZWlnaHQ9IjI0Ii8+CjxwYXRoIGZpbGwtcnVsZT0iZXZlbm9kZCIgY2xpcC1ydWxlPSJldmVub2RkIiBkPSJNMjAuMDEwNSAxMi4wOTg3QzE4LjQ5IDEwLjU4OTQgMTcuMTU5NCA4LjEwODE0IDE2LjE3OTkgNi4wMTEwM0MxNi4xNTIgNi4wMDQ1MSAxNi4xMTc2IDYgMTYuMDc5NCA2QzE2LjA0MTEgNiAxNi4wMDY2IDYuMDA0NTEgMTUuOTc4OCA2LjAxMTA0QzE0Ljk5OTQgOC4xMDgxNCAxMy42Njk3IDEwLjU4ODkgMTIuMTQ4MSAxMi4wOTgxQzEwLjYyNjkgMTMuNjA3MSA4LjEyNTY4IDE0LjkyNjQgNi4wMTE1NyAxNS44OTgxQzYuMDA0NzQgMTUuOTI2MSA2IDE1Ljk2MTEgNiAxNkM2IDE2LjAzODcgNi4wMDQ2OCAxNi4wNzM2IDYuMDExNDQgMTYuMTAxNEM4LjEyNTE5IDE3LjA3MjkgMTAuNjI2MiAxOC4zOTE5IDEyLjE0NzcgMTkuOTAxNkMxMy42Njk3IDIxLjQxMDcgMTQuOTk5NiAyMy44OTIgMTUuOTc5MSAyNS45ODlDMTYuMDA2OCAyNS45OTU2IDE2LjA0MTEgMjYgMTYuMDc5MyAyNkMxNi4xMTc1IDI2IDE2LjE1MTkgMjUuOTk1NCAxNi4xNzk2IDI1Ljk4OUMxNy4xNTkxIDIzLjg5MiAxOC40ODg4IDIxLjQxMSAyMC4wMDk5IDE5LjkwMjFNMjAuMDA5OSAxOS45MDIxQzIxLjUyNTMgMTguMzk4NyAyMy45NDY1IDE3LjA2NjkgMjUuOTkxNSAxNi4wODI0QzI1Ljk5NjUgMTYuMDU5MyAyNiAxNi4wMzEgMjYgMTUuOTk5N0MyNiAxNS45Njg0IDI1Ljk5NjUgMTUuOTQwMyAyNS45OTE1IDE1LjkxNzFDMjMuOTQ3NCAxNC45MzI3IDIxLjUyNTkgMTMuNjAxIDIwLjAxMDUgMTIuMDk4NyIvPgo8L21hc2s+CjxwYXRoIGZpbGwtcnVsZT0iZXZlbm9kZCIgY2xpcC1ydWxlPSJldmVub2RkIiBkPSJNMjAuMDEwNSAxMi4wOTg3QzE4LjQ5IDEwLjU4OTQgMTcuMTU5NCA4LjEwODE0IDE2LjE3OTkgNi4wMTEwM0MxNi4xNTIgNi4wMDQ1MSAxNi4xMTc2IDYgMTYuMDc5NCA2QzE2LjA0MTEgNiAxNi4wMDY2IDYuMDA0NTEgMTUuOTc4OCA2LjAxMTA0QzE0Ljk5OTQgOC4xMDgxNCAxMy42Njk3IDEwLjU4ODkgMTIuMTQ4MSAxMi4wOTgxQzEwLjYyNjkgMTMuNjA3MSA4LjEyNTY4IDE0LjkyNjQgNi4wMTE1NyAxNS44OTgxQzYuMDA0NzQgMTUuOTI2MSA2IDE1Ljk2MTEgNiAxNkM2IDE2LjAzODcgNi4wMDQ2OCAxNi4wNzM2IDYuMDExNDQgMTYuMTAxNEM4LjEyNTE5IDE3LjA3MjkgMTAuNjI2MiAxOC4zOTE5IDEyLjE0NzcgMTkuOTAxNkMxMy42Njk3IDIxLjQxMDcgMTQuOTk5NiAyMy44OTIgMTUuOTc5MSAyNS45ODlDMTYuMDA2OCAyNS45OTU2IDE2LjA0MTEgMjYgMTYuMDc5MyAyNkMxNi4xMTc1IDI2IDE2LjE1MTkgMjUuOTk1NCAxNi4xNzk2IDI1Ljk4OUMxNy4xNTkxIDIzLjg5MiAxOC40ODg4IDIxLjQxMSAyMC4wMDk5IDE5LjkwMjFNMjAuMDA5OSAxOS45MDIxQzIxLjUyNTMgMTguMzk4NyAyMy45NDY1IDE3LjA2NjkgMjUuOTkxNSAxNi4wODI0QzI1Ljk5NjUgMTYuMDU5MyAyNiAxNi4wMzEgMjYgMTUuOTk5N0MyNiAxNS45Njg0IDI1Ljk5NjUgMTUuOTQwMyAyNS45OTE1IDE1LjkxNzFDMjMuOTQ3NCAxNC45MzI3IDIxLjUyNTkgMTMuNjAxIDIwLjAxMDUgMTIuMDk4NyIgZmlsbD0id2hpdGUiLz4KPHBhdGggZmlsbC1ydWxlPSJldmVub2RkIiBjbGlwLXJ1bGU9ImV2ZW5vZGQiIGQ9Ik0yMC4wMTA1IDEyLjA5ODdDMTguNDkgMTAuNTg5NCAxNy4xNTk0IDguMTA4MTQgMTYuMTc5OSA2LjAxMTAzQzE2LjE1MiA2LjAwNDUxIDE2LjExNzYgNiAxNi4wNzk0IDZDMTYuMDQxMSA2IDE2LjAwNjYgNi4wMDQ1MSAxNS45Nzg4IDYuMDExMDRDMTQuOTk5NCA4LjEwODE0IDEzLjY2OTcgMTAuNTg4OSAxMi4xNDgxIDEyLjA5ODFDMTAuNjI2OSAxMy42MDcxIDguMTI1NjggMTQuOTI2NCA2LjAxMTU3IDE1Ljg5ODFDNi4wMDQ3NCAxNS45MjYxIDYgMTUuOTYxMSA2IDE2QzYgMTYuMDM4NyA2LjAwNDY4IDE2LjA3MzYgNi4wMTE0NCAxNi4xMDE0QzguMTI1MTkgMTcuMDcyOSAxMC42MjYyIDE4LjM5MTkgMTIuMTQ3NyAxOS45MDE2QzEzLjY2OTcgMjEuNDEwNyAxNC45OTk2IDIzLjg5MiAxNS45NzkxIDI1Ljk4OUMxNi4wMDY4IDI1Ljk5NTYgMTYuMDQxMSAyNiAxNi4wNzkzIDI2QzE2LjExNzUgMjYgMTYuMTUxOSAyNS45OTU0IDE2LjE3OTYgMjUuOTg5QzE3LjE1OTEgMjMuODkyIDE4LjQ4ODggMjEuNDExIDIwLjAwOTkgMTkuOTAyMU0yMC4wMDk5IDE5LjkwMjFDMjEuNTI1MyAxOC4zOTg3IDIzLjk0NjUgMTcuMDY2OSAyNS45OTE1IDE2LjA4MjRDMjUuOTk2NSAxNi4wNTkzIDI2IDE2LjAzMSAyNiAxNS45OTk3QzI2IDE1Ljk2ODQgMjUuOTk2NSAxNS45NDAzIDI1Ljk5MTUgMTUuOTE3MUMyMy45NDc0IDE0LjkzMjcgMjEuNTI1OSAxMy42MDEgMjAuMDEwNSAxMi4wOTg3IiBzdHJva2U9IiM0OTFEOEIiIHN0cm9rZS13aWR0aD0iNCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIgbWFzaz0idXJsKCNwYXRoLTItb3V0c2lkZS0xXzE4NV82KSIvPgo8L3N2Zz4K" + } +} \ No newline at end of file diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/AiAgentFunction.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/AiAgentFunction.java index ecc263489d3..93f6e3df72f 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/AiAgentFunction.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/AiAgentFunction.java @@ -38,7 +38,7 @@ documentationRef = "https://docs.camunda.io/docs/8.10/components/connectors/out-of-the-box-connectors/agentic-ai-aiagent-task/", engineVersion = "^8.10", - version = 10, + version = 11, inputDataClass = OutboundConnectorAgentRequest.class, outputDataClass = AgentResponse.class, defaultResultVariable = "agent", diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java index 570432096a9..d8884abdc85 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java @@ -12,6 +12,7 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.ApiFamily; import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiConnection; import java.time.Duration; import java.util.Optional; @@ -29,7 +30,8 @@ */ public class OpenAiChatModelApiFactory implements ChatModelApiFactory { - public static final String API_FAMILY = "openai-completions"; + public static final String API_FAMILY_COMPLETIONS = "openai-completions"; + public static final String API_FAMILY_RESPONSES = "openai-responses"; private final ObjectMapper objectMapper; @Nullable private final Duration defaultTimeout; @@ -46,7 +48,8 @@ public String providerType() { @Override public String apiFamily() { - return API_FAMILY; + // Default — actual family depends on the per-call config, so this is informational only. + return API_FAMILY_COMPLETIONS; } @Override @@ -59,15 +62,18 @@ public ChatModelApi create(OpenAiProviderConfiguration configuration) { final var connection = configuration.openai(); final var client = buildClient(connection); final var parameters = connection.model().parameters(); - return new OpenAiChatCompletionsChatModelApi( - client, - connection.model().model(), - objectMapper, + final var maxTokens = parameters != null && parameters.maxCompletionTokens() != null ? parameters.maxCompletionTokens().longValue() - : null, - parameters != null ? parameters.temperature() : null, - parameters != null ? parameters.topP() : null); + : null; + final var temperature = parameters != null ? parameters.temperature() : null; + final var topP = parameters != null ? parameters.topP() : null; + + return connection.apiFamily() == ApiFamily.RESPONSES + ? new OpenAiResponsesChatModelApi( + client, connection.model().model(), objectMapper, maxTokens, temperature, topP) + : new OpenAiChatCompletionsChatModelApi( + client, connection.model().model(), objectMapper, maxTokens, temperature, topP); } private OpenAIClient buildClient(OpenAiConnection connection) { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApi.java new file mode 100644 index 00000000000..30756b110f9 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApi.java @@ -0,0 +1,378 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.openai; + +import static io.camunda.connector.agenticai.aiagent.agent.AgentErrorCodes.ERROR_CODE_FAILED_MODEL_CALL; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.client.OpenAIClient; +import com.openai.models.responses.EasyInputMessage; +import com.openai.models.responses.Response; +import com.openai.models.responses.ResponseCreateParams; +import com.openai.models.responses.ResponseFunctionToolCall; +import com.openai.models.responses.ResponseInputItem; +import com.openai.models.responses.ResponseOutputItem; +import com.openai.models.responses.ResponseOutputMessage; +import com.openai.models.responses.ResponseUsage; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatOptions; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; +import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; +import io.camunda.connector.agenticai.model.message.AssistantMessage; +import io.camunda.connector.agenticai.model.message.StopReason; +import io.camunda.connector.agenticai.model.message.SystemMessage; +import io.camunda.connector.agenticai.model.message.ToolCallResultMessage; +import io.camunda.connector.agenticai.model.message.UserMessage; +import io.camunda.connector.agenticai.model.message.content.Content; +import io.camunda.connector.agenticai.model.message.content.ObjectContent; +import io.camunda.connector.agenticai.model.message.content.TextContent; +import io.camunda.connector.agenticai.model.tool.ToolCall; +import io.camunda.connector.agenticai.model.tool.ToolCallResult; +import io.camunda.connector.api.error.ConnectorException; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import org.apache.commons.lang3.StringUtils; +import org.springframework.lang.Nullable; + +/** + * Native {@link ChatModelApi} for the OpenAI Responses endpoint, driving the {@code openai-java} + * SDK's blocking {@code responses().create(...)} call. + * + *

    Phase D scope (text-only): user / assistant / tool-result content is restricted to text. + * Encrypted reasoning round-tripping, prompt caching ({@code prompt_cache_key}), and multimodal + * input items are deferred to Phase E. Streaming is not yet wired in either. + * + *

    Used by the {@code openai} discriminator when {@code apiFamily = RESPONSES}. The factory still + * builds the same OkHttp {@link OpenAIClient}; the choice of impl class is the only thing that + * changes between the two API families. + */ +public class OpenAiResponsesChatModelApi implements ChatModelApi { + + private static final ModelCapabilities CAPABILITIES = + new ModelCapabilities( + List.of(Modality.TEXT), + List.of(Modality.TEXT), + List.of(Modality.TEXT), + false, + false, + false, + true, + null, + null); + + private static final TypeReference> MAP_TYPE_REF = new TypeReference<>() {}; + + private final OpenAIClient client; + private final String model; + private final ObjectMapper objectMapper; + @Nullable private final Long configuredMaxOutputTokens; + @Nullable private final Double temperature; + @Nullable private final Double topP; + + public OpenAiResponsesChatModelApi( + OpenAIClient client, + String model, + ObjectMapper objectMapper, + @Nullable Long configuredMaxOutputTokens, + @Nullable Double temperature, + @Nullable Double topP) { + this.client = Objects.requireNonNull(client, "client"); + this.model = Objects.requireNonNull(model, "model"); + this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper"); + this.configuredMaxOutputTokens = configuredMaxOutputTokens; + this.temperature = temperature; + this.topP = topP; + } + + @Override + public ModelCapabilities capabilities() { + return CAPABILITIES; + } + + @Override + public CompletableFuture complete( + ChatRequest request, ChatOptions options, ChatStreamListener listener) { + try { + final var params = buildParams(request, options); + final var response = client.responses().create(params); + return CompletableFuture.completedFuture(new ChatResponse(toAssistantMessage(response))); + } catch (RuntimeException e) { + return CompletableFuture.failedFuture(wrapModelCallFailure(e)); + } + } + + private ResponseCreateParams buildParams(ChatRequest request, ChatOptions options) { + final var builder = ResponseCreateParams.builder().model(model); + + final var maxTokens = resolveMaxOutputTokens(options); + if (maxTokens != null) { + builder.maxOutputTokens(maxTokens); + } + Optional.ofNullable(temperature).ifPresent(builder::temperature); + Optional.ofNullable(topP).ifPresent(builder::topP); + + final var inputItems = new ArrayList(); + final var messages = request.messages(); + if (messages != null) { + for (var message : messages) { + switch (message) { + case SystemMessage system -> builder.instructions(extractText(system.content())); + case UserMessage user -> + inputItems.add( + ResponseInputItem.ofEasyInputMessage( + EasyInputMessage.builder() + .role(EasyInputMessage.Role.USER) + .content(extractText(user.content())) + .build())); + case AssistantMessage assistant -> addAssistantInputItems(inputItems, assistant); + case ToolCallResultMessage toolResults -> + addToolResultInputItems(inputItems, toolResults); + default -> + throw new IllegalArgumentException( + "Unsupported message type: " + message.getClass().getSimpleName()); + } + } + } + if (!inputItems.isEmpty()) { + builder.inputOfResponse(inputItems); + } + + final var toolDefinitions = request.toolDefinitions(); + if (toolDefinitions != null && !toolDefinitions.isEmpty()) { + builder.tools(OpenAiToolConverter.toResponsesTools(toolDefinitions)); + } + + return builder.build(); + } + + @Nullable + private Long resolveMaxOutputTokens(ChatOptions options) { + if (options != null && options.maxOutputTokens() != null) { + return options.maxOutputTokens().longValue(); + } + return configuredMaxOutputTokens; + } + + private static String extractText(List content) { + if (content == null || content.isEmpty()) { + return ""; + } + final var sb = new StringBuilder(); + for (var c : content) { + if (c instanceof TextContent t) { + sb.append(t.text()); + } else if (c instanceof ObjectContent o) { + sb.append(String.valueOf(o.content())); + } else { + throw new IllegalArgumentException( + "Unsupported content block for text-only OpenAI Responses API: " + + c.getClass().getSimpleName()); + } + } + return sb.toString(); + } + + private void addAssistantInputItems( + List inputItems, AssistantMessage message) { + final var text = message.content() != null ? extractText(message.content()) : ""; + if (StringUtils.isNotEmpty(text)) { + inputItems.add( + ResponseInputItem.ofEasyInputMessage( + EasyInputMessage.builder() + .role(EasyInputMessage.Role.ASSISTANT) + .content(text) + .build())); + } + if (message.toolCalls() != null) { + for (var call : message.toolCalls()) { + inputItems.add(ResponseInputItem.ofFunctionCall(toFunctionToolCall(call))); + } + } + } + + private ResponseFunctionToolCall toFunctionToolCall(ToolCall call) { + final var args = call.arguments() != null ? toJsonString(call.arguments()) : "{}"; + return ResponseFunctionToolCall.builder() + .callId(call.id()) + .name(call.name()) + .arguments(args) + .build(); + } + + private void addToolResultInputItems( + List inputItems, ToolCallResultMessage message) { + for (var result : message.results()) { + inputItems.add(ResponseInputItem.ofFunctionCallOutput(toFunctionCallOutput(result))); + } + } + + private ResponseInputItem.FunctionCallOutput toFunctionCallOutput(ToolCallResult result) { + final var b = ResponseInputItem.FunctionCallOutput.builder(); + if (result.id() != null) { + b.callId(result.id()); + } + final var content = result.content(); + if (content == null) { + b.output(ToolCallResult.CONTENT_NO_RESULT); + } else if (content instanceof String s) { + b.output(s); + } else { + b.output(toJsonString(content)); + } + return b.build(); + } + + private String toJsonString(Object value) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception e) { + throw new IllegalStateException("Failed to serialize tool argument value", e); + } + } + + private static Optional resolveModelId(Response response) { + final var model = response.model(); + if (model.isString()) { + return Optional.of(model.asString()); + } + if (model.isChat()) { + return Optional.of(model.asChat().asString()); + } + if (model.isOnly()) { + return Optional.of(model.asOnly().asString()); + } + return Optional.empty(); + } + + private AssistantMessage toAssistantMessage(Response response) { + final var builder = AssistantMessage.builder(); + builder.metadata(Map.of("timestamp", ZonedDateTime.now())); + + if (StringUtils.isNotBlank(response.id())) { + builder.messageId(response.id()); + } + resolveModelId(response).filter(StringUtils::isNotBlank).ifPresent(builder::modelId); + + final var content = new ArrayList(); + final var toolCalls = new ArrayList(); + for (ResponseOutputItem item : response.output()) { + if (item.isMessage()) { + appendOutputMessageText(item.asMessage(), content); + } else if (item.isFunctionCall()) { + toolCalls.add(toToolCall(item.asFunctionCall())); + } + // ignore reasoning / web search / file search etc. for the text-only first cut + } + if (!content.isEmpty()) { + builder.content(content); + } + builder.toolCalls(toolCalls); + + builder.stopReason(toStopReason(response, !toolCalls.isEmpty())); + + response.usage().map(OpenAiResponsesChatModelApi::toTokenUsage).ifPresent(builder::usage); + + return builder.build(); + } + + private static void appendOutputMessageText( + ResponseOutputMessage outputMessage, List content) { + for (var part : outputMessage.content()) { + if (part.isOutputText()) { + final var text = part.asOutputText().text(); + if (StringUtils.isNotBlank(text)) { + content.add(TextContent.textContent(text)); + } + } + // refusal blocks are ignored in the text-only first cut + } + } + + private ToolCall toToolCall(ResponseFunctionToolCall functionCall) { + final var arguments = parseArguments(functionCall.arguments()); + return ToolCall.builder() + .id(functionCall.callId()) + .name(functionCall.name()) + .arguments(arguments) + .build(); + } + + private Map parseArguments(String json) { + if (json == null || json.isBlank()) { + return new LinkedHashMap<>(); + } + try { + return objectMapper.readValue(json, MAP_TYPE_REF); + } catch (Exception e) { + throw new IllegalStateException("Failed to parse tool call arguments JSON", e); + } + } + + private static StopReason toStopReason(Response response, boolean hasToolCalls) { + if (hasToolCalls) { + return StopReason.TOOL_USE; + } + final var status = response.status().flatMap(s -> Optional.ofNullable(s.known())).orElse(null); + if (status == null) { + return null; + } + return switch (status) { + case COMPLETED -> StopReason.STOP; + case INCOMPLETE -> + response + .incompleteDetails() + .flatMap(d -> d.reason()) + .flatMap(r -> Optional.ofNullable(r.known())) + .map( + known -> + switch (known) { + case MAX_OUTPUT_TOKENS -> StopReason.LENGTH; + case CONTENT_FILTER -> StopReason.CONTENT_FILTERED; + }) + .orElse(StopReason.STOP); + case FAILED -> StopReason.ERROR; + case CANCELLED -> StopReason.ABORTED; + case IN_PROGRESS, QUEUED -> null; + }; + } + + private static AgentMetrics.TokenUsage toTokenUsage(ResponseUsage usage) { + final var builder = + AgentMetrics.TokenUsage.builder() + .inputTokenCount((int) usage.inputTokens()) + .outputTokenCount((int) usage.outputTokens()); + final var inputDetails = usage.inputTokensDetails(); + if (inputDetails != null) { + builder.cacheReadInputTokenCount((int) inputDetails.cachedTokens()); + } + final var outputDetails = usage.outputTokensDetails(); + if (outputDetails != null) { + builder.reasoningTokenCount((int) outputDetails.reasoningTokens()); + } + return builder.build(); + } + + private static ConnectorException wrapModelCallFailure(RuntimeException e) { + final var message = + Optional.ofNullable(e.getMessage()) + .filter(StringUtils::isNotBlank) + .orElseGet(() -> e.getClass().getSimpleName()); + return new ConnectorException( + ERROR_CODE_FAILED_MODEL_CALL, "OpenAI Responses call failed: %s".formatted(message), e); + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiToolConverter.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiToolConverter.java index 10ddedfd9c3..78b0f109651 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiToolConverter.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiToolConverter.java @@ -11,6 +11,8 @@ import com.openai.models.FunctionParameters; import com.openai.models.chat.completions.ChatCompletionFunctionTool; import com.openai.models.chat.completions.ChatCompletionTool; +import com.openai.models.responses.FunctionTool; +import com.openai.models.responses.Tool; import io.camunda.connector.agenticai.model.tool.ToolDefinition; import java.util.List; import java.util.Map; @@ -50,4 +52,33 @@ private static FunctionParameters toFunctionParameters(Map schem schemaMap.forEach((key, value) -> builder.putAdditionalProperty(key, JsonValue.from(value))); return builder.build(); } + + /** + * Variant for the Responses API, which uses a different (but JSON-equivalent) Tool/Parameters + * shape. + */ + public static List toResponsesTools(List definitions) { + if (definitions == null || definitions.isEmpty()) { + return List.of(); + } + return definitions.stream().map(OpenAiToolConverter::toResponsesTool).toList(); + } + + public static Tool toResponsesTool(ToolDefinition definition) { + final var builder = FunctionTool.builder().name(definition.name()).strict(false); + if (definition.description() != null) { + builder.description(definition.description()); + } + builder.parameters(toResponsesParameters(definition.inputSchema())); + return Tool.ofFunction(builder.build()); + } + + private static FunctionTool.Parameters toResponsesParameters(Map schemaMap) { + final var builder = FunctionTool.Parameters.builder(); + if (schemaMap == null || schemaMap.isEmpty()) { + return builder.putAdditionalProperty("type", JsonValue.from("object")).build(); + } + schemaMap.forEach((key, value) -> builder.putAdditionalProperty(key, JsonValue.from(value))); + return builder.build(); + } } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiProviderConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiProviderConfiguration.java index 33ec82c4a29..5ddc543eb9a 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiProviderConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiProviderConfiguration.java @@ -32,7 +32,44 @@ public String providerType() { public record OpenAiConnection( @Valid @NotNull OpenAiAuthentication authentication, @Valid TimeoutConfiguration timeouts, - @Valid @NotNull OpenAiModel model) {} + @Valid @NotNull OpenAiModel model, + @TemplateProperty( + group = "provider", + label = "API", + description = + "Which OpenAI API family to use. Chat Completions is the default for backward compatibility; Responses is the newer endpoint with structured input/output items.", + type = TemplateProperty.PropertyType.Dropdown, + choices = { + @TemplateProperty.DropdownPropertyChoice( + label = "Chat Completions (default)", + value = "completions"), + @TemplateProperty.DropdownPropertyChoice(label = "Responses", value = "responses") + }, + feel = FeelMode.disabled, + optional = true, + defaultValue = "completions", + defaultValueType = TemplateProperty.DefaultValueType.String) + ApiFamily apiFamily) { + + public OpenAiConnection { + if (apiFamily == null) { + apiFamily = ApiFamily.COMPLETIONS; + } + } + + /** Convenience constructor used by existing call sites that pre-date the apiFamily field. */ + public OpenAiConnection( + OpenAiAuthentication authentication, TimeoutConfiguration timeouts, OpenAiModel model) { + this(authentication, timeouts, model, ApiFamily.COMPLETIONS); + } + } + + public enum ApiFamily { + @com.fasterxml.jackson.annotation.JsonProperty("completions") + COMPLETIONS, + @com.fasterxml.jackson.annotation.JsonProperty("responses") + RESPONSES + } public record OpenAiAuthentication( @NotBlank diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApiTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApiTest.java new file mode 100644 index 00000000000..ed52f7a9cf7 --- /dev/null +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApiTest.java @@ -0,0 +1,246 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.openai; + +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.assistantMessage; +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.systemMessage; +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.toolCallResultMessage; +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.userMessage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.client.OpenAIClient; +import com.openai.core.ObjectMappers; +import com.openai.models.responses.Response; +import com.openai.models.responses.ResponseCreateParams; +import com.openai.services.blocking.ResponseService; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatOptions; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; +import io.camunda.connector.agenticai.model.tool.ToolCall; +import io.camunda.connector.agenticai.model.tool.ToolCallResult; +import io.camunda.connector.agenticai.model.tool.ToolDefinition; +import io.camunda.connector.api.error.ConnectorException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletionException; +import org.assertj.core.groups.Tuple; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class OpenAiResponsesChatModelApiTest { + + private static final String MODEL_ID = "gpt-5"; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @Mock private OpenAIClient client; + @Mock private ResponseService responseService; + + @Captor private ArgumentCaptor paramsCaptor; + + private OpenAiResponsesChatModelApi api; + + @BeforeEach + void setUp() { + when(client.responses()).thenReturn(responseService); + api = new OpenAiResponsesChatModelApi(client, MODEL_ID, OBJECT_MAPPER, 1024L, null, null); + } + + @Test + void buildsExpectedParamsForSimpleConversation() { + when(responseService.create((ResponseCreateParams) paramsCaptor.capture())) + .thenReturn(textOnlyResponse("hello")); + + var request = + new ChatRequest(List.of(systemMessage("be helpful"), userMessage("hi")), List.of(), null); + api.complete(request, defaultOptions(), ChatStreamListener.NOOP).join(); + + var params = paramsCaptor.getValue(); + assertThat(params.model().flatMap(m -> m.string())).hasValue(MODEL_ID); + assertThat(params.maxOutputTokens()).hasValue(1024L); + assertThat(params.instructions()).hasValue("be helpful"); + assertThat(params.input().get().asResponse()).hasSize(1); // user + } + + @Test + void mapsAssistantToolCallsBackToContentBlocks() { + when(responseService.create(any(ResponseCreateParams.class))) + .thenReturn(toolCallResponse("getWeather", "abc", "{\"location\":\"MUC\"}")); + + var response = + api.complete( + new ChatRequest(List.of(userMessage("weather?")), tools(), null), + defaultOptions(), + ChatStreamListener.NOOP) + .join(); + + var assistant = response.assistantMessage(); + assertThat(assistant.toolCalls()) + .extracting(ToolCall::id, ToolCall::name) + .containsExactly(Tuple.tuple("abc", "getWeather")); + assertThat(assistant.toolCalls().getFirst().arguments()).containsEntry("location", "MUC"); + assertThat(assistant.stopReason()) + .isEqualTo(io.camunda.connector.agenticai.model.message.StopReason.TOOL_USE); + } + + @Test + void priorAssistantToolCallsAndResultsRoundTripIntoInputItems() { + when(responseService.create((ResponseCreateParams) paramsCaptor.capture())) + .thenReturn(textOnlyResponse("ack")); + + var prior = + assistantMessage( + "let me check", + List.of( + ToolCall.builder() + .id("abc") + .name("getWeather") + .arguments(Map.of("location", "MUC")) + .build())); + var results = + toolCallResultMessage( + List.of( + ToolCallResult.builder().id("abc").name("getWeather").content("Sunny").build())); + + api.complete( + new ChatRequest( + List.of(userMessage("weather?"), prior, results, userMessage("thanks")), + tools(), + null), + defaultOptions(), + ChatStreamListener.NOOP) + .join(); + + var params = paramsCaptor.getValue(); + var items = params.input().get().asResponse(); + // user, assistant text, function call, function call output, user → 5 input items + assertThat(items).hasSize(5); + } + + @Test + void wrapsSdkExceptionInConnectorException() { + when(responseService.create(any(ResponseCreateParams.class))) + .thenThrow(new RuntimeException("boom")); + + var future = + api.complete( + new ChatRequest(List.of(userMessage("hi")), List.of(), null), + defaultOptions(), + ChatStreamListener.NOOP); + + assertThatThrownBy(future::join) + .isInstanceOf(CompletionException.class) + .cause() + .isInstanceOf(ConnectorException.class) + .hasMessageContaining("OpenAI Responses call failed"); + } + + private static ChatOptions defaultOptions() { + return new ChatOptions(null, null, null, Map.of()); + } + + private static List tools() { + return List.of( + ToolDefinition.builder() + .name("getWeather") + .description("Returns the current weather") + .inputSchema( + Map.of( + "type", "object", + "properties", Map.of("location", Map.of("type", "string")), + "required", List.of("location"))) + .build()); + } + + private static Response textOnlyResponse(String text) { + return responseFromJson( + """ + { + "id": "resp_1", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "%s", + "parallel_tool_calls": true, + "tool_choice": "auto", + "tools": [], + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [ + { "type": "output_text", "text": "%s", "annotations": [] } + ] + } + ], + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "input_tokens_details": { "cached_tokens": 0 }, + "output_tokens_details": { "reasoning_tokens": 0 } + } + } + """ + .formatted(MODEL_ID, text)); + } + + private static Response toolCallResponse(String name, String callId, String argumentsJson) { + final var escapedArgs = argumentsJson.replace("\"", "\\\""); + return responseFromJson( + """ + { + "id": "resp_2", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "%s", + "parallel_tool_calls": true, + "tool_choice": "auto", + "tools": [], + "output": [ + { + "type": "function_call", + "call_id": "%s", + "name": "%s", + "arguments": "%s" + } + ], + "usage": { + "input_tokens": 15, + "output_tokens": 8, + "total_tokens": 23, + "input_tokens_details": { "cached_tokens": 0 }, + "output_tokens_details": { "reasoning_tokens": 0 } + } + } + """ + .formatted(MODEL_ID, callId, name, escapedArgs)); + } + + private static Response responseFromJson(String json) { + try { + return ObjectMappers.jsonMapper().readValue(json, Response.class); + } catch (Exception e) { + throw new IllegalStateException("Failed to deserialize test Response fixture", e); + } + } +} From 3cdfd6b1f9dfed2f833c1ac0a3fb83be75572049 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 15:44:39 +0200 Subject: [PATCH 55/81] fix(agentic-ai): align native baseUrl handling + cover Responses wire format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The native Anthropic SDK appends `/v1/messages` to its `baseUrl` internally, while the LangChain4j Anthropic client (which existing element-template configs target) expected the user to put the `/v1` in the baseUrl itself. Strip a trailing `/v1` in `AnthropicMessagesChatModelApiFactory.normalizeBaseUrl(...)` so the existing wire-format e2e — and any existing user configs — keep working when the native impl takes over. Also expose a `Custom API endpoint` field on `OpenAiProviderConfiguration` (optional, used for OpenAI proxies/gateways and by the new Responses wire-format e2e to redirect to WireMock). The existing OpenAI direct config exposed no endpoint override, which made it impossible to write a wire-format test against the native `openai-responses` impl without this field. Wire-format e2e changes: - Rename the existing `OpenAiResponsesApiAiAgentJobWorkerTests` to `OpenAiChatCompletionsApiAiAgentJobWorkerTests` to match what the test actually covers (`/v1/chat/completions` via openaiCompatible). - Add a new `OpenAiResponsesApiAiAgentJobWorkerTests` that drives the `openai` discriminator with `apiFamily = responses`, stubs `POST /v1/responses` against a Responses-API JSON body, and verifies the same tool-call → final-text scenario the other two tests cover. --- ...atCompletionsApiAiAgentJobWorkerTests.java | 151 ++++++++++++++++++ ...enAiResponsesApiAiAgentJobWorkerTests.java | 98 ++++++------ .../agenticai-aiagent-job-worker.json | 17 ++ .../agenticai-aiagent-outbound-connector.json | 17 ++ .../agenticai-aiagent-job-worker-hybrid.json | 17 ++ ...cai-aiagent-outbound-connector-hybrid.json | 17 ++ .../AnthropicMessagesChatModelApiFactory.java | 15 +- .../openai/OpenAiChatModelApiFactory.java | 3 + .../provider/OpenAiProviderConfiguration.java | 15 +- 9 files changed, 298 insertions(+), 52 deletions(-) create mode 100644 connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/OpenAiChatCompletionsApiAiAgentJobWorkerTests.java diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/OpenAiChatCompletionsApiAiAgentJobWorkerTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/OpenAiChatCompletionsApiAiAgentJobWorkerTests.java new file mode 100644 index 00000000000..585d8d8cf29 --- /dev/null +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/OpenAiChatCompletionsApiAiAgentJobWorkerTests.java @@ -0,0 +1,151 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information regarding copyright + * ownership. Camunda licenses this file to you under the Apache License, + * Version 2.0; you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.camunda.connector.e2e.agenticai.aiagent.wireformat; + +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; +import static org.assertj.core.api.Assertions.assertThat; + +import com.github.tomakehurst.wiremock.client.MappingBuilder; +import io.camunda.connector.e2e.agenticai.assertj.JobWorkerAgentResponseAssert; +import io.camunda.connector.test.utils.annotation.SlowTest; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * Wire-format regression test for the OpenAI Chat Completions endpoint ({@code POST + * /v1/chat/completions}). Uses the {@code openaiCompatible} provider so the test exercises the same + * wire format that the {@code openai} discriminator produces with {@code apiFamily = COMPLETIONS}. + */ +@SlowTest +public class OpenAiChatCompletionsApiAiAgentJobWorkerTests + extends BaseWireFormatAiAgentJobWorkerTest { + + @Override + protected String llmApiPath() { + return "/v1/chat/completions"; + } + + @Override + protected Map elementTemplateProperties() { + return Map.ofEntries( + Map.entry("agentContext", "=agent.context"), + Map.entry("provider.type", "openaiCompatible"), + Map.entry("provider.openaiCompatible.endpoint", "http://localhost:" + wireMockPort + "/v1"), + Map.entry("provider.openaiCompatible.authentication.apiKey", "test-api-key"), + Map.entry("provider.openaiCompatible.model.model", "gpt-4o"), + Map.entry( + "data.systemPrompt.prompt", + "=\"You are a helpful AI assistant. Answer all the questions, but always be nice.\""), + Map.entry( + "data.userPrompt.prompt", + "=if (is defined(followUpUserPrompt)) then followUpUserPrompt else userPrompt"), + Map.entry("data.userPrompt.documents", "=[]"), + Map.entry("data.memory.storage.type", "in-process"), + Map.entry("data.response.includeAssistantMessage", "=true"), + Map.entry("data.response.includeAgentContext", "=true")); + } + + @Override + protected MappingBuilder withApiKeyHeaderMatcher(MappingBuilder stub) { + return stub.withHeader("Authorization", equalTo("Bearer test-api-key")); + } + + @Override + protected String toolCallResponseBody() { + return """ + { + "id": "chatcmpl-abc123", + "object": "chat.completion", + "created": 1728933352, + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_xyz789", + "type": "function", + "function": { + "name": "SuperfluxProduct", + "arguments": "{\\"a\\": 5, \\"b\\": 3}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150 + } + } + """; + } + + @Override + protected String finalResponseBody() { + return """ + { + "id": "chatcmpl-def456", + "object": "chat.completion", + "created": 1728933353, + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "%s" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 200, + "completion_tokens": 30, + "total_tokens": 230 + } + } + """ + .formatted(RESPONSE_TEXT); + } + + @Test + void executesAgentWithToolCallAgainstOpenAiChatCompletionsApi() throws Exception { + final var zeebeTest = runToolCallScenario(); + + assertAgentResponse( + zeebeTest, + agentResponse -> + JobWorkerAgentResponseAssert.assertThat(agentResponse) + .isReady() + .hasMetrics(EXPECTED_METRICS) + .hasResponseMessageText(RESPONSE_TEXT) + .hasResponseText(RESPONSE_TEXT)); + + assertThat(userFeedbackJobWorkerCounter.get()).isEqualTo(1); + verify(2, postRequestedFor(urlEqualTo(llmApiPath()))); + } +} diff --git a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/OpenAiResponsesApiAiAgentJobWorkerTests.java b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/OpenAiResponsesApiAiAgentJobWorkerTests.java index 2e125f8899a..fec9dbcd42f 100644 --- a/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/OpenAiResponsesApiAiAgentJobWorkerTests.java +++ b/connectors-e2e-test/connectors-e2e-test-agentic-ai/src/test/java/io/camunda/connector/e2e/agenticai/aiagent/wireformat/OpenAiResponsesApiAiAgentJobWorkerTests.java @@ -29,28 +29,27 @@ import org.junit.jupiter.api.Test; /** - * Wire-format regression test for the OpenAI API. Uses the {@code openAiCompatible} provider - * (OpenAI Chat Completions, {@code POST /v1/chat/completions}) so the test runs against the - * LangChain4j implementation today. Once ADR 004's native {@code openai-responses} API family - * ships, this test will be updated to target the Responses API endpoint ({@code POST - * /v1/responses}) with the corresponding request/response schema. + * Wire-format regression test for the OpenAI Responses endpoint ({@code POST /v1/responses}). + * Drives the {@code openai} discriminator with {@code apiFamily = responses}, exercising the native + * {@code OpenAiResponsesChatModelApi} end-to-end against a WireMock-stubbed responses API. */ @SlowTest public class OpenAiResponsesApiAiAgentJobWorkerTests extends BaseWireFormatAiAgentJobWorkerTest { @Override protected String llmApiPath() { - return "/v1/chat/completions"; + return "/v1/responses"; } @Override protected Map elementTemplateProperties() { return Map.ofEntries( Map.entry("agentContext", "=agent.context"), - Map.entry("provider.type", "openaiCompatible"), - Map.entry("provider.openaiCompatible.endpoint", "http://localhost:" + wireMockPort + "/v1"), - Map.entry("provider.openaiCompatible.authentication.apiKey", "test-api-key"), - Map.entry("provider.openaiCompatible.model.model", "gpt-4o"), + Map.entry("provider.type", "openai"), + Map.entry("provider.openai.apiFamily", "responses"), + Map.entry("provider.openai.endpoint", "http://localhost:" + wireMockPort + "/v1"), + Map.entry("provider.openai.authentication.apiKey", "test-api-key"), + Map.entry("provider.openai.model.model", "gpt-5"), Map.entry( "data.systemPrompt.prompt", "=\"You are a helpful AI assistant. Answer all the questions, but always be nice.\""), @@ -72,34 +71,28 @@ protected MappingBuilder withApiKeyHeaderMatcher(MappingBuilder stub) { protected String toolCallResponseBody() { return """ { - "id": "chatcmpl-abc123", - "object": "chat.completion", - "created": 1728933352, - "model": "gpt-4o", - "choices": [ + "id": "resp_abc123", + "object": "response", + "created_at": 1728933352, + "status": "completed", + "model": "gpt-5", + "parallel_tool_calls": true, + "tool_choice": "auto", + "tools": [], + "output": [ { - "index": 0, - "message": { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "call_xyz789", - "type": "function", - "function": { - "name": "SuperfluxProduct", - "arguments": "{\\"a\\": 5, \\"b\\": 3}" - } - } - ] - }, - "finish_reason": "tool_calls" + "type": "function_call", + "call_id": "call_xyz789", + "name": "SuperfluxProduct", + "arguments": "{\\"a\\": 5, \\"b\\": 3}" } ], "usage": { - "prompt_tokens": 100, - "completion_tokens": 50, - "total_tokens": 150 + "input_tokens": 100, + "output_tokens": 50, + "total_tokens": 150, + "input_tokens_details": { "cached_tokens": 0 }, + "output_tokens_details": { "reasoning_tokens": 0 } } } """; @@ -109,24 +102,31 @@ protected String toolCallResponseBody() { protected String finalResponseBody() { return """ { - "id": "chatcmpl-def456", - "object": "chat.completion", - "created": 1728933353, - "model": "gpt-4o", - "choices": [ + "id": "resp_def456", + "object": "response", + "created_at": 1728933353, + "status": "completed", + "model": "gpt-5", + "parallel_tool_calls": true, + "tool_choice": "auto", + "tools": [], + "output": [ { - "index": 0, - "message": { - "role": "assistant", - "content": "%s" - }, - "finish_reason": "stop" + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [ + { "type": "output_text", "text": "%s", "annotations": [] } + ] } ], "usage": { - "prompt_tokens": 200, - "completion_tokens": 30, - "total_tokens": 230 + "input_tokens": 200, + "output_tokens": 30, + "total_tokens": 230, + "input_tokens_details": { "cached_tokens": 0 }, + "output_tokens_details": { "reasoning_tokens": 0 } } } """ @@ -134,7 +134,7 @@ protected String finalResponseBody() { } @Test - void executesAgentWithToolCallAgainstOpenAiChatCompletionsApi() throws Exception { + void executesAgentWithToolCallAgainstOpenAiResponsesApi() throws Exception { final var zeebeTest = runToolCallScenario(); assertAgentResponse( diff --git a/connectors/agentic-ai/element-templates/agenticai-aiagent-job-worker.json b/connectors/agentic-ai/element-templates/agenticai-aiagent-job-worker.json index b1a5f826352..c100c2cffb1 100644 --- a/connectors/agentic-ai/element-templates/agenticai-aiagent-job-worker.json +++ b/connectors/agentic-ai/element-templates/agenticai-aiagent-job-worker.json @@ -705,6 +705,23 @@ "name" : "Responses", "value" : "responses" } ] + }, { + "id" : "provider.openai.endpoint", + "label" : "Custom API endpoint", + "description" : "Optional. Override the default OpenAI base URL (e.g. for an OpenAI proxy or gateway). Leave blank to use the SDK default.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" }, { "id" : "provider.openaiCompatible.endpoint", "label" : "API endpoint", diff --git a/connectors/agentic-ai/element-templates/agenticai-aiagent-outbound-connector.json b/connectors/agentic-ai/element-templates/agenticai-aiagent-outbound-connector.json index e28221c51be..512e85a5036 100644 --- a/connectors/agentic-ai/element-templates/agenticai-aiagent-outbound-connector.json +++ b/connectors/agentic-ai/element-templates/agenticai-aiagent-outbound-connector.json @@ -684,6 +684,23 @@ "name" : "Responses", "value" : "responses" } ] + }, { + "id" : "provider.openai.endpoint", + "label" : "Custom API endpoint", + "description" : "Optional. Override the default OpenAI base URL (e.g. for an OpenAI proxy or gateway). Leave blank to use the SDK default.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" }, { "id" : "provider.openaiCompatible.endpoint", "label" : "API endpoint", diff --git a/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-job-worker-hybrid.json b/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-job-worker-hybrid.json index 5b575ac6d6e..3c9cb6904d2 100644 --- a/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-job-worker-hybrid.json +++ b/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-job-worker-hybrid.json @@ -710,6 +710,23 @@ "name" : "Responses", "value" : "responses" } ] + }, { + "id" : "provider.openai.endpoint", + "label" : "Custom API endpoint", + "description" : "Optional. Override the default OpenAI base URL (e.g. for an OpenAI proxy or gateway). Leave blank to use the SDK default.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" }, { "id" : "provider.openaiCompatible.endpoint", "label" : "API endpoint", diff --git a/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-outbound-connector-hybrid.json b/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-outbound-connector-hybrid.json index ed8c9f282e0..b85ec062563 100644 --- a/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-outbound-connector-hybrid.json +++ b/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-outbound-connector-hybrid.json @@ -689,6 +689,23 @@ "name" : "Responses", "value" : "responses" } ] + }, { + "id" : "provider.openai.endpoint", + "label" : "Custom API endpoint", + "description" : "Optional. Override the default OpenAI base URL (e.g. for an OpenAI proxy or gateway). Leave blank to use the SDK default.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" }, { "id" : "provider.openaiCompatible.endpoint", "label" : "API endpoint", diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java index 920af38a6f7..6dc904b24a0 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java @@ -73,7 +73,7 @@ private AnthropicClient buildClient(AnthropicConnection connection) { AnthropicOkHttpClient.builder().apiKey(connection.authentication().apiKey()); if (StringUtils.isNotBlank(connection.endpoint())) { - builder.baseUrl(connection.endpoint()); + builder.baseUrl(normalizeBaseUrl(connection.endpoint())); } final var timeout = resolveTimeout(connection); @@ -88,4 +88,17 @@ private AnthropicClient buildClient(AnthropicConnection connection) { private Duration resolveTimeout(AnthropicConnection connection) { return Optional.ofNullable(connection.timeouts()).map(t -> t.timeout()).orElse(defaultTimeout); } + + /** + * The {@code anthropic-java} SDK expects {@code baseUrl} to be the host without the {@code /v1} + * prefix and appends the full {@code /v1/messages} path itself. The LangChain4j Anthropic client + * used the opposite convention (callers set {@code https://host/v1}, L4J appends just {@code + * /messages}). Strip a trailing {@code /v1} (with or without trailing slash) so existing + * element-template configurations keep working when users switch to the native impl. + */ + static String normalizeBaseUrl(String endpoint) { + final var trimmed = + endpoint.endsWith("/") ? endpoint.substring(0, endpoint.length() - 1) : endpoint; + return trimmed.endsWith("/v1") ? trimmed.substring(0, trimmed.length() - 3) : trimmed; + } } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java index d8884abdc85..8cf3efce630 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java @@ -80,6 +80,9 @@ private OpenAIClient buildClient(OpenAiConnection connection) { final var builder = OpenAIOkHttpClient.builder(); builder.apiKey(connection.authentication().apiKey()); + if (StringUtils.isNotBlank(connection.endpoint())) { + builder.baseUrl(connection.endpoint()); + } if (StringUtils.isNotBlank(connection.authentication().organizationId())) { builder.organization(connection.authentication().organizationId()); } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiProviderConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiProviderConfiguration.java index 5ddc543eb9a..13e8c2b2ae8 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiProviderConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiProviderConfiguration.java @@ -8,6 +8,7 @@ import static io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OPENAI_ID; +import io.camunda.connector.agenticai.aiagent.model.request.provider.shared.HttpUrl; import io.camunda.connector.agenticai.aiagent.model.request.provider.shared.TimeoutConfiguration; import io.camunda.connector.generator.java.annotation.FeelMode; import io.camunda.connector.generator.java.annotation.TemplateProperty; @@ -49,7 +50,17 @@ public record OpenAiConnection( optional = true, defaultValue = "completions", defaultValueType = TemplateProperty.DefaultValueType.String) - ApiFamily apiFamily) { + ApiFamily apiFamily, + @HttpUrl + @TemplateProperty( + group = "provider", + label = "Custom API endpoint", + description = + "Optional. Override the default OpenAI base URL (e.g. for an OpenAI proxy or gateway). Leave blank to use the SDK default.", + type = TemplateProperty.PropertyType.String, + feel = FeelMode.optional, + optional = true) + String endpoint) { public OpenAiConnection { if (apiFamily == null) { @@ -60,7 +71,7 @@ public record OpenAiConnection( /** Convenience constructor used by existing call sites that pre-date the apiFamily field. */ public OpenAiConnection( OpenAiAuthentication authentication, TimeoutConfiguration timeouts, OpenAiModel model) { - this(authentication, timeouts, model, ApiFamily.COMPLETIONS); + this(authentication, timeouts, model, ApiFamily.COMPLETIONS, null); } } From df9aea569af8deb37b77eb353a484d411807492f Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 17:11:03 +0200 Subject: [PATCH 56/81] feat(agentic-ai): capability matrix + resolver (Phase E1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a per-model capability matrix loaded as Spring Boot configuration so library consumers can override or extend any entry via their own application.yml without restating the bundled defaults. The bundled YAML ships at classpath:capabilities/model-capabilities.yaml and is registered as a low-precedence PropertySource by an EnvironmentPostProcessor; deep merging with consumer overrides and per-entry capability inheritance follow Spring Boot semantics (maps deep-merge, lists replace, scalars override). The 4-step resolver chain — override -> exact id/alias -> longest pattern match -> conservative defaults — materialises a flat ModelCapabilities for use by the native ChatModelApi impls in later sub-phases. --- .../framework/api/ModelCapabilities.java | 6 + .../AgenticAiCapabilitiesConfiguration.java | 39 ++ .../AgenticAiFrameworkProperties.java | 65 +++ .../capabilities/CapabilityMatrix.java | 46 +++ ...abilityMatrixEnvironmentPostProcessor.java | 50 +++ .../capabilities/CapabilityMatrixFactory.java | 106 +++++ .../ModelCapabilitiesResolver.java | 201 +++++++++ .../capabilities/ModelCapabilitiesYaml.java | 78 ++++ .../AgenticAiConnectorsAutoConfiguration.java | 2 + ....boot.env.EnvironmentPostProcessor.imports | 1 + .../capabilities/model-capabilities.yaml | 217 ++++++++++ .../BundledCapabilityMatrixTest.java | 181 +++++++++ .../CapabilityMatrixOverrideTest.java | 115 ++++++ .../ModelCapabilitiesResolverTest.java | 380 ++++++++++++++++++ 14 files changed, 1487 insertions(+) create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/AgenticAiCapabilitiesConfiguration.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/AgenticAiFrameworkProperties.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrix.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixEnvironmentPostProcessor.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixFactory.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesResolver.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesYaml.java create mode 100644 connectors/agentic-ai/src/main/resources/META-INF/spring/org.springframework.boot.env.EnvironmentPostProcessor.imports create mode 100644 connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml create mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/BundledCapabilityMatrixTest.java create mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixOverrideTest.java create mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesResolverTest.java diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java index c83980bc188..1a8d35d7130 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java @@ -6,6 +6,7 @@ */ package io.camunda.connector.agenticai.aiagent.framework.api; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; import org.springframework.lang.Nullable; @@ -30,10 +31,15 @@ public record ModelCapabilities( @Nullable Integer maxOutputTokens) { public enum Modality { + @JsonProperty("text") TEXT, + @JsonProperty("image") IMAGE, + @JsonProperty("pdf") PDF, + @JsonProperty("audio") AUDIO, + @JsonProperty("video") VIDEO } } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/AgenticAiCapabilitiesConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/AgenticAiCapabilitiesConfiguration.java new file mode 100644 index 00000000000..92d2eda0c6e --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/AgenticAiCapabilitiesConfiguration.java @@ -0,0 +1,39 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.capabilities; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.connector.runtime.annotation.ConnectorsObjectMapper; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Spring configuration for the model capability matrix. Bundled defaults from {@code + * resources/capabilities/model-capabilities.yaml} are loaded as a low-precedence property source by + * {@link CapabilityMatrixEnvironmentPostProcessor}; library-consumer overrides under the same + * {@code camunda.connector.agenticai.aiagent.framework.capabilities.*} prefix land on top. + */ +@Configuration +@EnableConfigurationProperties(AgenticAiFrameworkProperties.class) +public class AgenticAiCapabilitiesConfiguration { + + @Bean + @ConditionalOnMissingBean + public CapabilityMatrix aiAgentCapabilityMatrix( + AgenticAiFrameworkProperties properties, @ConnectorsObjectMapper ObjectMapper objectMapper) { + return CapabilityMatrixFactory.build(properties, objectMapper); + } + + @Bean + @ConditionalOnMissingBean + public ModelCapabilitiesResolver aiAgentModelCapabilitiesResolver( + CapabilityMatrix matrix, @ConnectorsObjectMapper ObjectMapper objectMapper) { + return new ModelCapabilitiesResolver(matrix, objectMapper); + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/AgenticAiFrameworkProperties.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/AgenticAiFrameworkProperties.java new file mode 100644 index 00000000000..189c9d5e2be --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/AgenticAiFrameworkProperties.java @@ -0,0 +1,65 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.capabilities; + +import java.util.List; +import java.util.Map; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.lang.Nullable; + +/** + * Spring Boot configuration binding for the agentic-ai framework. The {@code capabilities} map is + * populated from two layers, deep-merged by Spring Boot: + * + *

      + *
    1. Bundled defaults: {@code resources/capabilities/model-capabilities.yaml}, registered as a + * low-precedence {@link org.springframework.core.env.PropertySource} by {@link + * CapabilityMatrixEnvironmentPostProcessor} at startup. + *
    2. Application overrides: any property under {@code + * camunda.connector.agenticai.aiagent.framework.capabilities.*} declared by the library + * consumer (typically in their own {@code application.yml}). Library consumers can override + * an individual model's capability fields, replace a sub-modality list, or add a brand-new + * model entry under any api family without restating the bundled matrix. + *
    + * + * Map keys under {@code models} carry the discriminator: keys containing {@code *} are treated as + * glob patterns, otherwise as model ids. Optional explicit {@code id} / {@code pattern} fields + * inside an entry override the key derivation when needed. + * + *

    Capability sub-trees ({@code defaults} and per-entry {@code capabilities}) are bound to the + * sparse {@link ModelCapabilitiesYaml} record so Spring Boot's relaxed binding can rebuild modality + * lists from indexed property keys; the resolver projects them onto a flat {@link + * io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities} via Jackson tree merge at + * lookup time. + */ +@ConfigurationProperties("camunda.connector.agenticai.aiagent.framework") +public record AgenticAiFrameworkProperties(Map capabilities) { + + public AgenticAiFrameworkProperties { + capabilities = capabilities == null ? Map.of() : Map.copyOf(capabilities); + } + + public record ApiFamilyProperties( + @Nullable ModelCapabilitiesYaml defaults, Map models) { + + public ApiFamilyProperties { + models = models == null ? Map.of() : Map.copyOf(models); + } + } + + public record ModelEntryProperties( + @Nullable String id, + List pattern, + List aliases, + @Nullable ModelCapabilitiesYaml capabilities) { + + public ModelEntryProperties { + aliases = aliases == null ? List.of() : List.copyOf(aliases); + pattern = pattern == null ? List.of() : List.copyOf(pattern); + } + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrix.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrix.java new file mode 100644 index 00000000000..0385bbf0369 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrix.java @@ -0,0 +1,46 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.capabilities; + +import com.fasterxml.jackson.databind.JsonNode; +import java.util.List; +import java.util.Map; +import org.springframework.lang.Nullable; + +/** + * Built capability matrix used by the {@link ModelCapabilitiesResolver}. Materialised at startup + * from {@link AgenticAiFrameworkProperties} (Spring Boot config) by {@link + * CapabilityMatrixFactory}: bundled defaults from the classpath YAML are loaded as a low-precedence + * {@link org.springframework.core.env.PropertySource} and library-consumer overrides land on top + * before this matrix is built. + * + *

    Capability sub-trees are kept as raw {@link JsonNode}s so the resolver can deep-merge + * (Spring-Boot semantics: maps deep-merge, lists replace, scalars override) at lookup time. + */ +public record CapabilityMatrix(Map families) { + + public CapabilityMatrix { + families = families == null ? Map.of() : Map.copyOf(families); + } + + public record ApiFamily(@Nullable JsonNode defaults, List models) { + public ApiFamily { + models = models == null ? List.of() : List.copyOf(models); + } + } + + public record ModelEntry( + @Nullable String id, + List aliases, + List patterns, + @Nullable JsonNode capabilities) { + public ModelEntry { + aliases = aliases == null ? List.of() : List.copyOf(aliases); + patterns = patterns == null ? List.of() : List.copyOf(patterns); + } + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixEnvironmentPostProcessor.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixEnvironmentPostProcessor.java new file mode 100644 index 00000000000..98e3e5048d0 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixEnvironmentPostProcessor.java @@ -0,0 +1,50 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.capabilities; + +import java.io.IOException; +import java.util.List; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.env.EnvironmentPostProcessor; +import org.springframework.boot.env.YamlPropertySourceLoader; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.PropertySource; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; + +/** + * Loads the bundled model capability matrix YAML as a low-precedence {@link PropertySource} so + * library consumers can override any value via their own {@code application.yml}. + * + *

    The bundled file lives at {@code classpath:capabilities/model-capabilities.yaml} and is + * structured under the {@code camunda.connector.agenticai.aiagent.framework.capabilities} prefix. + * It is registered with {@code addLast(...)} so any user-supplied source — including {@code + * application.yml}, environment variables and command-line arguments — wins. + */ +public class CapabilityMatrixEnvironmentPostProcessor implements EnvironmentPostProcessor { + + private static final String BUNDLED_RESOURCE = "capabilities/model-capabilities.yaml"; + private static final String PROPERTY_SOURCE_NAME = "agentic-ai-bundled-capability-matrix"; + + private final YamlPropertySourceLoader loader = new YamlPropertySourceLoader(); + + @Override + public void postProcessEnvironment( + ConfigurableEnvironment environment, SpringApplication application) { + final Resource resource = new ClassPathResource(BUNDLED_RESOURCE); + if (!resource.exists()) { + return; + } + try { + final List> sources = loader.load(PROPERTY_SOURCE_NAME, resource); + sources.forEach(environment.getPropertySources()::addLast); + } catch (IOException e) { + throw new IllegalStateException( + "Failed to load bundled capability matrix from classpath:" + BUNDLED_RESOURCE, e); + } + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixFactory.java new file mode 100644 index 00000000000..e4a6a0844f1 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixFactory.java @@ -0,0 +1,106 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.capabilities; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.connector.agenticai.aiagent.framework.capabilities.AgenticAiFrameworkProperties.ApiFamilyProperties; +import io.camunda.connector.agenticai.aiagent.framework.capabilities.AgenticAiFrameworkProperties.ModelEntryProperties; +import io.camunda.connector.agenticai.aiagent.framework.capabilities.CapabilityMatrix.ApiFamily; +import io.camunda.connector.agenticai.aiagent.framework.capabilities.CapabilityMatrix.ModelEntry; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Materialises a {@link CapabilityMatrix} from {@link AgenticAiFrameworkProperties} bound by Spring + * Boot. Per-entry capability sub-trees ({@code defaults} and {@code capabilities}) are converted to + * {@link JsonNode} so the resolver can deep-merge them at lookup time. + * + *

    Each {@code models} map entry must carry exactly one discriminator: + * + *

      + *
    • An explicit {@code pattern} field — a glob string (uses {@code *} only) or a list of globs. + * The entry matches when any glob in the list matches the requested model id; longest-match + * across entries wins. Aliases are not allowed for pattern entries. + *
    • An explicit {@code id} field — the entry matches that model id exactly. + *
    • Neither — the map key itself is treated as the model id. + *
    + * + *

    {@code *} and {@code .} cannot appear in the map key because Spring Boot's map binding strips + * them; pattern entries must declare the glob in the {@code pattern} field while the map key stays + * a stable, override-friendly identifier. + */ +public final class CapabilityMatrixFactory { + + private CapabilityMatrixFactory() {} + + public static CapabilityMatrix build( + AgenticAiFrameworkProperties properties, ObjectMapper objectMapper) { + final Map families = new LinkedHashMap<>(); + properties + .capabilities() + .forEach( + (familyName, family) -> + families.put(familyName, buildFamily(familyName, family, objectMapper))); + return new CapabilityMatrix(families); + } + + private static ApiFamily buildFamily( + String familyName, ApiFamilyProperties family, ObjectMapper objectMapper) { + final JsonNode defaults = + family.defaults() == null ? null : objectMapper.valueToTree(family.defaults()); + final List entries = new ArrayList<>(); + family + .models() + .forEach( + (mapKey, entry) -> entries.add(buildEntry(familyName, mapKey, entry, objectMapper))); + return new ApiFamily(defaults, entries); + } + + private static ModelEntry buildEntry( + String familyName, String mapKey, ModelEntryProperties entry, ObjectMapper objectMapper) { + final boolean idExplicit = entry.id() != null && !entry.id().isBlank(); + final List patterns = nonBlank(entry.pattern()); + final boolean patternExplicit = !patterns.isEmpty(); + + if (idExplicit && patternExplicit) { + throw new IllegalStateException( + "Capability matrix entry '%s' under api family '%s' must specify at most one of `id` or `pattern`" + .formatted(mapKey, familyName)); + } + + final String id; + final List resolvedPatterns; + if (patternExplicit) { + id = null; + resolvedPatterns = patterns; + } else { + // id explicit, or id derived from the map key. + id = idExplicit ? entry.id() : mapKey; + resolvedPatterns = List.of(); + } + + if (!resolvedPatterns.isEmpty() && !entry.aliases().isEmpty()) { + throw new IllegalStateException( + "Capability matrix pattern entry '%s' under api family '%s' cannot declare aliases" + .formatted(mapKey, familyName)); + } + + final JsonNode capabilities = + entry.capabilities() == null + ? objectMapper.createObjectNode() + : objectMapper.valueToTree(entry.capabilities()); + + return new ModelEntry(id, entry.aliases(), resolvedPatterns, capabilities); + } + + private static List nonBlank(List values) { + return values.stream().filter(p -> p != null && !p.isBlank()).toList(); + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesResolver.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesResolver.java new file mode 100644 index 00000000000..74b8b8e2535 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesResolver.java @@ -0,0 +1,201 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.capabilities; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; +import io.camunda.connector.agenticai.aiagent.framework.capabilities.CapabilityMatrix.ApiFamily; +import io.camunda.connector.agenticai.aiagent.framework.capabilities.CapabilityMatrix.ModelEntry; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.lang.Nullable; + +/** + * Resolves a runtime {@link ModelCapabilities} from the capability matrix using the four-step chain + * defined in ADR-004: + * + *

      + *
    1. Connector config override (if present) + *
    2. Exact id or alias match within the api family + *
    3. Glob pattern match (longest match wins) + *
    4. Conservative defaults (text-only, all flags false, null limits) + *
    + * + * Pattern and default fall-throughs emit one INFO log per (api family, model id) so operators + * notice when they're running on best-effort capabilities. Exact / alias matches are silent — + * they're verified declarations. + */ +public class ModelCapabilitiesResolver { + + private static final Logger LOG = LoggerFactory.getLogger(ModelCapabilitiesResolver.class); + + static final ModelCapabilities CONSERVATIVE_DEFAULTS = + new ModelCapabilities( + List.of(Modality.TEXT), + List.of(Modality.TEXT), + List.of(Modality.TEXT), + false, + false, + false, + false, + null, + null); + + private static final ModelCapabilitiesYaml CONSERVATIVE_DEFAULTS_YAML = + new ModelCapabilitiesYaml( + new ModelCapabilitiesYaml.InputModalities(List.of(Modality.TEXT), List.of(Modality.TEXT)), + new ModelCapabilitiesYaml.OutputModalities(List.of(Modality.TEXT)), + false, + false, + false, + false, + null, + null); + + private final CapabilityMatrix matrix; + private final ObjectMapper mapper; + private final JsonNode conservativeBase; + private final Set loggedKeys = ConcurrentHashMap.newKeySet(); + + public ModelCapabilitiesResolver(CapabilityMatrix matrix, ObjectMapper mapper) { + this.matrix = matrix; + this.mapper = mapper; + this.conservativeBase = mapper.valueToTree(CONSERVATIVE_DEFAULTS_YAML); + } + + public ModelCapabilities resolve( + String apiFamily, String modelId, Optional override) { + if (override.isPresent()) { + return override.get(); + } + + final ApiFamily family = matrix.families().get(apiFamily); + if (family == null) { + logOnce( + "missing-family:" + apiFamily, + "No capability matrix entry for api family '{}'; using conservative defaults", + apiFamily); + return CONSERVATIVE_DEFAULTS; + } + + final ModelEntry exact = findExact(family.models(), modelId); + if (exact != null) { + return merge(family.defaults(), exact.capabilities()); + } + + final MatchedPattern matched = findLongestPattern(family.models(), modelId); + if (matched != null) { + logOnce( + "pattern:" + apiFamily + ":" + modelId, + "Capability matrix pattern '{}' matched model '{}' (api family '{}')", + matched.pattern(), + modelId, + apiFamily); + return merge(family.defaults(), matched.entry().capabilities()); + } + + logOnce( + "default:" + apiFamily + ":" + modelId, + "No capability matrix entry for model '{}' under api family '{}'; using conservative defaults", + modelId, + apiFamily); + return CONSERVATIVE_DEFAULTS; + } + + @Nullable + private static ModelEntry findExact(List models, String modelId) { + for (ModelEntry entry : models) { + if (modelId.equals(entry.id()) || entry.aliases().contains(modelId)) { + return entry; + } + } + return null; + } + + @Nullable + private static MatchedPattern findLongestPattern(List models, String modelId) { + ModelEntry bestEntry = null; + String bestPattern = null; + int bestLength = -1; + for (ModelEntry entry : models) { + for (String pattern : entry.patterns()) { + if (matchesGlob(pattern, modelId) && pattern.length() > bestLength) { + bestEntry = entry; + bestPattern = pattern; + bestLength = pattern.length(); + } + } + } + return bestEntry == null ? null : new MatchedPattern(bestEntry, bestPattern); + } + + private record MatchedPattern(ModelEntry entry, String pattern) {} + + static boolean matchesGlob(String glob, String value) { + final String[] parts = glob.split("\\*", -1); + final StringBuilder regex = new StringBuilder("^"); + for (int i = 0; i < parts.length; i++) { + if (i > 0) { + regex.append(".*"); + } + regex.append(Pattern.quote(parts[i])); + } + regex.append('$'); + return Pattern.matches(regex.toString(), value); + } + + private ModelCapabilities merge( + @Nullable JsonNode familyDefaults, @Nullable JsonNode modelOverrides) { + final JsonNode merged = deepMerge(deepMerge(conservativeBase, familyDefaults), modelOverrides); + try { + return mapper.treeToValue(merged, ModelCapabilitiesYaml.class).toModelCapabilities(); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Failed to materialise model capabilities", e); + } + } + + /** + * Spring-Boot-style deep merge: object maps merge recursively, list and scalar values from the + * overlay replace the base verbatim. + */ + static JsonNode deepMerge(@Nullable JsonNode base, @Nullable JsonNode overlay) { + if (overlay == null || overlay.isNull() || overlay.isMissingNode()) { + return base != null ? base : NullNode.getInstance(); + } + if (base == null + || base.isNull() + || base.isMissingNode() + || !base.isObject() + || !overlay.isObject()) { + return overlay; + } + final ObjectNode merged = base.deepCopy(); + overlay + .fields() + .forEachRemaining( + entry -> + merged.set( + entry.getKey(), deepMerge(merged.get(entry.getKey()), entry.getValue()))); + return merged; + } + + private void logOnce(String dedupKey, String format, Object... args) { + if (loggedKeys.add(dedupKey)) { + LOG.info(format, args); + } + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesYaml.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesYaml.java new file mode 100644 index 00000000000..123c546d465 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesYaml.java @@ -0,0 +1,78 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.capabilities; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; +import java.util.List; +import org.springframework.lang.Nullable; + +/** + * Sparse capability block — bound by Spring Boot from {@code application.yml} (relaxed + * camel/snake-case binding) and serialised by Jackson to/from a {@link + * com.fasterxml.jackson.databind.JsonNode} tree (snake-case via {@link JsonNaming}, nulls preserved + * so the resolver's deep-merge can fall through to the base layer). + * + *

    Each field is nullable: a missing field means "inherit from the lower layer" (api-family + * {@code defaults} → conservative defaults). The fully-merged result is projected onto the flat + * {@link ModelCapabilities} SPI shape via {@link #toModelCapabilities()}. + */ +@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) +@JsonInclude(JsonInclude.Include.ALWAYS) // keep null fields visible to deep-merge +public record ModelCapabilitiesYaml( + @Nullable InputModalities inputModalities, + @Nullable OutputModalities outputModalities, + @Nullable Boolean supportsReasoning, + @Nullable Boolean supportsReasoningSignatureRoundtrip, + @Nullable Boolean supportsPromptCaching, + @Nullable Boolean supportsParallelToolCalls, + @Nullable Integer contextWindow, + @Nullable Integer maxOutputTokens) { + + @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) + @JsonInclude(JsonInclude.Include.ALWAYS) + public record InputModalities( + @Nullable List userMessage, @Nullable List toolResult) {} + + @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) + @JsonInclude(JsonInclude.Include.ALWAYS) + public record OutputModalities(@Nullable List assistantMessage) {} + + ModelCapabilities toModelCapabilities() { + return new ModelCapabilities( + userMessageModalities(), + toolResultModalities(), + assistantMessageModalities(), + Boolean.TRUE.equals(supportsReasoning), + Boolean.TRUE.equals(supportsReasoningSignatureRoundtrip), + Boolean.TRUE.equals(supportsPromptCaching), + Boolean.TRUE.equals(supportsParallelToolCalls), + contextWindow, + maxOutputTokens); + } + + private List userMessageModalities() { + return inputModalities != null && inputModalities.userMessage() != null + ? inputModalities.userMessage() + : List.of(Modality.TEXT); + } + + private List toolResultModalities() { + return inputModalities != null && inputModalities.toolResult() != null + ? inputModalities.toolResult() + : List.of(Modality.TEXT); + } + + private List assistantMessageModalities() { + return outputModalities != null && outputModalities.assistantMessage() != null + ? outputModalities.assistantMessage() + : List.of(Modality.TEXT); + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java index c0b2139790a..32ad93c0c13 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java @@ -45,6 +45,7 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatClient; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiRegistry; +import io.camunda.connector.agenticai.aiagent.framework.capabilities.AgenticAiCapabilitiesConfiguration; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelHttpProxySupport; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.configuration.AgenticAiLangchain4JFrameworkConfiguration; import io.camunda.connector.agenticai.aiagent.framework.openai.OpenAiChatModelApiConfiguration; @@ -84,6 +85,7 @@ @ConditionalOnBooleanProperty(value = "camunda.connector.agenticai.enabled", matchIfMissing = true) @EnableConfigurationProperties(AgenticAiConnectorsConfigurationProperties.class) @Import({ + AgenticAiCapabilitiesConfiguration.class, AnthropicMessagesApiConfiguration.class, OpenAiChatModelApiConfiguration.class, AgenticAiLangchain4JFrameworkConfiguration.class, diff --git a/connectors/agentic-ai/src/main/resources/META-INF/spring/org.springframework.boot.env.EnvironmentPostProcessor.imports b/connectors/agentic-ai/src/main/resources/META-INF/spring/org.springframework.boot.env.EnvironmentPostProcessor.imports new file mode 100644 index 00000000000..a03e83065b0 --- /dev/null +++ b/connectors/agentic-ai/src/main/resources/META-INF/spring/org.springframework.boot.env.EnvironmentPostProcessor.imports @@ -0,0 +1 @@ +io.camunda.connector.agenticai.aiagent.framework.capabilities.CapabilityMatrixEnvironmentPostProcessor diff --git a/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml b/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml new file mode 100644 index 00000000000..756625c920f --- /dev/null +++ b/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml @@ -0,0 +1,217 @@ +# Model capability matrix for the agentic-ai connector. +# +# This file is loaded as a Spring Boot PropertySource at startup (lowest +# precedence) by `CapabilityMatrixEnvironmentPostProcessor`. Library consumers +# can override or extend any value via `application.yml` using the same +# property prefix, no full-file replacement required: +# +# camunda.connector.agenticai.aiagent.framework.capabilities: +# anthropic-messages: +# models: +# claude-opus-4-7: # map key reused -> override existing entry +# capabilities: +# max-output-tokens: 64000 # only this field overridden +# +# Structure under `capabilities`: +# : +# defaults: capability block applied to every model entry in the family +# models: map of opaque identifiers to entries. Each entry has one of: +# - `id` (defaults to the map key when neither is set) +# - `pattern` (glob using `*` only; longest match wins at +# resolve-time) +# plus an optional `aliases` list (id entries only) and a +# `capabilities` overlay. +# NB: `*` and `.` in property keys interfere with Spring Boot's +# map binding, so glob characters always live in the +# `pattern` field, never in the map key. +# +# Merge semantics (Spring Boot config + ADR-004 capability matrix): +# * Maps merge recursively (sub-keys of input-modalities / output-modalities +# are inherited individually) +# * Lists replace wholesale +# * Scalars and booleans replace +# +# Resolution chain (per ADR-004 §"Resolution order"): +# 1. Connector config override (per-call, future) +# 2. Exact id or alias match +# 3. Pattern (longest-match wins) +# 4. Conservative defaults (text-only, all flags false) +# +# Modality vocabulary: text | image | pdf | audio | video. Modality lists per +# location (user-message, tool-result, assistant-message) are symmetric. + +camunda: + connector: + agenticai: + aiagent: + framework: + capabilities: + + anthropic-messages: + defaults: + input-modalities: + user-message: [text, image, pdf] + tool-result: [text, image] + output-modalities: + assistant-message: [text] + supports-reasoning: false + supports-reasoning-signature-roundtrip: false + supports-prompt-caching: true + supports-parallel-tool-calls: true + context-window: 200000 + max-output-tokens: 8192 + models: + claude-opus-4: + pattern: claude-opus-4-* + capabilities: + supports-reasoning: true + supports-reasoning-signature-roundtrip: true + max-output-tokens: 32000 + claude-sonnet-4: + pattern: claude-sonnet-4-* + capabilities: + supports-reasoning: true + supports-reasoning-signature-roundtrip: true + max-output-tokens: 64000 + claude-haiku-4: + pattern: claude-haiku-4-* + capabilities: + input-modalities: + # tool-result inherited from defaults + user-message: [text, image] + max-output-tokens: 16000 + claude-3-7-sonnet: + pattern: claude-3-7-sonnet-* + capabilities: + supports-reasoning: true + supports-reasoning-signature-roundtrip: true + max-output-tokens: 64000 + claude-3-5-sonnet: + pattern: claude-3-5-sonnet-* + capabilities: {} + claude-3-5-haiku: + pattern: claude-3-5-haiku-* + capabilities: + input-modalities: + user-message: [text] + tool-result: [text] + claude-fallback: + pattern: claude-* + capabilities: {} + + # Chat Completions tool messages are text-only (the SDK enforces + # ChatCompletionContentPartText for tool message content arrays); + # multimodal tool results require the Responses API. + openai-completions: + defaults: + input-modalities: + user-message: [text, image] + tool-result: [text] + output-modalities: + assistant-message: [text] + supports-reasoning: false + supports-reasoning-signature-roundtrip: false + supports-prompt-caching: true + supports-parallel-tool-calls: true + context-window: 128000 + max-output-tokens: 4096 + models: + gpt-5: + pattern: gpt-5* + capabilities: + supports-reasoning: true + context-window: 400000 + max-output-tokens: 128000 + o1: + pattern: o1* + capabilities: + supports-reasoning: true + supports-parallel-tool-calls: false + context-window: 200000 + max-output-tokens: 100000 + o3: + pattern: o3* + capabilities: + supports-reasoning: true + context-window: 200000 + max-output-tokens: 100000 + o4: + pattern: o4* + capabilities: + supports-reasoning: true + context-window: 200000 + max-output-tokens: 100000 + gpt-4o: + pattern: gpt-4o* + capabilities: + input-modalities: + user-message: [text, image, audio] + max-output-tokens: 16384 + gpt-4-1: + pattern: gpt-4.1* + capabilities: + context-window: 1000000 + max-output-tokens: 32768 + gpt-fallback: + pattern: gpt-* + capabilities: {} + + # Responses API: function call outputs accept text, image and file + # content items per the SDK's FunctionCallOutput.Output union. + openai-responses: + defaults: + input-modalities: + user-message: [text, image] + tool-result: [text, image, pdf] + output-modalities: + assistant-message: [text] + supports-reasoning: false + supports-reasoning-signature-roundtrip: false + supports-prompt-caching: true + supports-parallel-tool-calls: true + context-window: 128000 + max-output-tokens: 4096 + models: + gpt-5: + pattern: gpt-5* + capabilities: + supports-reasoning: true + supports-reasoning-signature-roundtrip: true + context-window: 400000 + max-output-tokens: 128000 + o1: + pattern: o1* + capabilities: + supports-reasoning: true + supports-reasoning-signature-roundtrip: true + supports-parallel-tool-calls: false + context-window: 200000 + max-output-tokens: 100000 + o3: + pattern: o3* + capabilities: + supports-reasoning: true + supports-reasoning-signature-roundtrip: true + context-window: 200000 + max-output-tokens: 100000 + o4: + pattern: o4* + capabilities: + supports-reasoning: true + supports-reasoning-signature-roundtrip: true + context-window: 200000 + max-output-tokens: 100000 + gpt-4o: + pattern: gpt-4o* + capabilities: + input-modalities: + user-message: [text, image, audio] + max-output-tokens: 16384 + gpt-4-1: + pattern: gpt-4.1* + capabilities: + context-window: 1000000 + max-output-tokens: 32768 + gpt-fallback: + pattern: gpt-* + capabilities: {} diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/BundledCapabilityMatrixTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/BundledCapabilityMatrixTest.java new file mode 100644 index 00000000000..ed6b1cabfce --- /dev/null +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/BundledCapabilityMatrixTest.java @@ -0,0 +1,181 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.capabilities; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; +import io.camunda.connector.agenticai.aiagent.framework.openai.OpenAiChatModelApiFactory; +import io.camunda.connector.runtime.annotation.ConnectorsObjectMapper; +import java.util.Optional; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Spring-Boot-style integration test for the bundled capability matrix. Runs the {@link + * CapabilityMatrixEnvironmentPostProcessor} against the test {@link + * org.springframework.context.ConfigurableApplicationContext} so the bundled YAML is loaded as a + * {@link org.springframework.core.env.PropertySource}, then exercises the full {@link + * AgenticAiFrameworkProperties} → {@link CapabilityMatrixFactory} → {@link + * ModelCapabilitiesResolver} pipeline. + */ +class BundledCapabilityMatrixTest { + + private final ApplicationContextRunner contextRunner = + new ApplicationContextRunner() + .withInitializer( + context -> + new CapabilityMatrixEnvironmentPostProcessor() + .postProcessEnvironment(context.getEnvironment(), null)) + .withUserConfiguration(TestObjectMapperConfig.class) + .withUserConfiguration(AgenticAiCapabilitiesConfiguration.class); + + @Test + void coversAllShippedApiFamilies() { + contextRunner.run( + context -> { + final var matrix = context.getBean(CapabilityMatrix.class); + assertThat(matrix.families()) + .containsKeys( + "anthropic-messages", + OpenAiChatModelApiFactory.API_FAMILY_COMPLETIONS, + OpenAiChatModelApiFactory.API_FAMILY_RESPONSES); + }); + } + + @Test + void claudeSonnet4ResolvesToFullCapabilities() { + contextRunner.run( + context -> { + final var caps = resolve(context, "anthropic-messages", "claude-sonnet-4-6"); + + assertThat(caps.userMessageModalities()) + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); + assertThat(caps.toolResultModalities()).containsExactly(Modality.TEXT, Modality.IMAGE); + assertThat(caps.supportsReasoning()).isTrue(); + assertThat(caps.supportsReasoningSignatureRoundtrip()).isTrue(); + assertThat(caps.supportsPromptCaching()).isTrue(); + assertThat(caps.supportsParallelToolCalls()).isTrue(); + assertThat(caps.maxOutputTokens()).isEqualTo(64000); + }); + } + + @Test + void claudeHaiku4InheritsToolResultButOverridesUserMessage() { + contextRunner.run( + context -> { + final var caps = resolve(context, "anthropic-messages", "claude-haiku-4-5"); + + assertThat(caps.userMessageModalities()).containsExactly(Modality.TEXT, Modality.IMAGE); + assertThat(caps.toolResultModalities()).containsExactly(Modality.TEXT, Modality.IMAGE); + assertThat(caps.supportsReasoning()).isFalse(); + }); + } + + @Test + void unknownClaudeModelFallsThroughToFamilyCatchAll() { + contextRunner.run( + context -> { + final var caps = resolve(context, "anthropic-messages", "claude-some-future-model"); + + assertThat(caps.userMessageModalities()) + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); + assertThat(caps.supportsPromptCaching()).isTrue(); + }); + } + + @Test + void gpt5OnCompletionsHasReasoningButNotRoundtrip() { + contextRunner.run( + context -> { + final var caps = + resolve(context, OpenAiChatModelApiFactory.API_FAMILY_COMPLETIONS, "gpt-5"); + + assertThat(caps.supportsReasoning()).isTrue(); + assertThat(caps.supportsReasoningSignatureRoundtrip()).isFalse(); + assertThat(caps.toolResultModalities()).containsExactly(Modality.TEXT); + assertThat(caps.contextWindow()).isEqualTo(400000); + }); + } + + @Test + void gpt5OnResponsesHasReasoningRoundtripAndMultimodalToolResults() { + contextRunner.run( + context -> { + final var caps = + resolve(context, OpenAiChatModelApiFactory.API_FAMILY_RESPONSES, "gpt-5"); + + assertThat(caps.supportsReasoning()).isTrue(); + assertThat(caps.supportsReasoningSignatureRoundtrip()).isTrue(); + assertThat(caps.toolResultModalities()) + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); + }); + } + + @Test + void gpt4oAddsAudioToUserMessageButKeepsToolResultFromDefaults() { + contextRunner.run( + context -> { + final var caps = + resolve(context, OpenAiChatModelApiFactory.API_FAMILY_RESPONSES, "gpt-4o-mini"); + + assertThat(caps.userMessageModalities()) + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.AUDIO); + assertThat(caps.toolResultModalities()) + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); + assertThat(caps.supportsReasoning()).isFalse(); + }); + } + + @Test + void o1OnCompletionsHasReasoningButNoParallelToolCalls() { + contextRunner.run( + context -> { + final var caps = + resolve(context, OpenAiChatModelApiFactory.API_FAMILY_COMPLETIONS, "o1-mini"); + + assertThat(caps.supportsReasoning()).isTrue(); + assertThat(caps.supportsParallelToolCalls()).isFalse(); + }); + } + + @Test + void gpt41OnResponsesHasLargeContextWindowDespiteDotInPattern() { + contextRunner.run( + context -> { + final var caps = + resolve(context, OpenAiChatModelApiFactory.API_FAMILY_RESPONSES, "gpt-4.1-mini"); + + assertThat(caps.contextWindow()).isEqualTo(1000000); + assertThat(caps.maxOutputTokens()).isEqualTo(32768); + }); + } + + private static ModelCapabilities resolve( + org.springframework.context.ApplicationContext context, String apiFamily, String modelId) { + return context + .getBean(ModelCapabilitiesResolver.class) + .resolve(apiFamily, modelId, Optional.empty()); + } + + /** + * Provides the {@link com.fasterxml.jackson.databind.ObjectMapper} bean that {@link + * AgenticAiCapabilitiesConfiguration} expects (qualified with {@link ConnectorsObjectMapper}). + */ + @Configuration + static class TestObjectMapperConfig { + @Bean + @ConnectorsObjectMapper + ObjectMapper objectMapper() { + return new ObjectMapper(); + } + } +} diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixOverrideTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixOverrideTest.java new file mode 100644 index 00000000000..67002e340a8 --- /dev/null +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixOverrideTest.java @@ -0,0 +1,115 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.capabilities; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; +import io.camunda.connector.agenticai.aiagent.framework.capabilities.BundledCapabilityMatrixTest.TestObjectMapperConfig; +import java.util.Optional; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; + +/** + * Verifies that consumer-supplied properties (higher precedence than the bundled YAML) deep-merge + * into the capability matrix. Uses {@link ApplicationContextRunner} with both the bundled {@link + * CapabilityMatrixEnvironmentPostProcessor} and {@code withPropertyValues(...)} overrides — exactly + * the layering a library consumer gets when adding entries to their {@code application.yml}. + */ +class CapabilityMatrixOverrideTest { + + private final ApplicationContextRunner baseRunner = + new ApplicationContextRunner() + .withInitializer( + context -> + new CapabilityMatrixEnvironmentPostProcessor() + .postProcessEnvironment(context.getEnvironment(), null)) + .withUserConfiguration(TestObjectMapperConfig.class) + .withUserConfiguration(AgenticAiCapabilitiesConfiguration.class); + + private static final String PREFIX = "camunda.connector.agenticai.aiagent.framework.capabilities"; + + @Test + void overrideTunesScalarFieldsOnExistingPattern() { + baseRunner + .withPropertyValues( + PREFIX + + ".anthropic-messages.models.claude-sonnet-4.capabilities.max-output-tokens=999999") + .run( + context -> { + final var caps = resolve(context, "anthropic-messages", "claude-sonnet-4-6"); + + assertThat(caps.maxOutputTokens()).isEqualTo(999999); + // Other bundled fields untouched: + assertThat(caps.supportsReasoning()).isTrue(); + assertThat(caps.userMessageModalities()) + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); + }); + } + + @Test + void overrideAddsBrandNewModelEntryUnderExistingFamily() { + baseRunner + .withPropertyValues( + PREFIX + + ".anthropic-messages.models.my-org-tuned-claude.capabilities.max-output-tokens=12345", + PREFIX + + ".anthropic-messages.models.my-org-tuned-claude.capabilities.supports-reasoning=true") + .run( + context -> { + final var caps = resolve(context, "anthropic-messages", "my-org-tuned-claude"); + + assertThat(caps.maxOutputTokens()).isEqualTo(12345); + assertThat(caps.supportsReasoning()).isTrue(); + // Inherited from anthropic-messages defaults: + assertThat(caps.userMessageModalities()) + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); + assertThat(caps.toolResultModalities()) + .containsExactly(Modality.TEXT, Modality.IMAGE); + }); + } + + @Test + void overrideReplacesModalityListWholesaleViaSpringBootSemantics() { + baseRunner + .withPropertyValues( + PREFIX + + ".openai-completions.models.gpt-4o.capabilities.input-modalities.user-message[0]=text") + .run( + context -> { + final var caps = resolve(context, "openai-completions", "gpt-4o-mini"); + + assertThat(caps.userMessageModalities()).containsExactly(Modality.TEXT); + // tool_result still inherited from openai-completions defaults: + assertThat(caps.toolResultModalities()).containsExactly(Modality.TEXT); + }); + } + + @Test + void overrideTunesFamilyDefaultsForUnpinnedModels() { + baseRunner + .withPropertyValues(PREFIX + ".openai-completions.defaults.max-output-tokens=7777") + .run( + context -> { + // Pinned entries (gpt-5*) keep their own override: + final var gpt5 = resolve(context, "openai-completions", "gpt-5"); + assertThat(gpt5.maxOutputTokens()).isEqualTo(128000); + + // Models that match only the gpt-* fallback inherit the new default: + final var generic = resolve(context, "openai-completions", "gpt-3.5-turbo"); + assertThat(generic.maxOutputTokens()).isEqualTo(7777); + }); + } + + private static ModelCapabilities resolve( + org.springframework.context.ApplicationContext context, String apiFamily, String modelId) { + return context + .getBean(ModelCapabilitiesResolver.class) + .resolve(apiFamily, modelId, Optional.empty()); + } +} diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesResolverTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesResolverTest.java new file mode 100644 index 00000000000..dfa853ec377 --- /dev/null +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesResolverTest.java @@ -0,0 +1,380 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.capabilities; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; +import io.camunda.connector.agenticai.aiagent.framework.capabilities.AgenticAiFrameworkProperties.ApiFamilyProperties; +import io.camunda.connector.agenticai.aiagent.framework.capabilities.AgenticAiFrameworkProperties.ModelEntryProperties; +import io.camunda.connector.agenticai.aiagent.framework.capabilities.ModelCapabilitiesYaml.InputModalities; +import io.camunda.connector.agenticai.aiagent.framework.capabilities.ModelCapabilitiesYaml.OutputModalities; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +class ModelCapabilitiesResolverTest { + + private final ObjectMapper objectMapper = new ObjectMapper(); + + // --- Builder helpers ----------------------------------------------------- + + private static AgenticAiFrameworkProperties props(Map families) { + return new AgenticAiFrameworkProperties(families); + } + + private static ApiFamilyProperties family( + ModelCapabilitiesYaml defaults, Map models) { + return new ApiFamilyProperties(defaults, models); + } + + private static ModelEntryProperties entry(ModelCapabilitiesYaml capabilities) { + return new ModelEntryProperties(null, null, List.of(), capabilities); + } + + private static ModelEntryProperties entryWithAliases( + List aliases, ModelCapabilitiesYaml capabilities) { + return new ModelEntryProperties(null, null, aliases, capabilities); + } + + /** A fully-populated capability block usable as an api-family default. */ + private static ModelCapabilitiesYaml fullDefaults( + List userMessage, List toolResult) { + return new ModelCapabilitiesYaml( + new InputModalities(userMessage, toolResult), + new OutputModalities(List.of(Modality.TEXT)), + false, + false, + true, + true, + 200000, + 8192); + } + + private ModelCapabilitiesResolver resolverFor(AgenticAiFrameworkProperties props) { + return new ModelCapabilitiesResolver( + CapabilityMatrixFactory.build(props, objectMapper), objectMapper); + } + + // --- Tests --------------------------------------------------------------- + + @Test + void overrideShortCircuitsResolution() { + final var resolver = resolverFor(props(Map.of())); + final var override = + new ModelCapabilities( + List.of(Modality.TEXT, Modality.IMAGE, Modality.AUDIO), + List.of(Modality.TEXT), + List.of(Modality.TEXT), + true, + true, + true, + true, + 12345, + 6789); + + final var result = resolver.resolve("anthropic-messages", "anything", Optional.of(override)); + + assertThat(result).isSameAs(override); + } + + @Test + void exactIdMatchInheritsDefaultsAndAppliesOverrides() { + // defaults: PDF in user_message, max_output_tokens=8192, no reasoning + // override on claude-opus-4-7: enable reasoning, max_output_tokens=32000 + final var defaults = + fullDefaults( + List.of(Modality.TEXT, Modality.IMAGE, Modality.PDF), + List.of(Modality.TEXT, Modality.IMAGE)); + final var override = new ModelCapabilitiesYaml(null, null, true, true, null, null, null, 32000); + + final var resolver = + resolverFor( + props( + Map.of( + "anthropic-messages", + family( + defaults, + Map.of( + "claude-opus-4-7", + entryWithAliases(List.of("claude-opus-latest"), override)))))); + + final var caps = resolver.resolve("anthropic-messages", "claude-opus-4-7", Optional.empty()); + + assertThat(caps.userMessageModalities()) + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); + assertThat(caps.toolResultModalities()).containsExactly(Modality.TEXT, Modality.IMAGE); + assertThat(caps.supportsReasoning()).isTrue(); + assertThat(caps.supportsReasoningSignatureRoundtrip()).isTrue(); + assertThat(caps.supportsPromptCaching()).isTrue(); + assertThat(caps.contextWindow()).isEqualTo(200000); + assertThat(caps.maxOutputTokens()).isEqualTo(32000); + } + + @Test + void aliasMatchResolvesToSameEntry() { + final var resolver = + resolverFor( + props( + Map.of( + "anthropic-messages", + family( + fullDefaults(List.of(Modality.TEXT), List.of(Modality.TEXT)), + Map.of( + "claude-opus-4-7", + entryWithAliases( + List.of("claude-opus-latest"), + new ModelCapabilitiesYaml( + null, null, null, null, null, null, null, 32000))))))); + + final var byAlias = + resolver.resolve("anthropic-messages", "claude-opus-latest", Optional.empty()); + final var byId = resolver.resolve("anthropic-messages", "claude-opus-4-7", Optional.empty()); + + assertThat(byAlias).isEqualTo(byId); + } + + @Test + void longestPatternWins() { + // claude-opus-* (12 chars) beats claude-* (8 chars). + final var models = new LinkedHashMap(); + models.put( + "claude-fallback", + new ModelEntryProperties( + null, + List.of("claude-*"), + List.of(), + new ModelCapabilitiesYaml(null, null, false, null, null, null, null, null))); + models.put( + "claude-opus", + new ModelEntryProperties( + null, + List.of("claude-opus-*"), + List.of(), + new ModelCapabilitiesYaml(null, null, true, null, null, null, null, null))); + + final var resolver = + resolverFor( + props( + Map.of( + "anthropic-messages", + family(fullDefaults(List.of(Modality.TEXT), List.of(Modality.TEXT)), models)))); + + final var caps = resolver.resolve("anthropic-messages", "claude-opus-3-5", Optional.empty()); + + assertThat(caps.supportsReasoning()).isTrue(); + } + + @Test + void deepMergeKeepsInheritedSubKeyWhenOverlayOverridesSibling() { + // Defaults provide tool_result=[text,image]; override only user_message. + final var defaults = + fullDefaults( + List.of(Modality.TEXT, Modality.IMAGE), List.of(Modality.TEXT, Modality.IMAGE)); + final var override = + new ModelCapabilitiesYaml( + new InputModalities(List.of(Modality.TEXT), null), + null, + null, + null, + null, + null, + null, + null); + + final var resolver = + resolverFor( + props( + Map.of( + "anthropic-messages", + family( + defaults, + Map.of( + "claude-haiku", + new ModelEntryProperties( + null, List.of("claude-haiku-*"), List.of(), override)))))); + + final var caps = resolver.resolve("anthropic-messages", "claude-haiku-4-5", Optional.empty()); + + assertThat(caps.userMessageModalities()).containsExactly(Modality.TEXT); + assertThat(caps.toolResultModalities()).containsExactly(Modality.TEXT, Modality.IMAGE); + } + + @Test + void unknownApiFamilyFallsThroughToConservativeDefaults() { + final var resolver = resolverFor(props(Map.of())); + + final var caps = resolver.resolve("does-not-exist", "claude-opus-4-7", Optional.empty()); + + assertThat(caps).isEqualTo(ModelCapabilitiesResolver.CONSERVATIVE_DEFAULTS); + } + + @Test + void unknownModelFallsThroughToConservativeDefaultsWhenNoFallbackPattern() { + final var resolver = + resolverFor( + props( + Map.of( + "anthropic-messages", + family( + fullDefaults( + List.of(Modality.TEXT, Modality.IMAGE), List.of(Modality.TEXT)), + Map.of("claude-opus-4-7", entry(emptyOverride())))))); + + final var caps = resolver.resolve("anthropic-messages", "claude-mystery", Optional.empty()); + + assertThat(caps).isEqualTo(ModelCapabilitiesResolver.CONSERVATIVE_DEFAULTS); + } + + @Test + void rejectsEntryWithBothExplicitIdAndPattern() { + final var props = + props( + Map.of( + "anthropic-messages", + family( + fullDefaults(List.of(Modality.TEXT), List.of(Modality.TEXT)), + Map.of( + "claude-opus-4-7", + new ModelEntryProperties( + "claude-opus-4-7", + List.of("claude-opus-*"), + List.of(), + emptyOverride()))))); + + assertThatThrownBy(() -> CapabilityMatrixFactory.build(props, objectMapper)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("at most one of"); + } + + @Test + void rejectsPatternEntryWithAliases() { + final var props = + props( + Map.of( + "anthropic-messages", + family( + fullDefaults(List.of(Modality.TEXT), List.of(Modality.TEXT)), + Map.of( + "claude-opus", + new ModelEntryProperties( + null, + List.of("claude-opus-*"), + List.of("some-alias"), + emptyOverride()))))); + + assertThatThrownBy(() -> CapabilityMatrixFactory.build(props, objectMapper)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("cannot declare aliases"); + } + + @Test + void mapKeyServesAsImplicitIdAndExactBeatsPattern() { + final var defaults = fullDefaults(List.of(Modality.TEXT), List.of(Modality.TEXT)); + final var models = new LinkedHashMap(); + // Map key is the id (no explicit id/pattern fields). + models.put( + "claude-opus-4-7", + entry(new ModelCapabilitiesYaml(null, null, null, null, null, null, null, 64000))); + // Pattern entry — explicit pattern field. + models.put( + "claude-opus", + new ModelEntryProperties( + null, + List.of("claude-opus-*"), + List.of(), + new ModelCapabilitiesYaml(null, null, null, null, null, null, null, 8192))); + + final var resolver = resolverFor(props(Map.of("anthropic-messages", family(defaults, models)))); + + final var byId = resolver.resolve("anthropic-messages", "claude-opus-4-7", Optional.empty()); + assertThat(byId.maxOutputTokens()).isEqualTo(64000); + + final var byPattern = + resolver.resolve("anthropic-messages", "claude-opus-3-5", Optional.empty()); + assertThat(byPattern.maxOutputTokens()).isEqualTo(8192); + } + + @Test + void multiplePatternsPerEntry() { + // pattern as a list — entry matches if any glob matches; longest matching glob's length + // determines the entry's score for cross-entry longest-match selection. + final var defaults = fullDefaults(List.of(Modality.TEXT), List.of(Modality.TEXT)); + final var entry = + new ModelEntryProperties( + null, + List.of("gpt-4o*", "gpt-4-turbo*"), + List.of(), + new ModelCapabilitiesYaml(null, null, null, null, null, null, null, 16384)); + + final var resolver = + resolverFor( + props(Map.of("openai-completions", family(defaults, Map.of("gpt-4o-family", entry))))); + + assertThat( + resolver + .resolve("openai-completions", "gpt-4o-mini", Optional.empty()) + .maxOutputTokens()) + .isEqualTo(16384); + assertThat( + resolver + .resolve("openai-completions", "gpt-4-turbo-2024-04-09", Optional.empty()) + .maxOutputTokens()) + .isEqualTo(16384); + } + + @Test + void deepMergeReplacesListsWholesale() { + final var defaults = + fullDefaults(List.of(Modality.TEXT, Modality.IMAGE), List.of(Modality.TEXT)); + final var override = + new ModelCapabilitiesYaml( + new InputModalities(List.of(Modality.TEXT), null), + null, + null, + null, + null, + null, + null, + null); + + final var resolver = + resolverFor( + props( + Map.of( + "openai-completions", + family( + defaults, + Map.of( + "gpt-4o", + new ModelEntryProperties( + null, List.of("gpt-4o*"), List.of(), override)))))); + + final var caps = resolver.resolve("openai-completions", "gpt-4o-mini", Optional.empty()); + + assertThat(caps.userMessageModalities()).containsExactly(Modality.TEXT); + } + + @Test + void globMatcherTreatsStarAsWildcard() { + assertThat(ModelCapabilitiesResolver.matchesGlob("claude-opus-*", "claude-opus-4-7")).isTrue(); + assertThat(ModelCapabilitiesResolver.matchesGlob("claude-opus-*", "claude-sonnet-4-7")) + .isFalse(); + assertThat(ModelCapabilitiesResolver.matchesGlob("gpt-*", "gpt-4o-mini")).isTrue(); + assertThat(ModelCapabilitiesResolver.matchesGlob("gpt-4.1*", "gpt-4.1-nano")).isTrue(); + assertThat(ModelCapabilitiesResolver.matchesGlob("gpt-4.1*", "gpt-4o")).isFalse(); + } + + private static ModelCapabilitiesYaml emptyOverride() { + return new ModelCapabilitiesYaml(null, null, null, null, null, null, null, null); + } +} From 390aa5b7810ab7d01c99c78950609c58ebbaf9e7 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 17:25:18 +0200 Subject: [PATCH 57/81] feat(agentic-ai): wire ChatModelApi.capabilities() through resolver (Phase E2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each native impl (Anthropic Messages, OpenAI Chat Completions, OpenAI Responses) now stores a ModelCapabilities passed in by its factory rather than a hardcoded conservative profile. The factories take a ModelCapabilitiesResolver dependency and resolve at create() time using the matching api family — for the OpenAI factory the family branches on configuration.apiFamily(); the openaiCompatible factory always resolves under openai-completions. Spring configurations inject the resolver bean exposed by Phase E1's AgenticAiCapabilitiesConfiguration. Impl unit tests now pass an explicit capabilities instance and assert pass-through; end-to-end wire-format tests confirm the EnvironmentPostProcessor + matrix bean wiring lights up at full Spring Boot startup. --- .../AnthropicMessagesApiConfiguration.java | 4 ++- .../AnthropicMessagesChatModelApi.java | 18 +++-------- .../AnthropicMessagesChatModelApiFactory.java | 9 +++++- .../OpenAiChatCompletionsChatModelApi.java | 18 +++-------- .../OpenAiChatModelApiConfiguration.java | 11 +++++-- .../openai/OpenAiChatModelApiFactory.java | 31 +++++++++++++++++-- .../OpenAiCompatibleChatModelApiFactory.java | 10 +++++- .../openai/OpenAiResponsesChatModelApi.java | 18 +++-------- .../AnthropicMessagesChatModelApiTest.java | 25 +++++++++------ ...OpenAiChatCompletionsChatModelApiTest.java | 27 ++++++++++------ .../OpenAiResponsesChatModelApiTest.java | 23 +++++++++++++- 11 files changed, 125 insertions(+), 69 deletions(-) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesApiConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesApiConfiguration.java index 259e9535fc9..85a5c8ef852 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesApiConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesApiConfiguration.java @@ -8,6 +8,7 @@ import com.anthropic.client.AnthropicClient; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; +import io.camunda.connector.agenticai.aiagent.framework.capabilities.ModelCapabilitiesResolver; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsConfigurationProperties; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; @@ -28,8 +29,9 @@ public class AnthropicMessagesApiConfiguration { @Bean(name = "langchain4JAnthropicChatModelApiFactory") @ConditionalOnMissingBean(name = "langchain4JAnthropicChatModelApiFactory") public ChatModelApiFactory anthropicMessagesChatModelApiFactory( + ModelCapabilitiesResolver capabilitiesResolver, AgenticAiConnectorsConfigurationProperties properties) { return new AnthropicMessagesChatModelApiFactory( - properties.aiagent().chatModel().api().defaultTimeout()); + capabilitiesResolver, properties.aiagent().chatModel().api().defaultTimeout()); } } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java index 319732231a5..cd3fd74283d 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java @@ -27,7 +27,6 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; -import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.agenticai.model.message.AssistantMessage; import io.camunda.connector.agenticai.model.message.AssistantMessageBuilder; @@ -68,20 +67,9 @@ public class AnthropicMessagesChatModelApi implements ChatModelApi { private static final long DEFAULT_MAX_TOKENS = 4096L; - private static final ModelCapabilities CAPABILITIES = - new ModelCapabilities( - List.of(Modality.TEXT), - List.of(Modality.TEXT), - List.of(Modality.TEXT), - false, - false, - false, - true, - null, - null); - private final AnthropicClient client; private final String model; + private final ModelCapabilities capabilities; @Nullable private final Long configuredMaxTokens; @Nullable private final Double temperature; @Nullable private final Double topP; @@ -90,12 +78,14 @@ public class AnthropicMessagesChatModelApi implements ChatModelApi { public AnthropicMessagesChatModelApi( AnthropicClient client, String model, + ModelCapabilities capabilities, @Nullable Long configuredMaxTokens, @Nullable Double temperature, @Nullable Double topP, @Nullable Long topK) { this.client = Objects.requireNonNull(client, "client"); this.model = Objects.requireNonNull(model, "model"); + this.capabilities = Objects.requireNonNull(capabilities, "capabilities"); this.configuredMaxTokens = configuredMaxTokens; this.temperature = temperature; this.topP = topP; @@ -104,7 +94,7 @@ public AnthropicMessagesChatModelApi( @Override public ModelCapabilities capabilities() { - return CAPABILITIES; + return capabilities; } @Override diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java index 6dc904b24a0..fb3fa68c6a4 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java @@ -10,6 +10,7 @@ import com.anthropic.client.okhttp.AnthropicOkHttpClient; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; +import io.camunda.connector.agenticai.aiagent.framework.capabilities.ModelCapabilitiesResolver; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicConnection; import java.time.Duration; @@ -31,9 +32,12 @@ public class AnthropicMessagesChatModelApiFactory public static final String API_FAMILY = "anthropic-messages"; + private final ModelCapabilitiesResolver capabilitiesResolver; @Nullable private final Duration defaultTimeout; - public AnthropicMessagesChatModelApiFactory(@Nullable Duration defaultTimeout) { + public AnthropicMessagesChatModelApiFactory( + ModelCapabilitiesResolver capabilitiesResolver, @Nullable Duration defaultTimeout) { + this.capabilitiesResolver = capabilitiesResolver; this.defaultTimeout = defaultTimeout; } @@ -57,9 +61,12 @@ public ChatModelApi create(AnthropicProviderConfiguration configuration) { final var connection = configuration.anthropic(); final var client = buildClient(connection); final var parameters = connection.model().parameters(); + final var capabilities = + capabilitiesResolver.resolve(API_FAMILY, connection.model().model(), Optional.empty()); return new AnthropicMessagesChatModelApi( client, connection.model().model(), + capabilities, parameters != null && parameters.maxTokens() != null ? parameters.maxTokens().longValue() : null, diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java index 64590d3f791..9ad4d366082 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java @@ -24,7 +24,6 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; -import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.agenticai.model.message.AssistantMessage; import io.camunda.connector.agenticai.model.message.StopReason; @@ -63,23 +62,12 @@ */ public class OpenAiChatCompletionsChatModelApi implements ChatModelApi { - private static final ModelCapabilities CAPABILITIES = - new ModelCapabilities( - List.of(Modality.TEXT), - List.of(Modality.TEXT), - List.of(Modality.TEXT), - false, - false, - false, - true, - null, - null); - private static final TypeReference> MAP_TYPE_REF = new TypeReference<>() {}; private final OpenAIClient client; private final String model; private final ObjectMapper objectMapper; + private final ModelCapabilities capabilities; @Nullable private final Long configuredMaxCompletionTokens; @Nullable private final Double temperature; @Nullable private final Double topP; @@ -88,12 +76,14 @@ public OpenAiChatCompletionsChatModelApi( OpenAIClient client, String model, ObjectMapper objectMapper, + ModelCapabilities capabilities, @Nullable Long configuredMaxCompletionTokens, @Nullable Double temperature, @Nullable Double topP) { this.client = Objects.requireNonNull(client, "client"); this.model = Objects.requireNonNull(model, "model"); this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper"); + this.capabilities = Objects.requireNonNull(capabilities, "capabilities"); this.configuredMaxCompletionTokens = configuredMaxCompletionTokens; this.temperature = temperature; this.topP = topP; @@ -101,7 +91,7 @@ public OpenAiChatCompletionsChatModelApi( @Override public ModelCapabilities capabilities() { - return CAPABILITIES; + return capabilities; } @Override diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiConfiguration.java index 6a24d1faa4a..bcfd1e49fbe 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiConfiguration.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.openai.client.OpenAIClient; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; +import io.camunda.connector.agenticai.aiagent.framework.capabilities.ModelCapabilitiesResolver; import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsConfigurationProperties; @@ -33,9 +34,12 @@ public class OpenAiChatModelApiConfiguration { @ConditionalOnMissingBean(name = "langchain4JOpenAiChatModelApiFactory") public ChatModelApiFactory openAiChatModelApiFactory( @ConnectorsObjectMapper ObjectMapper objectMapper, + ModelCapabilitiesResolver capabilitiesResolver, AgenticAiConnectorsConfigurationProperties properties) { return new OpenAiChatModelApiFactory( - objectMapper, properties.aiagent().chatModel().api().defaultTimeout()); + objectMapper, + capabilitiesResolver, + properties.aiagent().chatModel().api().defaultTimeout()); } @Bean(name = "langchain4JOpenAiCompatibleChatModelApiFactory") @@ -43,8 +47,11 @@ public ChatModelApiFactory openAiChatModelApiFactor public ChatModelApiFactory openAiCompatibleChatModelApiFactory( @ConnectorsObjectMapper ObjectMapper objectMapper, + ModelCapabilitiesResolver capabilitiesResolver, AgenticAiConnectorsConfigurationProperties properties) { return new OpenAiCompatibleChatModelApiFactory( - objectMapper, properties.aiagent().chatModel().api().defaultTimeout()); + objectMapper, + capabilitiesResolver, + properties.aiagent().chatModel().api().defaultTimeout()); } } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java index 8cf3efce630..181be912e94 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java @@ -11,6 +11,7 @@ import com.openai.client.okhttp.OpenAIOkHttpClient; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; +import io.camunda.connector.agenticai.aiagent.framework.capabilities.ModelCapabilitiesResolver; import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.ApiFamily; import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiConnection; @@ -34,10 +35,15 @@ public class OpenAiChatModelApiFactory implements ChatModelApiFactory> MAP_TYPE_REF = new TypeReference<>() {}; private final OpenAIClient client; private final String model; private final ObjectMapper objectMapper; + private final ModelCapabilities capabilities; @Nullable private final Long configuredMaxOutputTokens; @Nullable private final Double temperature; @Nullable private final Double topP; @@ -88,12 +76,14 @@ public OpenAiResponsesChatModelApi( OpenAIClient client, String model, ObjectMapper objectMapper, + ModelCapabilities capabilities, @Nullable Long configuredMaxOutputTokens, @Nullable Double temperature, @Nullable Double topP) { this.client = Objects.requireNonNull(client, "client"); this.model = Objects.requireNonNull(model, "model"); this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper"); + this.capabilities = Objects.requireNonNull(capabilities, "capabilities"); this.configuredMaxOutputTokens = configuredMaxOutputTokens; this.temperature = temperature; this.topP = topP; @@ -101,7 +91,7 @@ public OpenAiResponsesChatModelApi( @Override public ModelCapabilities capabilities() { - return CAPABILITIES; + return capabilities; } @Override diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java index 94a69884108..025be699f4e 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java @@ -56,6 +56,18 @@ class AnthropicMessagesChatModelApiTest { private static final String MODEL_ID = "claude-sonnet-4-6"; + private static final ModelCapabilities CAPABILITIES = + new ModelCapabilities( + List.of(Modality.TEXT, Modality.IMAGE, Modality.PDF), + List.of(Modality.TEXT, Modality.IMAGE), + List.of(Modality.TEXT), + true, + true, + true, + true, + 200000, + 64000); + @Mock private AnthropicClient client; @Mock private MessageService messageService; @@ -66,18 +78,13 @@ class AnthropicMessagesChatModelApiTest { @BeforeEach void setUp() { when(client.messages()).thenReturn(messageService); - api = new AnthropicMessagesChatModelApi(client, MODEL_ID, 1024L, null, null, null); + api = + new AnthropicMessagesChatModelApi(client, MODEL_ID, CAPABILITIES, 1024L, null, null, null); } @Test - void capabilitiesReturnsTextOnlyConservativeProfile() { - ModelCapabilities caps = api.capabilities(); - assertThat(caps.userMessageModalities()).containsExactly(Modality.TEXT); - assertThat(caps.toolResultModalities()).containsExactly(Modality.TEXT); - assertThat(caps.assistantMessageModalities()).containsExactly(Modality.TEXT); - assertThat(caps.supportsReasoning()).isFalse(); - assertThat(caps.supportsPromptCaching()).isFalse(); - assertThat(caps.supportsParallelToolCalls()).isTrue(); + void capabilitiesReturnsConfiguredInstance() { + assertThat(api.capabilities()).isSameAs(CAPABILITIES); } @Test diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApiTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApiTest.java index bbddc38c226..b2e93904791 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApiTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApiTest.java @@ -28,6 +28,7 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatOptions; import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; import io.camunda.connector.agenticai.model.tool.ToolCall; import io.camunda.connector.agenticai.model.tool.ToolCallResult; @@ -55,6 +56,18 @@ class OpenAiChatCompletionsChatModelApiTest { private static final String MODEL_ID = "gpt-4o"; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final ModelCapabilities CAPABILITIES = + new ModelCapabilities( + List.of(Modality.TEXT, Modality.IMAGE, Modality.AUDIO), + List.of(Modality.TEXT), + List.of(Modality.TEXT), + false, + false, + true, + true, + 128000, + 16384); + @Mock private OpenAIClient client; @Mock private ChatService chatService; @Mock private ChatCompletionService chatCompletionService; @@ -67,18 +80,14 @@ class OpenAiChatCompletionsChatModelApiTest { void setUp() { when(client.chat()).thenReturn(chatService); when(chatService.completions()).thenReturn(chatCompletionService); - api = new OpenAiChatCompletionsChatModelApi(client, MODEL_ID, OBJECT_MAPPER, 1024L, null, null); + api = + new OpenAiChatCompletionsChatModelApi( + client, MODEL_ID, OBJECT_MAPPER, CAPABILITIES, 1024L, null, null); } @Test - void capabilitiesAreTextOnly() { - var caps = api.capabilities(); - assertThat(caps.userMessageModalities()).containsExactly(Modality.TEXT); - assertThat(caps.toolResultModalities()).containsExactly(Modality.TEXT); - assertThat(caps.assistantMessageModalities()).containsExactly(Modality.TEXT); - assertThat(caps.supportsReasoning()).isFalse(); - assertThat(caps.supportsPromptCaching()).isFalse(); - assertThat(caps.supportsParallelToolCalls()).isTrue(); + void capabilitiesReturnsConfiguredInstance() { + assertThat(api.capabilities()).isSameAs(CAPABILITIES); } @Test diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApiTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApiTest.java index ed52f7a9cf7..b8b7fb67d70 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApiTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApiTest.java @@ -24,6 +24,8 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatOptions; import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; import io.camunda.connector.agenticai.model.tool.ToolCall; import io.camunda.connector.agenticai.model.tool.ToolCallResult; import io.camunda.connector.agenticai.model.tool.ToolDefinition; @@ -49,6 +51,18 @@ class OpenAiResponsesChatModelApiTest { private static final String MODEL_ID = "gpt-5"; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final ModelCapabilities CAPABILITIES = + new ModelCapabilities( + List.of(Modality.TEXT, Modality.IMAGE), + List.of(Modality.TEXT, Modality.IMAGE, Modality.PDF), + List.of(Modality.TEXT), + true, + true, + true, + true, + 400000, + 128000); + @Mock private OpenAIClient client; @Mock private ResponseService responseService; @@ -59,7 +73,14 @@ class OpenAiResponsesChatModelApiTest { @BeforeEach void setUp() { when(client.responses()).thenReturn(responseService); - api = new OpenAiResponsesChatModelApi(client, MODEL_ID, OBJECT_MAPPER, 1024L, null, null); + api = + new OpenAiResponsesChatModelApi( + client, MODEL_ID, OBJECT_MAPPER, CAPABILITIES, 1024L, null, null); + } + + @Test + void capabilitiesReturnsConfiguredInstance() { + assertThat(api.capabilities()).isSameAs(CAPABILITIES); } @Test From 0c7be3c48d51656a8d3224549dbb68510b5aba72 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 18:01:42 +0200 Subject: [PATCH 58/81] =?UTF-8?q?docs(agentic-ai):=20renumber=20ADR-004=20?= =?UTF-8?q?=E2=86=92=20ADR-005=20+=20refresh=20Phase=20E=20plan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The doc-branch ADR (`004-document-handling-in-tool-call-results.md`) was the original ADR-004 and is in active PR review. Our framework-replacement ADR moves to ADR-005 to avoid the collision after rebasing onto that branch. Renames: - `docs/adr/004-replace-langchain4j-framework.md` → `005-...` - `docs/adr-004-implementation-plan.md` → `adr-005-...` Also updates the Phase E section of the implementation plan and the capability-matrix portion of the ADR to reflect the as-built E1+E2 shape (Spring Boot config prefix, defaults+models map, pattern-as-list, longest-glob-score, deep-merge semantics) and outlines the E3/E4 design. Internal `ADR-004` references in javadoc, the bundled YAML header, `pom.xml`, and `ModelCapabilitiesResolver` are bumped to `ADR-005`. --- ...plan.md => adr-005-implementation-plan.md} | 208 ++++++++++++++---- ...d => 005-replace-langchain4j-framework.md} | 154 +++++++++---- connectors/agentic-ai/pom.xml | 2 +- .../aiagent/framework/api/CacheRetention.java | 2 +- .../aiagent/framework/api/ChatClient.java | 2 +- .../framework/api/ChatClientResult.java | 2 +- .../aiagent/framework/api/ChatModelApi.java | 2 +- .../framework/api/ChatModelApiFactory.java | 2 +- .../framework/api/ChatModelApiRegistry.java | 2 +- .../aiagent/framework/api/ChatOptions.java | 2 +- .../aiagent/framework/api/ChatRequest.java | 2 +- .../aiagent/framework/api/ChatResponse.java | 2 +- .../framework/api/ChatStreamListener.java | 2 +- .../framework/api/ModelCapabilities.java | 2 +- .../framework/api/ReasoningConfig.java | 2 +- .../framework/api/event/ChatModelEvent.java | 2 +- .../ModelCapabilitiesResolver.java | 2 +- .../capabilities/model-capabilities.yaml | 4 +- 18 files changed, 287 insertions(+), 109 deletions(-) rename connectors/agentic-ai/docs/{adr-004-implementation-plan.md => adr-005-implementation-plan.md} (69%) rename connectors/agentic-ai/docs/adr/{004-replace-langchain4j-framework.md => 005-replace-langchain4j-framework.md} (86%) diff --git a/connectors/agentic-ai/docs/adr-004-implementation-plan.md b/connectors/agentic-ai/docs/adr-005-implementation-plan.md similarity index 69% rename from connectors/agentic-ai/docs/adr-004-implementation-plan.md rename to connectors/agentic-ai/docs/adr-005-implementation-plan.md index f7524346f5f..4837cd807c5 100644 --- a/connectors/agentic-ai/docs/adr-004-implementation-plan.md +++ b/connectors/agentic-ai/docs/adr-005-implementation-plan.md @@ -1,8 +1,8 @@ -# ADR-004 Phase 1 — Incremental Implementation Plan +# ADR-005 Phase 1 — Incremental Implementation Plan ## Context -[ADR-004](connectors/agentic-ai/docs/adr/004-replace-langchain4j-framework.md) replaces the +[ADR-005](connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md) replaces the LangChain4j-backed `AiFrameworkAdapter` with a native provider layer over official vendor SDKs. The ADR names this "Phase 1 — one shipping unit," but it is large enough to review and merge in chunks. This plan breaks it into eight green-build checkpoints (Phases A–H) on the working branch. @@ -10,8 +10,12 @@ chunks. This plan breaks it into eight green-build checkpoints (Phases A–H) on **Plan revision (2026-05-07)** — re-ordered to ship two real native providers ahead of the capability/multimodality infrastructure. Validating the SPI against two divergent wire formats before generalising leads to a smaller, better-shaped abstraction in Phase E. Reasoning, prompt -caching, multimodal user/tool-result content, Azure OpenAI, Anthropic cloud backends, Google GenAI -and Bedrock-Converse are all explicitly **deferred** out of the first native cut. +caching, Azure OpenAI, Anthropic cloud backends, Google GenAI and Bedrock-Converse are all +explicitly **deferred** out of the first native cut. + +Phase E is split into four sub-phases that ship as independent commits (E1–E4 below). Phase E +is layered on top of PR #6999 (`agentic-ai-document-tool-call-results`); our branch was rebased +onto that PR while it was in active review. **Actual starting state** (`agentic-ai/custom-llm-layer`): - Phase 0 done: `AssistantMessage` gains `modelId`/`apiId`/`stopReason`/`usage`; `TokenUsage` @@ -21,9 +25,17 @@ and Bedrock-Converse are all explicitly **deferred** out of the first native cut `ChatClient`; LangChain4j wired as the bridge `ChatModelApi` for all six provider discriminators (one factory bean per discriminator). `AiFrameworkAdapter` and `AiFrameworkChatResponse` already removed from the source tree (ahead of the original Phase F schedule). -- Wire-format e2e regression tests added for Anthropic Messages API and OpenAI Responses API - (WireMock-based, currently exercising the bridge). -- ADR-004 document committed. +- **Phases B / C / D done**: native `AnthropicMessagesChatModelApi` (text-only), + `OpenAiChatCompletionsChatModelApi` (text-only), `OpenAiResponsesChatModelApi` + (text-only) plus the `apiFamily` switch on the `openai` discriminator and element template + bump 10 → 11. +- **Phase E1 / E2 done**: capability matrix loaded as Spring Boot config (bundled + `model-capabilities.yaml` registered via `EnvironmentPostProcessor`); each native impl now + consumes a `ModelCapabilities` resolved at factory time. +- Wire-format e2e regression tests added for the Anthropic Messages API, OpenAI Chat + Completions API and OpenAI Responses API (WireMock-based, currently exercising the native + impls). +- ADR-005 document committed. **End state**: `BaseAgentRequestHandler` calls `ChatClient`. Native `ChatModelApi` impls ship for Anthropic Messages (direct), OpenAI Chat Completions, OpenAI Responses, Azure OpenAI, Anthropic @@ -138,17 +150,19 @@ The first native cut deliberately ignores these — they re-enter in Phase E or | Topic | Re-enters | |-------|-----------| -| Multimodal user-message / tool-result content (image, PDF, audio, video) | E (capabilities + strategy) | -| Reasoning content (signed thinking blocks, encrypted reasoning items) | E | -| Prompt caching (`cache_control`, `prompt_cache_key`) | E | -| Capability matrix YAML + resolver | E | -| `ToolCallResultStrategy` (always inline-text in B–D) | E | +| Capability matrix YAML + resolver | E1 (done) | +| `ChatModelApi.capabilities()` resolved per call | E2 (done) | +| `ToolCallResultStrategy` (always inline-text in B–D) | E3 | +| Multimodal user-message / tool-result content — **image + PDF only** | E4 | +| Multimodal — audio / video | G+ (when matching native impls land) | +| Reasoning content (signed thinking blocks, encrypted reasoning items) | post-E (own sub-phase) | +| Prompt caching (`cache_control`, `prompt_cache_key`) | post-E (own sub-phase) | +| JDK `java.net.http.HttpClient` adapter for the Anthropic / OpenAI SDKs (replaces OkHttp transport) | post-E | | Azure OpenAI native impl | G | | Anthropic cloud backends (Bedrock / Vertex / Foundry) | G | | Google GenAI native impl | G | | Bedrock-Converse native impl (non-Anthropic models) | G | | `ProviderConfiguration` discriminator restructure + Jackson migration | F | -| JDK `java.net.http.HttpClient` adapter for the Anthropic / OpenAI SDKs (replaces OkHttp transport) | E (follow-up) | Under this scope each native impl returns a hardcoded `ModelCapabilities` (text-only, no reasoning, no caching, parallel tool calls true). `ChatOptions.cacheRetention` and @@ -267,43 +281,147 @@ pass against the native impl. --- -## Phase E — Capability matrix + tool-result strategy + multimodality + reasoning + caching - -**Goal**: re-enable the deferred features now that we have three real native impls and one bridge -to generalise from. +## Phase E — Capability matrix + tool-result strategy + multimodality + +**Plan revision (2026-05-07)** — Phase E is split into four sub-phases that ship as independent +commits. Reasoning and prompt caching are explicitly **deferred out of Phase E** to keep the cut +focused; the matrix already declares the flags so the slot is reserved. + +Phase E is layered on top of the work from PR #6999 (`agentic-ai-document-tool-call-results`), +which contributes the `ToolCallResultDocumentExtractor` (recursive walker over lists / maps / +MCP-shaped content), per-handler extraction hooks on `GatewayToolHandler`, the synthetic +`UserMessage` injection with `METADATA_TOOL_CALL_DOCUMENTS`, the XML document tags inserted in +tool result message text, and the window-count handling that excludes synthetic document +messages. Our branch was rebased onto that PR while it was in active review. + +### Sub-phase E1 — Capability matrix + resolver (done) + +Spring-Boot-native configuration: bundled YAML registered as a low-precedence +`PropertySource` via `EnvironmentPostProcessor`, library consumers override via their own +`application.yml` under the same prefix. + +**Configuration prefix**: `camunda.connector.agenticai.aiagent.framework.capabilities`. + +**Bundled YAML location**: `resources/capabilities/model-capabilities.yaml`. + +**Structure under `capabilities`** (each api family): +- `defaults`: capability block applied to every model entry in the family +- `models`: map of opaque identifiers → entries. Each entry has one of: + - explicit `id` (defaults to the map key when neither field is set), or + - explicit `pattern` — string OR list of strings, glob using `*` only. + Plus an optional `aliases` list (id entries only) and a `capabilities` overlay. + + Note: `*` and `.` cannot appear in the map key (Spring Boot's `MapBinder` strips them). + Pattern entries always declare the glob in the `pattern` field while the map key stays a + stable, override-friendly identifier. + +**Merge semantics** (Spring Boot config + ADR-005 capability matrix): +- Maps merge recursively (sub-keys of `input-modalities` / `output-modalities` are inherited + individually) +- Lists replace wholesale +- Scalars and booleans replace + +**Resolution chain**: +1. Connector config override (per-call, future hook on `ChatOptions`) +2. Exact id or alias match +3. Pattern (longest matching glob across entries; entry score = longest matching glob in its + pattern list) +4. Conservative defaults (text-only, all flags false) + +**Files added** (`framework/capabilities/`): +- `AgenticAiFrameworkProperties` — `@ConfigurationProperties` record (sparse fields) +- `ModelCapabilitiesYaml` — sparse capability block bound by Spring Boot, projected onto + `ModelCapabilities` after deep-merge +- `CapabilityMatrixEnvironmentPostProcessor` — registers the bundled YAML at lowest precedence +- `CapabilityMatrixFactory` — derives `id`/`pattern` from the entry shape, validates entries, + converts capability sub-trees to `JsonNode` for the resolver +- `CapabilityMatrix` — built matrix used by the resolver +- `ModelCapabilitiesResolver` — 4-step chain with INFO logs on pattern / default fall-throughs +- `AgenticAiCapabilitiesConfiguration` — Spring config wiring + +**Tests added**: `ModelCapabilitiesResolverTest` (13 unit cases), +`BundledCapabilityMatrixTest` (Spring `ApplicationContextRunner` integration, 9 cases), +`CapabilityMatrixOverrideTest` (override deep-merge via `withPropertyValues`, 4 cases). + +### Sub-phase E2 — Wire `ChatModelApi.capabilities()` through the resolver (done) + +Each native impl (`AnthropicMessagesChatModelApi`, +`OpenAiChatCompletionsChatModelApi`, `OpenAiResponsesChatModelApi`) accepts a +`ModelCapabilities` via constructor instead of holding a hardcoded conservative profile. The +factories (`AnthropicMessagesChatModelApiFactory`, `OpenAiChatModelApiFactory`, +`OpenAiCompatibleChatModelApiFactory`) take a `ModelCapabilitiesResolver` dependency and resolve +at `create()` time. The OpenAI native factory branches on `OpenAiConnection.apiFamily()` for the +resolver lookup (`openai-completions` vs. `openai-responses`); the openaiCompatible factory +always resolves under `openai-completions`. The L4J bridge keeps its own conservative defaults +(it stays as a fallback path; replacing it through the resolver is not worth the change at this +point). + +### Sub-phase E3 — `ToolCallResultStrategy` + +**Goal**: per-block routing of tool result content. Wired into `ChatClientImpl` so the impls +receive a pre-routed request. **Files to create**: -- `resources/capabilities/model-capabilities.yaml` — populated entries for Anthropic Claude - families, OpenAI Chat Completions models, OpenAI Responses models. Schema per ADR §"Capability - Matrix". -- `ModelCapabilitiesResolver` — 4-step resolution chain: connector override → exact id/alias → - glob (longest match) → conservative defaults. INFO log on pattern or default use. - `ToolCallResultStrategy` — pure function `apply(ToolCallResult, ModelCapabilities)` returning - per-block routing decision (inline `contentBlocks` vs. synthetic `UserMessage` fallback through - the existing `AgentMessagesHandlerImpl` path from PR #6999). + a routing decision per content block. Decision is one of: + - `INLINE` — keep the block in `ToolCallResult.contentBlocks`; the native impl emits it as + provider-native multimodal content (E4) + - `EXTRACT_TO_USER_MESSAGE` — delegate to the doc-branch's + `ToolCallResultDocumentExtractor` synthesis path. The block is replaced in the tool result + body by an XML document tag; the original `DocumentContent` is appended to a synthetic + `UserMessage` with `METADATA_TOOL_CALL_DOCUMENTS`. **Files to modify**: -- Each native `ChatModelApi`: replace the hardcoded `ModelCapabilities` with the resolved one; - start emitting/consuming the deferred features: - - **Anthropic**: extended thinking blocks (signed reasoning roundtrip), `cache_control` markers - on the last system / message / tool-definition block, image/PDF user content, image tool-result - content. - - **OpenAI Responses**: encrypted reasoning items roundtrip, `prompt_cache_key`, image input. - - **OpenAI Completions**: `prompt_cache_key`, image input. - - **L4J bridge**: keeps conservative defaults; cache → NONE. -- `ChatClientImpl`: default `ChatOptions.cacheRetention` to `SHORT` (clamped to `NONE` when - `!supportsPromptCaching()`); query `capabilities()` once per call; apply - `ToolCallResultStrategy`. +- `ChatClientImpl` — query `capabilities()`, apply `ToolCallResultStrategy` on tool-result + messages before the impl call +- `AgentMessagesHandlerImpl` — adjust the existing PR #6999 synthesis call site so blocks + marked `INLINE` skip extraction **Tests to add**: -- `ModelCapabilitiesResolverTest` — all 4 steps, alias match, glob longest-match. -- `ToolCallResultStrategyTest` — table-driven across modality × capability combos. -- YAML round-trip test (Jackson reads the resource, validates required fields). -- Per-impl: extend tests with multimodal / reasoning / cache cases. - -**Verification**: `mvn clean test -pl connectors/agentic-ai`; both wire-format e2e tests still -green; new multimodal / reasoning / cache cases added under `wireformat/` exercise the new code -paths. +- `ToolCallResultStrategyTest` — table-driven across modality × capability combos +- `ChatClientImplTest` — extend with strategy wiring case + +### Sub-phase E4 — Multimodality in native impls (image + PDF only) + +**Scope** matches L4J parity (`DocumentToContentConverterImpl` supports text + image + PDF). +Audio and video are **out of scope for E4**; the capability matrix slot remains for Phase G+. + +**Modality detection**: shared utility (or per-impl) maps Camunda +`Document.metadata().contentType()` (MIME) → `ModelCapabilities.Modality`. Sealed Content +hierarchy stays as-is — no new `ImageContent` / `PdfContent` subtypes; dispatch is MIME-based +at conversion time. + +**Per-impl native encoding**: +- **Anthropic Messages**: `ContentBlockParam.ofImage(...)` (base64 data + media_type), + `ContentBlockParam.ofDocument(...)` for PDFs. Tool result and user message both supported. +- **OpenAI Chat Completions**: `ChatCompletionContentPart.ofImageUrl(...)` with data URL on + user messages. Tool messages are text-only (SDK enforces + `ChatCompletionContentPartText`-only for tool message content arrays); any image block in a + tool result must route through E3's `EXTRACT_TO_USER_MESSAGE`. +- **OpenAI Responses**: `ResponseInputContent.ofInputImage(...)` (base64) and + `ofInputFile(...)` (PDF base64). Multimodal tool results via + `ResponseInputItem.FunctionCallOutput.Output.ofResponseFunctionCallOutputItemList(...)`. + +**User-message capability mismatch** (image in user prompt for a text-only model): fail loud +with `ConnectorException(ERROR_CODE_FAILED_MODEL_CALL, ...)` and a clear message. Mirrors L4J's +`DocumentConversionException` semantics for unsupported types. + +**Tests to add**: per-impl multimodal cases, including a wire-format e2e per provider that +exercises the inline-native path; existing PR #6999 e2e covers the fallback path. + +### Deferred out of Phase E + +- **Reasoning** (extended thinking blocks, signed reasoning roundtrip, encrypted reasoning + items): kept on the matrix flag list but no impl work in E. Phase F or its own sub-phase. +- **Prompt caching** (`cache_control` markers, `prompt_cache_key`): same. +- **JDK `java.net.http.HttpClient` adapter** (replacing OkHttp): same — purely a transport + swap, no behavioural change. +- **Audio + video modalities**: Phase G+ when the corresponding native impl lands (gpt-4o-audio + on Completions, Gemini for video). + +**Verification**: `mvn clean test -pl connectors/agentic-ai` after each sub-phase; the three +wire-format e2e tests stay green; new multimodal cases under `wireformat/` exercise the +inline-native path in E4. --- @@ -386,7 +504,7 @@ green. by default. Document in release notes / `docs/reference/ai-agent.md`. - `AgenticAiConnectorsAutoConfiguration`: drop `AgenticAiLangchain4JFrameworkConfiguration` default import. -- ADR-004 status: Proposed → Implemented (final date set when shipping). +- ADR-005 status: Proposed → Implemented (final date set when shipping). - `docs/reference/ai-agent.md` + `AGENTS.md` (agentic-ai): `ChatModelApi` is the framework; LangChain4j bridge is legacy opt-in. @@ -417,7 +535,7 @@ mvn test -pl connectors-e2e-test/connectors-e2e-test-agentic-ai # full suite ( | `element-templates/agenticai-aiagent-outbound-connector.json` | D (v11), F (v12) | | `element-templates/README.md` | D, F | | `AgenticAiConnectorsAutoConfiguration.java` | A (done), B–G (provider imports) | -| `docs/adr/004-replace-langchain4j-framework.md` | H (status update) | +| `docs/adr/005-replace-langchain4j-framework.md` | H (status update) | ## Reusable existing code diff --git a/connectors/agentic-ai/docs/adr/004-replace-langchain4j-framework.md b/connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md similarity index 86% rename from connectors/agentic-ai/docs/adr/004-replace-langchain4j-framework.md rename to connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md index d70f2645bfe..dad6f75d192 100644 --- a/connectors/agentic-ai/docs/adr/004-replace-langchain4j-framework.md +++ b/connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md @@ -285,74 +285,134 @@ for provider-specific data when needed. ## Capability Matrix -A YAML resource on the classpath records capabilities per supported model. The schema is -oriented around what the runtime needs to decide: +The capability matrix ships as a Spring Boot configuration — a bundled YAML resource is +registered as a low-precedence `PropertySource` by an `EnvironmentPostProcessor` at startup, +and library consumers override or extend any value via their own `application.yml` under the +same prefix. No bespoke YAML parser; standard Spring property binding handles everything. + +**Configuration prefix**: `camunda.connector.agenticai.aiagent.framework.capabilities`. + +**Bundled resource**: `classpath:capabilities/model-capabilities.yaml`. + +**Structure** (per api family): ```yaml -anthropic-messages: - models: - - id: claude-opus-4-7 - aliases: [claude-opus-latest] - capabilities: - input_modalities: - user_message: [text, image, pdf] - tool_result: [text, image] - output_modalities: - assistant_message: [text] - supports_reasoning: true - supports_reasoning_signature_roundtrip: true - supports_prompt_caching: true - supports_parallel_tool_calls: true - context_window: 200000 - max_output_tokens: 64000 - - pattern: claude-opus-* - capabilities: { ... } # best-effort family default - - pattern: claude-haiku-* - capabilities: - supports_reasoning: false # haiku family does not yet have extended thinking - ... +camunda.connector.agenticai.aiagent.framework.capabilities: + anthropic-messages: + defaults: + input-modalities: + user-message: [text, image, pdf] + tool-result: [text, image] + output-modalities: + assistant-message: [text] + supports-reasoning: false + supports-reasoning-signature-roundtrip: false + supports-prompt-caching: true + supports-parallel-tool-calls: true + context-window: 200000 + max-output-tokens: 8192 + models: + claude-opus-4-7: # map key is the id (no `pattern` field) + aliases: [claude-opus-latest] + capabilities: + supports-reasoning: true + supports-reasoning-signature-roundtrip: true + max-output-tokens: 32000 + claude-opus-4: # map key is opaque; `pattern` carries the glob + pattern: claude-opus-4-* + capabilities: + supports-reasoning: true + max-output-tokens: 32000 + claude-haiku: + pattern: [claude-haiku-4-*, claude-haiku-3-*] + capabilities: {} ``` +Each api family carries: +- `defaults`: capability block applied to every entry in the family (deep-merged with + per-entry overlays at resolve time) +- `models`: map of opaque identifiers → entries. Each entry has one of: + - explicit `id` (defaults to the map key when neither field is set), or + - explicit `pattern` — string OR list of strings, glob using `*` only. + Plus optional `aliases` (id entries only) and a `capabilities` overlay. + +**Map keys cannot contain `*` or `.`**: Spring Boot's `MapBinder` strips these characters, +so glob patterns always live in the `pattern` field while the map key stays a stable +override identifier. + Modality vocabulary: `text | image | pdf | audio | video`. Modality lists per location -(`user_message`, `tool_result`, `assistant_message`) are symmetric — every modality at every +(`user-message`, `tool-result`, `assistant-message`) are symmetric — every modality at every location has an explicit answer for each model. +### Merge semantics + +Spring Boot config defaults: maps merge recursively (sub-keys of `input-modalities` and +`output-modalities` are inherited individually), lists replace wholesale, scalars and +booleans replace. The same rules apply both within the bundled YAML (per-entry `capabilities` +on top of `defaults`) and across PropertySources (consumer `application.yml` on top of +bundled defaults). + ### Resolution order Most-specific-first, scoped to the api family of the connector configuration: -1. **Connector config override** — user-declared `modelCapabilities` block on the provider - configuration always wins. -2. **Exact id or alias match** — the `id` field of an entry, or any string in its `aliases` - list, equals the requested model id. -3. **Pattern match** — `pattern` (glob with `*` only) matches the requested model id; - longest-matching pattern wins. +1. **Connector config override** — per-call `Optional` passed into + `ModelCapabilitiesResolver.resolve(...)`. Reserved hook; not yet wired from + `ChatOptions`. +2. **Exact id or alias match** — the `id` field of an entry (or its derived map key), + or any string in its `aliases` list, equals the requested model id. +3. **Pattern match** — any glob in the entry's `pattern` field matches the requested model + id; the entry's score is the length of the longest matching glob, and longest score wins + across entries. 4. **Conservative defaults** — text-only across the board, all `supports_*` flags `false`, numeric limits null. Aliases resolve at step 2 directly (no pre-rewriting); patterns at step 3 match against -the original requested id. Resolution at steps 3 or 4 logs an INFO message on first use so -operators notice they are running on best-effort or default capabilities. Resolution at -step 2 is silent — alias mappings are verified declarations. +the original requested id. Resolution at steps 3 or 4 logs an INFO message once per +(api family, model id) so operators notice they are running on best-effort or default +capabilities. Resolution at step 2 is silent — alias mappings are verified declarations. ### Conservative defaults for unknown models ```yaml -input_modalities: - user_message: [text] - tool_result: [text] -output_modalities: - assistant_message: [text] -supports_reasoning: false -supports_reasoning_signature_roundtrip: false -supports_prompt_caching: false -supports_parallel_tool_calls: false -context_window: null -max_output_tokens: null +input-modalities: + user-message: [text] + tool-result: [text] +output-modalities: + assistant-message: [text] +supports-reasoning: false +supports-reasoning-signature-roundtrip: false +supports-prompt-caching: false +supports-parallel-tool-calls: false +context-window: null +max-output-tokens: null +``` + +Unknown api families fall through to the conservative defaults at lookup time (the +resolver has no entry to match) and emit an INFO log so the operator notices. + +### Library-consumer overrides + +Consumers override or extend any value by declaring properties under the same prefix in +their `application.yml`: + +```yaml +camunda.connector.agenticai.aiagent.framework.capabilities: + anthropic-messages: + models: + claude-opus-4-7: + capabilities: + max-output-tokens: 64000 # tune existing entry + my-org-tuned-claude: # add a new entry + capabilities: + supports-reasoning: true + max-output-tokens: 12345 ``` -Unknown api families fail at validation, not at runtime — they have no factory bean to -resolve, so requests can never start. +Map-key reuse means a consumer override deep-merges into the bundled entry; a new map key +adds a new entry. Modality lists replace wholesale (Spring Boot list semantics) — overriding +`tool-result: [text]` discards the bundled `[text, image]`. To add a modality, restate the +full list including the inherited entries. ## Tool Call Result Routing diff --git a/connectors/agentic-ai/pom.xml b/connectors/agentic-ai/pom.xml index c5c5b0fabbe..6ce947240e6 100644 --- a/connectors/agentic-ai/pom.xml +++ b/connectors/agentic-ai/pom.xml @@ -134,7 +134,7 @@ langchain4j-http-client-jdk - + com.anthropic anthropic-java-core diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/CacheRetention.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/CacheRetention.java index b4411c8c065..969ad342247 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/CacheRetention.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/CacheRetention.java @@ -13,7 +13,7 @@ * retention; {@code NONE} strips cache markers entirely. Concrete breakpoint placement is * implementation-specific. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + *

    Part of the ADR-005 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via * ChatModelApiRegistry. */ public enum CacheRetention { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClient.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClient.java index 42710259a9d..69bf174a652 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClient.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClient.java @@ -21,7 +21,7 @@ * see a synchronous facade matching the previous {@code AiFrameworkAdapter} contract. In-process * observability hooks attach via {@link ChatStreamListener}; the public surface is blocking. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + *

    Part of the ADR-005 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via * ChatModelApiRegistry. */ public interface ChatClient { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClientResult.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClientResult.java index 8f0fae783de..1a45f64f83f 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClientResult.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatClientResult.java @@ -15,7 +15,7 @@ * ChatModelApi}. Replaces {@code AiFrameworkChatResponse} at the {@code BaseAgentRequestHandler} * call site so the cutover stays a 1:1 swap. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + *

    Part of the ADR-005 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via * ChatModelApiRegistry. */ public record ChatClientResult(AgentContext agentContext, AssistantMessage assistantMessage) {} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApi.java index 6330ae735bb..6677e42b566 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApi.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApi.java @@ -17,7 +17,7 @@ * {@link CompletableFuture} surface to callers. The optional {@link ChatStreamListener} receives * discriminated stream events for in-process observability. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + *

    Part of the ADR-005 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via * ChatModelApiRegistry. */ public interface ChatModelApi { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java index 4b297439dae..fc06dfce386 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiFactory.java @@ -23,7 +23,7 @@ * unchecked cast — a friendlier error than {@link ClassCastException} when a factory is * accidentally registered against the wrong discriminator. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + *

    Part of the ADR-005 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via * ChatModelApiRegistry. * * @param the {@link ProviderConfiguration} subtype this factory handles diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java index 041d5f06552..b56fc4babf4 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatModelApiRegistry.java @@ -17,7 +17,7 @@ *

    Unknown provider types fail fast — there is no factory to resolve, so requests can never * start. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + *

    Part of the ADR-005 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via * ChatModelApiRegistry. */ public interface ChatModelApiRegistry { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java index 92a46e6710c..7711d1f3c19 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatOptions.java @@ -28,7 +28,7 @@ * caller-supplied value wins, otherwise the resolved {@link ModelCapabilities#maxOutputTokens()} is * used as a fallback, otherwise the implementation supplies its own per-API default. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + *

    Part of the ADR-005 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via * ChatModelApiRegistry. */ public record ChatOptions( diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java index 878954e0759..d5c7b8fb0b6 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatRequest.java @@ -23,7 +23,7 @@ * native shape; providers without a native structured-output mode (Anthropic Messages today) treat * the JSON variant as best-effort and rely on the system prompt to constrain output. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + *

    Part of the ADR-005 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via * ChatModelApiRegistry. */ public record ChatRequest( diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatResponse.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatResponse.java index eec24bfa9ee..4b11e73d2a6 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatResponse.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatResponse.java @@ -15,7 +15,7 @@ * tool-use) populate the message with {@code stopReason = ERROR}; transport / SDK / auth failures * complete the future exceptionally instead. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + *

    Part of the ADR-005 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via * ChatModelApiRegistry. */ public record ChatResponse(AssistantMessage assistantMessage) {} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatStreamListener.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatStreamListener.java index 589a612617f..a6655e18a05 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatStreamListener.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ChatStreamListener.java @@ -14,7 +14,7 @@ * listener is intentionally not exposed as a reactive type — the public chat surface remains a * blocking {@code CompletableFuture}. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + *

    Part of the ADR-005 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via * ChatModelApiRegistry. */ @FunctionalInterface diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java index 1a8d35d7130..f58e37e0ab7 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java @@ -16,7 +16,7 @@ * negotiation, and cache-marker placement. The vocabulary for {@link Modality} is fixed; modality * lists per location are symmetric so every location has an explicit answer. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + *

    Part of the ADR-005 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via * ChatModelApiRegistry. */ public record ModelCapabilities( diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ReasoningConfig.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ReasoningConfig.java index 3134a85ff7c..4436220771e 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ReasoningConfig.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ReasoningConfig.java @@ -12,7 +12,7 @@ * this onto provider-native fields (Anthropic adaptive effort / thinking budget, OpenAI Responses * reasoning effort, Gemini thinking budget, etc.). * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + *

    Part of the ADR-005 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via * ChatModelApiRegistry. */ public sealed interface ReasoningConfig diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/event/ChatModelEvent.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/event/ChatModelEvent.java index 9016529a1ad..4a01dd08f87 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/event/ChatModelEvent.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/event/ChatModelEvent.java @@ -16,7 +16,7 @@ * the assembled {@link ChatResponse}, while {@link ErrorEvent} carries the error message plus any * partial content / usage accumulated before the failure. * - *

    Part of the ADR-004 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via + *

    Part of the ADR-005 Phase 1 SPI scaffolding. Wired by ChatClientImpl, dispatched via * ChatModelApiRegistry. */ public sealed interface ChatModelEvent diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesResolver.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesResolver.java index 74b8b8e2535..83b6f1ff95d 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesResolver.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesResolver.java @@ -26,7 +26,7 @@ /** * Resolves a runtime {@link ModelCapabilities} from the capability matrix using the four-step chain - * defined in ADR-004: + * defined in ADR-005: * *

      *
    1. Connector config override (if present) diff --git a/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml b/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml index 756625c920f..c66f1592cb6 100644 --- a/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml +++ b/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml @@ -25,13 +25,13 @@ # map binding, so glob characters always live in the # `pattern` field, never in the map key. # -# Merge semantics (Spring Boot config + ADR-004 capability matrix): +# Merge semantics (Spring Boot config + ADR-005 capability matrix): # * Maps merge recursively (sub-keys of input-modalities / output-modalities # are inherited individually) # * Lists replace wholesale # * Scalars and booleans replace # -# Resolution chain (per ADR-004 §"Resolution order"): +# Resolution chain (per ADR-005 §"Resolution order"): # 1. Connector config override (per-call, future) # 2. Exact id or alias match # 3. Pattern (longest-match wins) From e960aa02acdb80bba9aba9f40b4f4bad7f8d9ae9 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 18:53:13 +0200 Subject: [PATCH 59/81] docs(agentic-ai): finalize E3 ToolCallResultStrategy design MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refresh ADR-005 §"Tool Call Result Routing" and the Phase E3 section of the implementation plan with the agreed design: - single decision point at the ChatClient SPI boundary; the strategy is a pure function `apply(ChatRequest, ModelCapabilities) → (ChatRequest, List)` that walks the request once and routes each document it finds — no extract-then-restore double pass - tool-result-message documents are routed against `capabilities.toolResultModalities()`: inline-supported docs stay on `ToolCallResult.contentBlocks`; the rest fall back to a synthetic `UserMessage` (existing PR #6999 shape, `METADATA_TOOL_CALL_DOCUMENTS`) - user-message and event-message documents are validated against `capabilities.userMessageModalities()`: supported docs stay inline, unsupported docs fail loud (`ConnectorException`) - synthetic UMs land in `RuntimeMemory` inside `ChatClient.chat(...)` so the persisted `agentContext.conversation` matches the wire exactly — replay across iterations stays deterministic - `AgentMessagesHandlerImpl` drops the `documentExtractor` field, the `createDocumentMessageForToolResults` private method, and the line-134 call site — strategy owns extraction - TODO captured: revisit `ChatClient` ↔ `BaseAgentRequestHandler` boundary post-Phase E; ChatClient now owns three responsibilities Also bumps the bundled `anthropic-messages` `tool-result` modalities from `[text, image]` to `[text, image, pdf]` — Anthropic's `ToolResultBlockParam.Content.Block.ofDocument(...)` SDK factory confirms PDF support in tool results. `BundledCapabilityMatrixTest` adjusted to match. --- .../docs/adr-005-implementation-plan.md | 97 +++++++++++++---- .../adr/005-replace-langchain4j-framework.md | 100 +++++++++++++++--- .../capabilities/model-capabilities.yaml | 2 +- .../BundledCapabilityMatrixTest.java | 6 +- 4 files changed, 167 insertions(+), 38 deletions(-) diff --git a/connectors/agentic-ai/docs/adr-005-implementation-plan.md b/connectors/agentic-ai/docs/adr-005-implementation-plan.md index 4837cd807c5..471cf370d80 100644 --- a/connectors/agentic-ai/docs/adr-005-implementation-plan.md +++ b/connectors/agentic-ai/docs/adr-005-implementation-plan.md @@ -358,28 +358,89 @@ point). ### Sub-phase E3 — `ToolCallResultStrategy` -**Goal**: per-block routing of tool result content. Wired into `ChatClientImpl` so the impls -receive a pre-routed request. - -**Files to create**: -- `ToolCallResultStrategy` — pure function `apply(ToolCallResult, ModelCapabilities)` returning - a routing decision per content block. Decision is one of: - - `INLINE` — keep the block in `ToolCallResult.contentBlocks`; the native impl emits it as - provider-native multimodal content (E4) - - `EXTRACT_TO_USER_MESSAGE` — delegate to the doc-branch's - `ToolCallResultDocumentExtractor` synthesis path. The block is replaced in the tool result - body by an XML document tag; the original `DocumentContent` is appended to a synthetic - `UserMessage` with `METADATA_TOOL_CALL_DOCUMENTS`. +**Goal**: single-pass per-block routing for every document in a chat request, executed at +the `ChatClient` boundary. No extract-then-restore: documents are routed once based on the +resolved `ModelCapabilities` and the appropriate modality slot (tool-result vs. user-message). +The synthetic context messages are written to `RuntimeMemory` inside `ChatClient.chat(...)` +so they are part of the persisted `agentContext.conversation` exactly as the model saw them +— replay across iterations is deterministic. + +> **TODO (post-Phase E):** revisit the `ChatClient`↔`BaseAgentRequestHandler` boundary +> once E3+E4 land. `ChatClient` will then own three responsibilities (request assembly, +> strategy routing with memory mutation, dispatch + metrics). If routing complexity grows +> further, consider extracting routing into a step `BARQ` owns directly, or collapsing +> `ChatClient` into `BARQ`. Don't chase this in Phase E. + +**Files to create** (`framework/strategy/`): +- `ToolCallResultStrategy` — interface. Single method: + ```java + StrategyResult apply(ChatRequest request, ModelCapabilities capabilities); + ``` + with `record StrategyResult(ChatRequest request, List syntheticContextMessages)`. +- `ToolCallResultStrategyImpl` — single-pass walker. Per document found via the existing + `ContentTreeDocumentWalker` / per-handler `extractDocuments` hook (PR #6999 reused + unchanged), routes: + 1. **Tool-result-message docs** vs. `capabilities.toolResultModalities()`: + - `INLINE`: append `DocumentContent` to `ToolCallResult.contentBlocks`; document stays + in the tree for textual rendering. + - `FALLBACK`: replace document with `DocumentXmlTag.from(doc, toolCallId, toolName).toXml()` + inline, append the original `DocumentContent` to a per-message bucket. After the + walk, emit one synthetic `UserMessage` per affected tool-result message — header + text + XML tag + DocumentContent pairs, `METADATA_TOOL_CALL_DOCUMENTS=true`. Same + shape as PR #6999's `createDocumentMessageForToolResults` output. + 2. **User-message / event-message docs** vs. `capabilities.userMessageModalities()`: + - `SUPPORTED`: leave the document where it sits (today's inlining in the same + `UserMessage` content list — no change). + - `UNSUPPORTED`: throw `ConnectorException(ERROR_CODE_FAILED_MODEL_CALL, ...)` with + a message naming the document, modality, and resolved model. **Files to modify**: -- `ChatClientImpl` — query `capabilities()`, apply `ToolCallResultStrategy` on tool-result - messages before the impl call -- `AgentMessagesHandlerImpl` — adjust the existing PR #6999 synthesis call site so blocks - marked `INLINE` skip extraction +- `ChatClientImpl` — inject `ToolCallResultStrategy`. After `registry.resolve(provider)` + and before `api.complete(...)`: + ```java + var capabilities = api.capabilities(); + var initialRequest = new ChatRequest(runtimeMemory.filteredMessages(), ...); + var routed = strategy.apply(initialRequest, capabilities); + routed.syntheticContextMessages().forEach(runtimeMemory::addMessage); + var chatResponse = joinChat(api.complete(routed.request(), options, listener)); + ``` +- `AgentMessagesHandlerImpl` — **remove** the `documentExtractor` field, the + `ToolCallResultDocumentExtractor` constructor parameter, the + `createDocumentMessageForToolResults` private method, and the call from `addUserMessages` + (line 134 in the post-rebase code). Tool-result messages now reach `ChatClientImpl` with + unmodified content trees and empty `ToolCallResult.contentBlocks`. +- `AgenticAiConnectorsAutoConfiguration` — drop the `ToolCallResultDocumentExtractor` + argument from the `AgentMessagesHandlerImpl` bean wiring; add a `ToolCallResultStrategy` + bean and inject into `ChatClientImpl`. The `ToolCallResultDocumentExtractor` bean stays + (now consumed by `ToolCallResultStrategyImpl`). + +**Behavior preserved on the bridge path**: Bridge-served providers report a conservative +capability profile (`tool-result: [text]`, `user-message: [text]`); every document falls +back to the synthetic `UserMessage`, identical to today's PR #6999 output. Bridge e2e +tests stay green without modification. **Tests to add**: -- `ToolCallResultStrategyTest` — table-driven across modality × capability combos -- `ChatClientImplTest` — extend with strategy wiring case +- `ToolCallResultStrategyImplTest` — pure-function table-driven cases. Covers: + - Tool-result image with `tool-result: [text, image, pdf]` → `INLINE`, no synthetic. + - Tool-result PDF with `tool-result: [text, image]` → `FALLBACK`, one synthetic UM with + one DocumentContent; XML placeholder substituted in tool result body. + - Mixed result (one image + one PDF in same tool result, capability `[text, image]`) → + image inline, PDF fallback, single synthetic UM with the PDF only. + - Multiple tool-result messages in one request, each with documents → one synthetic UM + per affected tool-result message, ordered. + - User-message PDF with `user-message: [text, image]` → throws `ConnectorException` + naming the doc + modality + model. + - Event-message image with `user-message: [text]` → throws. + - User-message text only → no-op, same `ChatRequest` returned. + - Nested documents in MCP / A2A tool result content (delegated to per-handler + `extractDocuments`) — image inline, PDF fallback works inside lists/maps. +- `AgentMessagesHandlerImplTest` — adjust existing test cases that asserted on the synthetic + `UserMessage`: those move to `ToolCallResultStrategyImplTest`. Add a new test asserting + `addUserMessages` no longer produces a synthetic UM (extraction is no longer its + responsibility). +- `ChatClientImplTest` — extend with: (a) strategy invoked with resolved capabilities, + (b) returned synthetic messages added to `RuntimeMemory` before dispatch, (c) request + passed to `api.complete` is the strategy's modified request. ### Sub-phase E4 — Multimodality in native impls (image + PDF only) diff --git a/connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md b/connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md index dad6f75d192..f16572e478e 100644 --- a/connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md +++ b/connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md @@ -302,7 +302,7 @@ camunda.connector.agenticai.aiagent.framework.capabilities: defaults: input-modalities: user-message: [text, image, pdf] - tool-result: [text, image] + tool-result: [text, image, pdf] output-modalities: assistant-message: [text] supports-reasoning: false @@ -411,29 +411,95 @@ camunda.connector.agenticai.aiagent.framework.capabilities: Map-key reuse means a consumer override deep-merges into the bundled entry; a new map key adds a new entry. Modality lists replace wholesale (Spring Boot list semantics) — overriding -`tool-result: [text]` discards the bundled `[text, image]`. To add a modality, restate the -full list including the inherited entries. +`tool-result: [text]` discards the bundled `[text, image, pdf]`. To add a modality, restate +the full list including the inherited entries. ## Tool Call Result Routing -`ToolCallResultStrategy` decides per tool result whether to pass `contentBlocks` through to -the provider as native multimodal content or to fall back to the existing user-message -extraction approach (PR #6999). The decision is per content block, driven by the capability -matrix: +`ToolCallResultStrategy` is the **single decision point** that routes every document found +in a chat request to one of three outcomes, in one pass over the request. There is no +extract-then-restore: documents are routed once at the SPI boundary based on the resolved +`ModelCapabilities`. +**Where it runs.** `ChatClientImpl.chat(...)` invokes the strategy after resolving the +`ChatModelApi` and before dispatch: + +``` +1. registry.resolve(provider) → ChatModelApi +2. capabilities = chatModelApi.capabilities() +3. (request, syntheticContextMessages) = strategy.apply(initialRequest, capabilities) +4. syntheticContextMessages.forEach(runtimeMemory::addMessage) ← side effect +5. chatModelApi.complete(request, options, listener) +6. agentContext metric update; return ChatClientResult ``` -for each Content block in tool result: - modality = modalityOf(block) // text | image | pdf | audio | video - if modality in capabilities.toolResultModalities(): - keep block inline as part of ToolCallResult.contentBlocks - else: - delegate to user-message fallback (synthetic UserMessage with DocumentContent) + +The synthetic-message memory write happens **inside** `ChatClient.chat(...)` so the +pre-dispatch context is part of the persisted `agentContext.conversation` exactly as the +model saw it. Replay across iterations is deterministic; the next iteration sees the +synthetic `UserMessage` as ordinary history. Requires no signature change on `ChatClient` +(it already takes `RuntimeMemory`). + +> **TODO (post-Phase E):** revisit the `ChatClient` ↔ `BaseAgentRequestHandler` boundary +> once E3+E4 land. `ChatClient` now performs three jobs (request assembly, strategy routing +> with memory mutation, dispatch + metrics). If the strategy responsibility grows further +> we should consider either (a) extracting routing into a separate pre-chat step `BARQ` +> owns directly, or (b) collapsing `ChatClient` into `BARQ` entirely. Defer until we see +> how the multimodal path settles. + +**Strategy contract.** Pure function: + +```java +record StrategyResult(ChatRequest request, List syntheticContextMessages) {} + +StrategyResult apply(ChatRequest request, ModelCapabilities capabilities); ``` -Models with `supports_*_in_tool_result` modalities for a given media type get the native -path. Models without — including all models served via the LangChain4j bridge — get the -fallback. This makes PR #6999 the safe default and the multimodal-native path the -opt-in-by-capability optimization. +Walks every message in the request once. For each `Document` encountered (delegating to the +existing PR #6999 walker / per-handler `extractDocuments` hook), three branches keyed on +which message type the document lives in: + +1. **Tool-result-message documents** — modality vs. `capabilities.toolResultModalities()`: + * **Inline**: append the `DocumentContent` to `ToolCallResult.contentBlocks`. The native + impl emits it as provider-native multimodal content on the same tool result. The + textual rendering of the result still includes the document's representation (filename, + short id, etc.) so structure is preserved. + * **Fallback**: replace the document inline with an XML placeholder + (``, today's PR #6999 format) + and append the original `DocumentContent` to a per-tool-result-message bucket. +2. **User-message and event-message documents** — modality vs. + `capabilities.userMessageModalities()`: + * **Supported**: leave the document where it sits (today's inlining in the same + `UserMessage` content list — no change). + * **Unsupported**: **fail loud** with `ConnectorException(ERROR_CODE_FAILED_MODEL_CALL, + ...)` and a clear message naming the document, modality, and resolved model. Mirrors + L4J `DocumentConversionException` semantics. There is no synthesis fallback for user + messages — the agent author must supply documents the model can read. + +**Synthesis output.** Whenever a tool-result message produced fallback documents, the +strategy emits one synthetic `UserMessage` per affected tool-result message (same shape PR +#6999 produces today: `METADATA_TOOL_CALL_DOCUMENTS=true`, header text, XML tag + +`DocumentContent` pairs). These are the `syntheticContextMessages` returned to +`ChatClientImpl`. `MessageWindowRuntimeMemory` already excludes messages with +`METADATA_TOOL_CALL_DOCUMENTS=true` from the window count, so synthesis volume cannot +push history out. + +**Single pass guarantees.** +* Every document is visited exactly once. +* Inline-eligible docs never enter the synthesis path. +* Fallback docs never appear on `ToolCallResult.contentBlocks`. +* Native impls (E4) read `ToolCallResult.contentBlocks` and emit blindly — they do **not** + consult capabilities. + +**Behavior of the L4J bridge.** Bridge-served providers report a conservative capability +profile (`tool-result: [text]`, `user-message: [text]`); every document falls back to the +synthetic `UserMessage`, identical to today's PR #6999 behavior — no regression. + +**Removal in `AgentMessagesHandlerImpl`.** The current PR #6999 call site +(`createDocumentMessageForToolResults` invoked from `addUserMessages`) and the +unconditional `documentExtractor` dependency move out — extraction is now solely the +strategy's responsibility. The XML-tag generator (`DocumentXmlTag`), the recursive +walker (`ContentTreeDocumentWalker`), and the per-handler `extractDocuments` hook on +`GatewayToolHandler` are all reused unchanged; only the call site relocates. ## Reasoning Support diff --git a/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml b/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml index c66f1592cb6..ed522a06856 100644 --- a/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml +++ b/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml @@ -51,7 +51,7 @@ camunda: defaults: input-modalities: user-message: [text, image, pdf] - tool-result: [text, image] + tool-result: [text, image, pdf] output-modalities: assistant-message: [text] supports-reasoning: false diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/BundledCapabilityMatrixTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/BundledCapabilityMatrixTest.java index ed6b1cabfce..f90ed4f156f 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/BundledCapabilityMatrixTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/BundledCapabilityMatrixTest.java @@ -59,7 +59,8 @@ void claudeSonnet4ResolvesToFullCapabilities() { assertThat(caps.userMessageModalities()) .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); - assertThat(caps.toolResultModalities()).containsExactly(Modality.TEXT, Modality.IMAGE); + assertThat(caps.toolResultModalities()) + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); assertThat(caps.supportsReasoning()).isTrue(); assertThat(caps.supportsReasoningSignatureRoundtrip()).isTrue(); assertThat(caps.supportsPromptCaching()).isTrue(); @@ -75,7 +76,8 @@ void claudeHaiku4InheritsToolResultButOverridesUserMessage() { final var caps = resolve(context, "anthropic-messages", "claude-haiku-4-5"); assertThat(caps.userMessageModalities()).containsExactly(Modality.TEXT, Modality.IMAGE); - assertThat(caps.toolResultModalities()).containsExactly(Modality.TEXT, Modality.IMAGE); + assertThat(caps.toolResultModalities()) + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); assertThat(caps.supportsReasoning()).isFalse(); }); } From 56bf33767966de7040fd0cb2ce2b0197b22b8074 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 20:14:51 +0200 Subject: [PATCH 60/81] feat(agentic-ai): tool-call-result routing + native multimodal emission (Phase E3+E4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Single-pass routing of every document found in a ChatRequest at the ChatClient boundary, plus the per-impl native multimodal emission paths that consume the routed tool-result `contentBlocks`. Combined into one phase because the bundled capability matrix declares modalities that the impls have to actually emit — shipping E3 alone would silently drop inline-routed documents on the floor between phases. ToolCallResultStrategy (`framework/strategy/`): - pure function `apply(ChatRequest, ModelCapabilities) -> (ChatRequest, List)` - single walk over the request: - tool-result documents -> `toolResultModalities`: inline-supported docs go onto `ToolCallResult.contentBlocks`; the rest fall back to a synthetic UserMessage (PR #6999 shape, `METADATA_TOOL_CALL_DOCUMENTS=true`) - user-message and event-message documents -> `userMessageModalities`: supported docs stay inline; unsupported docs throw `ConnectorException` (no synthesis fallback for user messages, mirroring L4J `DocumentConversionException`) ChatClientImpl runs the strategy after capability resolution and persists the synthetic context messages into `RuntimeMemory` immediately after the anchor `ToolCallResultMessage` so the persisted `agentContext.conversation` matches what the model saw on the wire (deterministic replay). The clear()+addMessages() insertion dance is flagged with a TODO to revisit once the ChatClient<->BARQ boundary settles after Phase E. AgentMessagesHandlerImpl drops `ToolCallResultDocumentExtractor` from its constructor and no longer creates `documentMessage` itself — extraction is now exclusively the strategy's responsibility. The PR #6999 walker, per-handler `extractDocuments` hook, XML correlation tag, and window-count exclusion are reused unchanged. Native multimodal emission (image + PDF only): - AnthropicMessagesChatModelApi: `ContentBlockParam.ofImage(...)` + `ofDocument(...)` on user messages; `ToolResultBlockParam.Content.Block.ofImage(...)` + `ofDocument(...)` with `contentOfBlocks(...)` on tool results when contentBlocks is populated. Adds `ObjectMapper` to the impl + factory + Spring config for JSON-serialised inline tool-result text bodies. - OpenAiResponsesChatModelApi: `ResponseInputContent.ofInputImage(...)` / `ofInputFile(...)` on user messages via `EasyInputMessage.contentOfResponseInputMessageContentList(...)`; `ResponseFunctionCallOutputItem.ofInputImage(...)` / `ofInputFile(...)` on tool results via `outputOfResponseFunctionCallOutputItemList(...)`. - OpenAiChatCompletionsChatModelApi: `ChatCompletionContentPart.ofImageUrl(...)` + `ofFile(...)` on user messages via `addUserMessageOfArrayOfContentParts(...)`. Tool messages stay text-only (SDK enforces `ChatCompletionContentPartText`-only). Bundled capability matrix: anthropic-messages tool-result modality bumped to `[text, image, pdf]` (verified via `ToolResultBlockParam.Content.Block.ofDocument`); openai-completions user-message bumped to `[text, image, pdf]`; openai-responses user-message bumped to `[text, image, pdf]`. Tests: 1380 unit tests + 3 wire-format e2e tests pass. New `ToolCallResultStrategyImplTest` (8 cases) covers inline routing, fallback synthesis, single-pass split, ordering, user-message validation, and the no-document no-op path. `AgentMessagesHandlerTest` synthesis assertions migrated to the strategy test; new test pins that `addUserMessages` no longer emits a synthetic UserMessage. --- .../docs/adr-005-implementation-plan.md | 79 ++--- .../agent/AgentMessagesHandlerImpl.java | 34 +-- .../aiagent/framework/ChatClientImpl.java | 56 +++- .../AnthropicMessagesApiConfiguration.java | 7 +- .../AnthropicMessagesChatModelApi.java | 164 ++++++++++- .../AnthropicMessagesChatModelApiFactory.java | 8 +- .../multimodal/DocumentModality.java | 98 +++++++ .../OpenAiChatCompletionsChatModelApi.java | 96 ++++++- .../openai/OpenAiResponsesChatModelApi.java | 180 +++++++++++- .../strategy/ToolCallResultStrategy.java | 56 ++++ .../strategy/ToolCallResultStrategyImpl.java | 188 ++++++++++++ .../AgenticAiConnectorsAutoConfiguration.java | 21 +- .../capabilities/model-capabilities.yaml | 4 +- .../agent/AgentMessagesHandlerTest.java | 86 +----- .../aiagent/framework/ChatClientImplTest.java | 21 +- .../AnthropicMessagesChatModelApiTest.java | 10 +- .../CapabilityMatrixOverrideTest.java | 2 +- .../ToolCallResultStrategyImplTest.java | 270 ++++++++++++++++++ 18 files changed, 1198 insertions(+), 182 deletions(-) create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/multimodal/DocumentModality.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategy.java create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategyImpl.java create mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategyImplTest.java diff --git a/connectors/agentic-ai/docs/adr-005-implementation-plan.md b/connectors/agentic-ai/docs/adr-005-implementation-plan.md index 471cf370d80..028ce30dd4e 100644 --- a/connectors/agentic-ai/docs/adr-005-implementation-plan.md +++ b/connectors/agentic-ai/docs/adr-005-implementation-plan.md @@ -13,7 +13,7 @@ before generalising leads to a smaller, better-shaped abstraction in Phase E. Re caching, Azure OpenAI, Anthropic cloud backends, Google GenAI and Bedrock-Converse are all explicitly **deferred** out of the first native cut. -Phase E is split into four sub-phases that ship as independent commits (E1–E4 below). Phase E +Phase E is split into three sub-phases (E1, E2 done; E3+E4 combined — see §"Phase E" below). Phase E is layered on top of PR #6999 (`agentic-ai-document-tool-call-results`); our branch was rebased onto that PR while it was in active review. @@ -152,8 +152,8 @@ The first native cut deliberately ignores these — they re-enter in Phase E or |-------|-----------| | Capability matrix YAML + resolver | E1 (done) | | `ChatModelApi.capabilities()` resolved per call | E2 (done) | -| `ToolCallResultStrategy` (always inline-text in B–D) | E3 | -| Multimodal user-message / tool-result content — **image + PDF only** | E4 | +| `ToolCallResultStrategy` (always inline-text in B–D) | E3+E4 (combined) | +| Multimodal user-message / tool-result content — **image + PDF only** | E3+E4 (combined) | | Multimodal — audio / video | G+ (when matching native impls land) | | Reasoning content (signed thinking blocks, encrypted reasoning items) | post-E (own sub-phase) | | Prompt caching (`cache_control`, `prompt_cache_key`) | post-E (own sub-phase) | @@ -283,9 +283,13 @@ pass against the native impl. ## Phase E — Capability matrix + tool-result strategy + multimodality -**Plan revision (2026-05-07)** — Phase E is split into four sub-phases that ship as independent -commits. Reasoning and prompt caching are explicitly **deferred out of Phase E** to keep the cut -focused; the matrix already declares the flags so the slot is reserved. +**Plan revision (2026-05-07)** — Phase E is split into three sub-phases. E1 + E2 ship as +independent commits; **E3 and E4 ship as one combined commit** because the strategy and the +native multimodal emission depend on each other (the bundled matrix declares modalities that +the impls have to actually emit, otherwise `ToolCallResult.contentBlocks` entries would be +silently dropped between phases). Reasoning and prompt caching are explicitly **deferred out +of Phase E** to keep the cut focused; the matrix already declares the flags so the slot is +reserved. Phase E is layered on top of the work from PR #6999 (`agentic-ai-document-tool-call-results`), which contributes the `ToolCallResultDocumentExtractor` (recursive walker over lists / maps / @@ -356,14 +360,13 @@ always resolves under `openai-completions`. The L4J bridge keeps its own conserv (it stays as a fallback path; replacing it through the resolver is not worth the change at this point). -### Sub-phase E3 — `ToolCallResultStrategy` +### Sub-phase E3 + E4 — `ToolCallResultStrategy` + native multimodal emission (combined) -**Goal**: single-pass per-block routing for every document in a chat request, executed at -the `ChatClient` boundary. No extract-then-restore: documents are routed once based on the -resolved `ModelCapabilities` and the appropriate modality slot (tool-result vs. user-message). -The synthetic context messages are written to `RuntimeMemory` inside `ChatClient.chat(...)` -so they are part of the persisted `agentContext.conversation` exactly as the model saw them -— replay across iterations is deterministic. +**Goal**: single-pass per-block routing for every document in a chat request (E3), plus +the native multimodal emission paths in each provider impl that consume the routed +`ToolCallResult.contentBlocks` and emit images / PDFs on the wire (E4). Shipped together so +the bundled capability matrix is honest — every modality the matrix declares for a family +has a working emitter on the same commit. > **TODO (post-Phase E):** revisit the `ChatClient`↔`BaseAgentRequestHandler` boundary > once E3+E4 land. `ChatClient` will then own three responsibilities (request assembly, @@ -442,33 +445,39 @@ tests stay green without modification. (b) returned synthetic messages added to `RuntimeMemory` before dispatch, (c) request passed to `api.complete` is the strategy's modified request. -### Sub-phase E4 — Multimodality in native impls (image + PDF only) - -**Scope** matches L4J parity (`DocumentToContentConverterImpl` supports text + image + PDF). -Audio and video are **out of scope for E4**; the capability matrix slot remains for Phase G+. - -**Modality detection**: shared utility (or per-impl) maps Camunda -`Document.metadata().contentType()` (MIME) → `ModelCapabilities.Modality`. Sealed Content -hierarchy stays as-is — no new `ImageContent` / `PdfContent` subtypes; dispatch is MIME-based -at conversion time. - -**Per-impl native encoding**: -- **Anthropic Messages**: `ContentBlockParam.ofImage(...)` (base64 data + media_type), - `ContentBlockParam.ofDocument(...)` for PDFs. Tool result and user message both supported. +**Multimodal emission (E4 portion)** — scope matches L4J parity +(`DocumentToContentConverterImpl` supports text + image + PDF). Audio and video are **out of +scope for the combined E3+E4 phase**; the capability matrix slot remains for Phase G+. + +- **Modality detection**: shared utility maps Camunda `Document.metadata().contentType()` + (MIME) → `ModelCapabilities.Modality`. Sealed `Content` hierarchy stays as-is — no new + `ImageContent` / `PdfContent` subtypes; dispatch is MIME-based at conversion time. Used by + both the strategy (capability check) and the per-impl emitters (block construction). +- **Anthropic Messages**: `ContentBlockParam.ofImage(...)` (base64 data + media_type) and + `ContentBlockParam.ofDocument(...)` for PDFs in user messages; + `ToolResultBlockParam.Content.Block.ofImage(...)` and `.ofDocument(...)` for tool results. - **OpenAI Chat Completions**: `ChatCompletionContentPart.ofImageUrl(...)` with data URL on user messages. Tool messages are text-only (SDK enforces - `ChatCompletionContentPartText`-only for tool message content arrays); any image block in a - tool result must route through E3's `EXTRACT_TO_USER_MESSAGE`. + `ChatCompletionContentPartText`-only for tool message content arrays). Bundled matrix: + `tool-result: [text]` for openai-completions — strategy always falls back tool-result + documents on this family. - **OpenAI Responses**: `ResponseInputContent.ofInputImage(...)` (base64) and - `ofInputFile(...)` (PDF base64). Multimodal tool results via + `ofInputFile(...)` (PDF base64) on user messages. Multimodal tool results via `ResponseInputItem.FunctionCallOutput.Output.ofResponseFunctionCallOutputItemList(...)`. -**User-message capability mismatch** (image in user prompt for a text-only model): fail loud -with `ConnectorException(ERROR_CODE_FAILED_MODEL_CALL, ...)` and a clear message. Mirrors L4J's -`DocumentConversionException` semantics for unsupported types. - -**Tests to add**: per-impl multimodal cases, including a wire-format e2e per provider that -exercises the inline-native path; existing PR #6999 e2e covers the fallback path. +**User-message capability mismatch** (image in user prompt for a text-only model): the +strategy fails loud with `ConnectorException(ERROR_CODE_FAILED_MODEL_CALL, ...)` and a clear +message — this is the strategy's responsibility (E3); native impls do **not** consult +capabilities and emit blindly. + +**Multimodal tests** (in addition to the strategy tests above): +- `AnthropicMessagesChatModelApiTest` — image + PDF in user message; image + PDF in tool + result; mixed text + image + PDF tool-result content list. +- `OpenAiResponsesChatModelApiTest` — same coverage as Anthropic. +- `OpenAiChatCompletionsChatModelApiTest` — image in user message; assert tool-result with + any non-text block routes via the strategy fallback (no inline emission attempted). +- Wire-format e2e per native family that exercises the inline-native path; existing PR #6999 + e2e covers the fallback path. ### Deferred out of Phase E diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java index 41856c51bd9..a00a0f6721c 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerImpl.java @@ -34,7 +34,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -59,15 +58,11 @@ public class AgentMessagesHandlerImpl implements AgentMessagesHandler { private final GatewayToolHandlerRegistry gatewayToolHandlers; private final SystemPromptComposer systemPromptComposer; - private final ToolCallResultDocumentExtractor documentExtractor; public AgentMessagesHandlerImpl( - GatewayToolHandlerRegistry gatewayToolHandlers, - SystemPromptComposer systemPromptComposer, - ToolCallResultDocumentExtractor documentExtractor) { + GatewayToolHandlerRegistry gatewayToolHandlers, SystemPromptComposer systemPromptComposer) { this.gatewayToolHandlers = gatewayToolHandlers; this.systemPromptComposer = systemPromptComposer; - this.documentExtractor = documentExtractor; } @Override @@ -131,10 +126,6 @@ public List addUserMessages( // if message is null, we wait on further tool call results to be added if (toolCallResultMessage != null) { messages.add(toolCallResultMessage); - var documentMessage = createDocumentMessageForToolResults(toolCallResultMessage.results()); - if (documentMessage != null) { - messages.add(documentMessage); - } messages.addAll(eventMessages); } } else { @@ -215,29 +206,6 @@ private ToolCallResultMessage createToolCallResultMessage( .build(); } - private UserMessage createDocumentMessageForToolResults(List results) { - final var toolCallDocuments = documentExtractor.extractDocuments(results); - if (toolCallDocuments.isEmpty()) { - return null; - } - - final var content = new ArrayList(); - content.add(textContent("Documents extracted from tool call results:")); - for (var entry : toolCallDocuments) { - for (var doc : entry.documents()) { - content.add( - textContent( - DocumentXmlTag.from(doc, entry.toolCallId(), entry.toolCallName()).toXml())); - content.add(DocumentContent.documentContent(doc)); - } - } - - final var metadata = new HashMap(defaultMessageMetadata()); - metadata.put(UserMessage.METADATA_TOOL_CALL_DOCUMENTS, true); - - return UserMessage.builder().content(content).metadata(metadata).build(); - } - private Message createEventMessage( ToolCallResult eventResult, boolean interruptToolCallsOnEventResults) { Object eventContent = eventResult.content(); diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImpl.java index 8518712a92e..6c61ae9012c 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImpl.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImpl.java @@ -12,11 +12,16 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatOptions; import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; +import io.camunda.connector.agenticai.aiagent.framework.strategy.ToolCallResultStrategy; import io.camunda.connector.agenticai.aiagent.memory.runtime.RuntimeMemory; import io.camunda.connector.agenticai.aiagent.model.AgentContext; import io.camunda.connector.agenticai.aiagent.model.AgentExecutionContext; import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.agenticai.aiagent.model.request.ResponseConfiguration; +import io.camunda.connector.agenticai.model.message.ToolCallResultMessage; +import io.camunda.connector.agenticai.model.message.UserMessage; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletionException; @@ -24,9 +29,12 @@ public class ChatClientImpl implements ChatClient { private final ChatModelApiRegistry registry; + private final ToolCallResultStrategy toolCallResultStrategy; - public ChatClientImpl(ChatModelApiRegistry registry) { + public ChatClientImpl( + ChatModelApiRegistry registry, ToolCallResultStrategy toolCallResultStrategy) { this.registry = registry; + this.toolCallResultStrategy = toolCallResultStrategy; } @Override @@ -36,18 +44,25 @@ public ChatClientResult chat( RuntimeMemory runtimeMemory, ChatStreamListener listener) { final var api = registry.resolve(executionContext.provider()); - final var request = + final var capabilities = api.capabilities(); + + final var initialRequest = new ChatRequest( runtimeMemory.filteredMessages(), agentContext.toolDefinitions(), Optional.ofNullable(executionContext.response()) .map(ResponseConfiguration::format) .orElse(null)); + + final var routed = toolCallResultStrategy.apply(initialRequest, capabilities); + persistSyntheticContextMessages(runtimeMemory, routed.syntheticContextMessages()); + final var options = new ChatOptions(null, null, null, Map.of()); final var chatResponse = joinChat( - api.complete(request, options, listener != null ? listener : ChatStreamListener.NOOP)); + api.complete( + routed.request(), options, listener != null ? listener : ChatStreamListener.NOOP)); final var assistantMessage = chatResponse.assistantMessage(); final var usage = @@ -61,6 +76,41 @@ public ChatClientResult chat( return new ChatClientResult(updatedAgentContext, assistantMessage); } + /** + * Inserts synthetic context messages into {@link RuntimeMemory} immediately after the most recent + * {@link ToolCallResultMessage}. Preserves the {@code [TCRM, syntheticUM, eventUMs]} order that + * {@code MessageWindowRuntimeMemory}'s eviction relies on (synthetic UMs immediately follow their + * anchor TCR so eviction co-removes them). + * + *

      TODO(adr-005): the {@code clear() + addMessages(all)} dance is ugly. Replace with either an + * {@code insertAfter(predicate, messages)} method on {@link RuntimeMemory}, or move synthetic-UM + * insertion ownership to {@code AgentMessagesHandler} (with a deferred slot the strategy fills), + * or fold {@code ChatClient}'s routing responsibility into {@code BaseAgentRequestHandler}. See + * the {@code ChatClient}↔{@code BARQ} TODO in ADR-005 §"Tool Call Result Routing". + */ + private static void persistSyntheticContextMessages( + RuntimeMemory runtimeMemory, List syntheticContextMessages) { + if (syntheticContextMessages.isEmpty()) { + return; + } + final var all = new ArrayList<>(runtimeMemory.allMessages()); + int anchorIdx = -1; + for (int i = all.size() - 1; i >= 0; i--) { + if (all.get(i) instanceof ToolCallResultMessage) { + anchorIdx = i; + break; + } + } + if (anchorIdx < 0) { + throw new IllegalStateException( + "Strategy produced synthetic context messages but no ToolCallResultMessage exists " + + "in runtime memory — strategy invariant violated"); + } + all.addAll(anchorIdx + 1, syntheticContextMessages); + runtimeMemory.clear(); + runtimeMemory.addMessages(all); + } + private static io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse joinChat( java.util.concurrent.CompletableFuture< io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse> diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesApiConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesApiConfiguration.java index 85a5c8ef852..605a59b5221 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesApiConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesApiConfiguration.java @@ -7,10 +7,12 @@ package io.camunda.connector.agenticai.aiagent.framework.anthropic; import com.anthropic.client.AnthropicClient; +import com.fasterxml.jackson.databind.ObjectMapper; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; import io.camunda.connector.agenticai.aiagent.framework.capabilities.ModelCapabilitiesResolver; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsConfigurationProperties; +import io.camunda.connector.runtime.annotation.ConnectorsObjectMapper; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; @@ -29,9 +31,12 @@ public class AnthropicMessagesApiConfiguration { @Bean(name = "langchain4JAnthropicChatModelApiFactory") @ConditionalOnMissingBean(name = "langchain4JAnthropicChatModelApiFactory") public ChatModelApiFactory anthropicMessagesChatModelApiFactory( + @ConnectorsObjectMapper ObjectMapper objectMapper, ModelCapabilitiesResolver capabilitiesResolver, AgenticAiConnectorsConfigurationProperties properties) { return new AnthropicMessagesChatModelApiFactory( - capabilitiesResolver, properties.aiagent().chatModel().api().defaultTimeout()); + objectMapper, + capabilitiesResolver, + properties.aiagent().chatModel().api().defaultTimeout()); } } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java index cd3fd74283d..ea5c0240730 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java @@ -10,8 +10,12 @@ import com.anthropic.client.AnthropicClient; import com.anthropic.core.JsonValue; +import com.anthropic.models.messages.Base64ImageSource; +import com.anthropic.models.messages.Base64PdfSource; import com.anthropic.models.messages.ContentBlock; import com.anthropic.models.messages.ContentBlockParam; +import com.anthropic.models.messages.DocumentBlockParam; +import com.anthropic.models.messages.ImageBlockParam; import com.anthropic.models.messages.Message; import com.anthropic.models.messages.MessageCreateParams; import com.anthropic.models.messages.MessageParam; @@ -21,12 +25,15 @@ import com.anthropic.models.messages.ToolUnion; import com.anthropic.models.messages.ToolUseBlockParam; import com.anthropic.models.messages.Usage; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; import io.camunda.connector.agenticai.aiagent.framework.api.ChatOptions; import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.multimodal.DocumentModality; import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.agenticai.model.message.AssistantMessage; import io.camunda.connector.agenticai.model.message.AssistantMessageBuilder; @@ -35,11 +42,13 @@ import io.camunda.connector.agenticai.model.message.ToolCallResultMessage; import io.camunda.connector.agenticai.model.message.UserMessage; import io.camunda.connector.agenticai.model.message.content.Content; +import io.camunda.connector.agenticai.model.message.content.DocumentContent; import io.camunda.connector.agenticai.model.message.content.ObjectContent; import io.camunda.connector.agenticai.model.message.content.TextContent; import io.camunda.connector.agenticai.model.tool.ToolCall; import io.camunda.connector.agenticai.model.tool.ToolCallResult; import io.camunda.connector.agenticai.model.tool.ToolDefinition; +import io.camunda.connector.api.document.Document; import io.camunda.connector.api.error.ConnectorException; import java.time.ZonedDateTime; import java.util.ArrayList; @@ -69,6 +78,7 @@ public class AnthropicMessagesChatModelApi implements ChatModelApi { private final AnthropicClient client; private final String model; + private final ObjectMapper objectMapper; private final ModelCapabilities capabilities; @Nullable private final Long configuredMaxTokens; @Nullable private final Double temperature; @@ -78,6 +88,7 @@ public class AnthropicMessagesChatModelApi implements ChatModelApi { public AnthropicMessagesChatModelApi( AnthropicClient client, String model, + ObjectMapper objectMapper, ModelCapabilities capabilities, @Nullable Long configuredMaxTokens, @Nullable Double temperature, @@ -85,6 +96,7 @@ public AnthropicMessagesChatModelApi( @Nullable Long topK) { this.client = Objects.requireNonNull(client, "client"); this.model = Objects.requireNonNull(model, "model"); + this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper"); this.capabilities = Objects.requireNonNull(capabilities, "capabilities"); this.configuredMaxTokens = configuredMaxTokens; this.temperature = temperature; @@ -159,7 +171,7 @@ private static String systemPrompt(SystemMessage system) { } private MessageParam toMessageParam(UserMessage message) { - final var blocks = textOnlyBlocks(message.content()); + final var blocks = messageContentBlocks(message.content()); return MessageParam.builder().role(MessageParam.Role.USER).contentOfBlockParams(blocks).build(); } @@ -180,11 +192,49 @@ private MessageParam toMessageParam(AssistantMessage message) { } private MessageParam toMessageParam(ToolCallResultMessage message) { - final var blocks = - message.results().stream().map(AnthropicMessagesChatModelApi::toolResultBlock).toList(); + final var blocks = message.results().stream().map(this::toolResultBlock).toList(); return MessageParam.builder().role(MessageParam.Role.USER).contentOfBlockParams(blocks).build(); } + /** + * User-message blocks: text + multimodal documents (image, PDF). The {@link + * io.camunda.connector.agenticai.aiagent.framework.strategy.ToolCallResultStrategy} has already + * validated user-message documents against the model's {@code userMessageModalities}, so any + * {@link DocumentContent} reaching this point is known to be supported. + */ + private static List messageContentBlocks(List content) { + if (content == null) { + return List.of(); + } + final var blocks = new ArrayList(); + for (var c : content) { + blocks.add(messageContentBlock(c)); + } + return blocks; + } + + private static ContentBlockParam messageContentBlock(Content content) { + if (content instanceof DocumentContent doc) { + final var modality = DocumentModality.of(doc.document()); + return switch (modality) { + case IMAGE -> ContentBlockParam.ofImage(imageBlockParam(doc.document())); + case PDF -> ContentBlockParam.ofDocument(pdfBlockParam(doc.document())); + default -> + throw new IllegalArgumentException( + "Document modality " + + modality + + " is not supported in Anthropic user/tool messages " + + "(only image + PDF emit natively); the strategy should have routed this " + + "document to a synthetic UserMessage or rejected it."); + }; + } + return textOnlyBlock(content); + } + + /** + * Assistant-message blocks: text only (assistant turns we send back to the model don't carry + * documents). Used by the assistant-message conversion path. + */ private static List textOnlyBlocks(List content) { if (content == null) { return List.of(); @@ -209,6 +259,35 @@ private static ContentBlockParam textOnlyBlock(Content content) { + content.getClass().getSimpleName()); } + private static ImageBlockParam imageBlockParam(Document document) { + final var mimeType = document.metadata().getContentType(); + return ImageBlockParam.builder() + .source( + Base64ImageSource.builder() + .data(document.asBase64()) + .mediaType(toAnthropicImageMediaType(mimeType)) + .build()) + .build(); + } + + private static DocumentBlockParam pdfBlockParam(Document document) { + return DocumentBlockParam.builder() + .source(Base64PdfSource.builder().data(document.asBase64()).build()) + .build(); + } + + private static Base64ImageSource.MediaType toAnthropicImageMediaType(String mimeType) { + return switch (mimeType.toLowerCase().trim()) { + case "image/jpeg" -> Base64ImageSource.MediaType.IMAGE_JPEG; + case "image/png" -> Base64ImageSource.MediaType.IMAGE_PNG; + case "image/gif" -> Base64ImageSource.MediaType.IMAGE_GIF; + case "image/webp" -> Base64ImageSource.MediaType.IMAGE_WEBP; + default -> + throw new IllegalArgumentException( + "Unsupported image MIME type for Anthropic image source: " + mimeType); + }; + } + private static ContentBlockParam toolUseBlock(ToolCall call) { final var inputBuilder = ToolUseBlockParam.Input.builder(); if (call.arguments() != null) { @@ -223,15 +302,20 @@ private static ContentBlockParam toolUseBlock(ToolCall call) { .build()); } - private static ContentBlockParam toolResultBlock(ToolCallResult result) { + private ContentBlockParam toolResultBlock(ToolCallResult result) { final var b = ToolResultBlockParam.builder().toolUseId(result.id()); - final var content = result.content(); - if (content == null) { - b.content(ToolCallResult.CONTENT_NO_RESULT); - } else if (content instanceof String s) { - b.content(s); + final var inlineBlocks = toolResultContentBlocks(result); + if (inlineBlocks != null) { + b.contentOfBlocks(inlineBlocks); } else { - b.contentAsJson(content); + final var content = result.content(); + if (content == null) { + b.content(ToolCallResult.CONTENT_NO_RESULT); + } else if (content instanceof String s) { + b.content(s); + } else { + b.contentAsJson(content); + } } final var interrupted = result.properties() != null @@ -242,6 +326,66 @@ private static ContentBlockParam toolResultBlock(ToolCallResult result) { return ContentBlockParam.ofToolResult(b.build()); } + /** + * Builds the {@code [text, image, document]} block list for a tool result whose {@code + * contentBlocks} were populated by the strategy with inline-supported documents. Returns null + * when no inline routing happened — caller falls back to the string / JSON content path. + * + *

      Shape: one text block with the serialised tool-result content (so the model still sees the + * structured JSON the result came from, with document references in place), followed by one image + * / document block per inline document. + */ + private List toolResultContentBlocks(ToolCallResult result) { + if (result.contentBlocks() == null || result.contentBlocks().isEmpty()) { + return null; + } + final var blocks = new ArrayList(); + blocks.add( + ToolResultBlockParam.Content.Block.ofText( + TextBlockParam.builder().text(serializedToolResultText(result)).build())); + for (var block : result.contentBlocks()) { + if (!(block instanceof DocumentContent doc)) { + throw new IllegalArgumentException( + "Unsupported inline tool-result content block type: " + + block.getClass().getSimpleName()); + } + final var modality = DocumentModality.of(doc.document()); + switch (modality) { + case IMAGE -> + blocks.add(ToolResultBlockParam.Content.Block.ofImage(imageBlockParam(doc.document()))); + case PDF -> + blocks.add( + ToolResultBlockParam.Content.Block.ofDocument(pdfBlockParam(doc.document()))); + default -> + throw new IllegalArgumentException( + "Document modality " + + modality + + " is not supported in Anthropic tool result blocks (only image + PDF emit " + + "natively); the strategy should have routed this document to a synthetic " + + "UserMessage."); + } + } + return blocks; + } + + private String serializedToolResultText(ToolCallResult result) { + final var content = result.content(); + if (content == null) { + return ToolCallResult.CONTENT_NO_RESULT; + } + if (content instanceof String s) { + return s; + } + try { + return objectMapper.writeValueAsString(content); + } catch (JsonProcessingException e) { + throw new ConnectorException( + ERROR_CODE_FAILED_MODEL_CALL, + "Failed to serialise tool call result content to JSON for tool '%s': %s" + .formatted(result.name(), e.getOriginalMessage())); + } + } + private ToolUnion toToolUnion(ToolDefinition definition) { final var tool = Tool.builder().name(definition.name()).inputSchema(toInputSchema(definition.inputSchema())); diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java index fb3fa68c6a4..153261b5557 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java @@ -8,6 +8,7 @@ import com.anthropic.client.AnthropicClient; import com.anthropic.client.okhttp.AnthropicOkHttpClient; +import com.fasterxml.jackson.databind.ObjectMapper; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; import io.camunda.connector.agenticai.aiagent.framework.capabilities.ModelCapabilitiesResolver; @@ -32,11 +33,15 @@ public class AnthropicMessagesChatModelApiFactory public static final String API_FAMILY = "anthropic-messages"; + private final ObjectMapper objectMapper; private final ModelCapabilitiesResolver capabilitiesResolver; @Nullable private final Duration defaultTimeout; public AnthropicMessagesChatModelApiFactory( - ModelCapabilitiesResolver capabilitiesResolver, @Nullable Duration defaultTimeout) { + ObjectMapper objectMapper, + ModelCapabilitiesResolver capabilitiesResolver, + @Nullable Duration defaultTimeout) { + this.objectMapper = objectMapper; this.capabilitiesResolver = capabilitiesResolver; this.defaultTimeout = defaultTimeout; } @@ -66,6 +71,7 @@ public ChatModelApi create(AnthropicProviderConfiguration configuration) { return new AnthropicMessagesChatModelApi( client, connection.model().model(), + objectMapper, capabilities, parameters != null && parameters.maxTokens() != null ? parameters.maxTokens().longValue() diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/multimodal/DocumentModality.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/multimodal/DocumentModality.java new file mode 100644 index 00000000000..f0347ca3f8e --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/multimodal/DocumentModality.java @@ -0,0 +1,98 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.multimodal; + +import static io.camunda.connector.agenticai.aiagent.agent.AgentErrorCodes.ERROR_CODE_FAILED_MODEL_CALL; + +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; +import io.camunda.connector.api.document.Document; +import io.camunda.connector.api.document.DocumentMetadata; +import io.camunda.connector.api.error.ConnectorException; +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; + +/** + * Maps a Camunda {@link Document}'s declared MIME type to a {@link Modality}. + * + *

      Modality vocabulary matches the capability matrix; the routing strategy uses this to decide + * whether a document fits the resolved model's modality slot, and the per-impl emitters use it to + * dispatch construction of provider-native content blocks. + * + *

      Coverage parity with the LangChain4j path ({@code DocumentToContentConverterImpl}): {@code + * text/*}, {@code application/json}, {@code application/xml}, {@code application/yaml} → {@link + * Modality#TEXT}; the four common image MIME types → {@link Modality#IMAGE}; {@code + * application/pdf} → {@link Modality#PDF}. Audio + video MIME types map for completeness but no + * emitter consumes them yet (Phase G+). + */ +public final class DocumentModality { + + private static final Set TEXT_MIME_TYPES = + Set.of("application/json", "application/xml", "application/yaml"); + + private static final Set IMAGE_MIME_TYPES = + Set.of("image/jpeg", "image/png", "image/gif", "image/webp"); + + private DocumentModality() {} + + /** + * Resolves the modality of a document, throwing a {@link ConnectorException} when the MIME type + * is missing or not part of the supported vocabulary. Mirrors the L4J converter's fail-loud + * semantics on unsupported types. + */ + public static Modality of(Document document) { + final var mimeType = mimeTypeOf(document); + return classify(mimeType) + .orElseThrow( + () -> + new ConnectorException( + ERROR_CODE_FAILED_MODEL_CALL, + "Unsupported content type '%s' for document with reference '%s'" + .formatted(mimeType, document.reference()))); + } + + /** + * Pure MIME → modality mapping. Returns empty for unsupported types so the caller can decide + * whether to throw, fall back, or skip. + */ + public static Optional classify(String mimeType) { + if (StringUtils.isBlank(mimeType)) { + return Optional.empty(); + } + final var normalised = mimeType.toLowerCase(Locale.ROOT).trim(); + final var bareType = stripParameters(normalised); + + if (bareType.startsWith("text/") || TEXT_MIME_TYPES.contains(bareType)) { + return Optional.of(Modality.TEXT); + } + if (IMAGE_MIME_TYPES.contains(bareType)) { + return Optional.of(Modality.IMAGE); + } + if ("application/pdf".equals(bareType)) { + return Optional.of(Modality.PDF); + } + if (bareType.startsWith("audio/")) { + return Optional.of(Modality.AUDIO); + } + if (bareType.startsWith("video/")) { + return Optional.of(Modality.VIDEO); + } + return Optional.empty(); + } + + private static String mimeTypeOf(Document document) { + return Optional.ofNullable(document.metadata()) + .map(DocumentMetadata::getContentType) + .orElse(null); + } + + private static String stripParameters(String mimeType) { + final var semicolon = mimeType.indexOf(';'); + return semicolon >= 0 ? mimeType.substring(0, semicolon).trim() : mimeType; + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java index 9ad4d366082..f37a94b78e1 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java @@ -13,6 +13,9 @@ import com.openai.client.OpenAIClient; import com.openai.models.chat.completions.ChatCompletion; import com.openai.models.chat.completions.ChatCompletionAssistantMessageParam; +import com.openai.models.chat.completions.ChatCompletionContentPart; +import com.openai.models.chat.completions.ChatCompletionContentPartImage; +import com.openai.models.chat.completions.ChatCompletionContentPartText; import com.openai.models.chat.completions.ChatCompletionCreateParams; import com.openai.models.chat.completions.ChatCompletionMessageFunctionToolCall; import com.openai.models.chat.completions.ChatCompletionMessageToolCall; @@ -24,6 +27,7 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.multimodal.DocumentModality; import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.agenticai.model.message.AssistantMessage; import io.camunda.connector.agenticai.model.message.StopReason; @@ -31,10 +35,12 @@ import io.camunda.connector.agenticai.model.message.ToolCallResultMessage; import io.camunda.connector.agenticai.model.message.UserMessage; import io.camunda.connector.agenticai.model.message.content.Content; +import io.camunda.connector.agenticai.model.message.content.DocumentContent; import io.camunda.connector.agenticai.model.message.content.ObjectContent; import io.camunda.connector.agenticai.model.message.content.TextContent; import io.camunda.connector.agenticai.model.tool.ToolCall; import io.camunda.connector.agenticai.model.tool.ToolCallResult; +import io.camunda.connector.api.document.Document; import io.camunda.connector.api.error.ConnectorException; import java.time.ZonedDateTime; import java.util.ArrayList; @@ -121,7 +127,7 @@ private ChatCompletionCreateParams buildParams(ChatRequest request, ChatOptions for (var message : messages) { switch (message) { case SystemMessage system -> builder.addSystemMessage(systemPrompt(system)); - case UserMessage user -> builder.addUserMessage(extractText(user.content())); + case UserMessage user -> addUserMessage(builder, user); case AssistantMessage assistant -> builder.addMessage(toAssistantParam(assistant)); case ToolCallResultMessage toolResults -> addToolResultMessages(builder, toolResults); default -> @@ -151,6 +157,94 @@ private static String systemPrompt(SystemMessage system) { return extractText(system.content()); } + /** + * Routes a {@link UserMessage} onto either the legacy text-only {@code addUserMessage(String)} + * path or the multimodal {@code addUserMessageOfArrayOfContentParts(...)} path. The latter is + * used when the message contains at least one {@link DocumentContent} (image / PDF, validated + * upstream by {@code ToolCallResultStrategy}). + */ + private static void addUserMessage(ChatCompletionCreateParams.Builder builder, UserMessage user) { + final var content = user.content(); + if (!hasMultimodalContent(content)) { + builder.addUserMessage(extractText(content)); + return; + } + final var parts = new ArrayList(); + for (var c : content) { + switch (c) { + case TextContent t -> + parts.add( + ChatCompletionContentPart.ofText( + ChatCompletionContentPartText.builder().text(t.text()).build())); + case ObjectContent o -> + parts.add( + ChatCompletionContentPart.ofText( + ChatCompletionContentPartText.builder() + .text(String.valueOf(o.content())) + .build())); + case DocumentContent doc -> parts.add(documentPart(doc.document())); + default -> + throw new IllegalArgumentException( + "Unsupported content block for OpenAI Chat Completions user message: " + + c.getClass().getSimpleName()); + } + } + builder.addUserMessageOfArrayOfContentParts(parts); + } + + private static boolean hasMultimodalContent(List content) { + if (content == null) { + return false; + } + for (var c : content) { + if (c instanceof DocumentContent) { + return true; + } + } + return false; + } + + private static ChatCompletionContentPart documentPart(Document document) { + final var modality = DocumentModality.of(document); + return switch (modality) { + case IMAGE -> + ChatCompletionContentPart.ofImageUrl( + ChatCompletionContentPartImage.builder() + .imageUrl( + ChatCompletionContentPartImage.ImageUrl.builder() + .url(toDataUrl(document)) + .detail(ChatCompletionContentPartImage.ImageUrl.Detail.AUTO) + .build()) + .build()); + case PDF -> + ChatCompletionContentPart.ofFile( + ChatCompletionContentPart.File.builder() + .file( + ChatCompletionContentPart.File.FileObject.builder() + .fileData(toDataUrl(document)) + .filename(safeFilename(document)) + .build()) + .build()); + default -> + throw new IllegalArgumentException( + "Document modality " + + modality + + " is not supported in OpenAI Chat Completions user-message content (only image " + + "+ PDF emit natively); the strategy should have rejected this user-message " + + "document."); + }; + } + + private static String toDataUrl(Document document) { + final var mimeType = document.metadata().getContentType(); + return "data:" + mimeType + ";base64," + document.asBase64(); + } + + private static String safeFilename(Document document) { + final var name = document.metadata() != null ? document.metadata().getFileName() : null; + return StringUtils.isNotBlank(name) ? name : "document"; + } + private static String extractText(List content) { if (content == null || content.isEmpty()) { return ""; diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApi.java index 5823eb1821e..92d05d41ce9 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApi.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApi.java @@ -14,8 +14,16 @@ import com.openai.models.responses.EasyInputMessage; import com.openai.models.responses.Response; import com.openai.models.responses.ResponseCreateParams; +import com.openai.models.responses.ResponseFunctionCallOutputItem; import com.openai.models.responses.ResponseFunctionToolCall; +import com.openai.models.responses.ResponseInputContent; +import com.openai.models.responses.ResponseInputFile; +import com.openai.models.responses.ResponseInputFileContent; +import com.openai.models.responses.ResponseInputImage; +import com.openai.models.responses.ResponseInputImageContent; import com.openai.models.responses.ResponseInputItem; +import com.openai.models.responses.ResponseInputText; +import com.openai.models.responses.ResponseInputTextContent; import com.openai.models.responses.ResponseOutputItem; import com.openai.models.responses.ResponseOutputMessage; import com.openai.models.responses.ResponseUsage; @@ -25,6 +33,7 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.multimodal.DocumentModality; import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.agenticai.model.message.AssistantMessage; import io.camunda.connector.agenticai.model.message.StopReason; @@ -32,10 +41,12 @@ import io.camunda.connector.agenticai.model.message.ToolCallResultMessage; import io.camunda.connector.agenticai.model.message.UserMessage; import io.camunda.connector.agenticai.model.message.content.Content; +import io.camunda.connector.agenticai.model.message.content.DocumentContent; import io.camunda.connector.agenticai.model.message.content.ObjectContent; import io.camunda.connector.agenticai.model.message.content.TextContent; import io.camunda.connector.agenticai.model.tool.ToolCall; import io.camunda.connector.agenticai.model.tool.ToolCallResult; +import io.camunda.connector.api.document.Document; import io.camunda.connector.api.error.ConnectorException; import java.time.ZonedDateTime; import java.util.ArrayList; @@ -122,13 +133,7 @@ private ResponseCreateParams buildParams(ChatRequest request, ChatOptions option for (var message : messages) { switch (message) { case SystemMessage system -> builder.instructions(extractText(system.content())); - case UserMessage user -> - inputItems.add( - ResponseInputItem.ofEasyInputMessage( - EasyInputMessage.builder() - .role(EasyInputMessage.Role.USER) - .content(extractText(user.content())) - .build())); + case UserMessage user -> inputItems.add(toUserInputItem(user)); case AssistantMessage assistant -> addAssistantInputItems(inputItems, assistant); case ToolCallResultMessage toolResults -> addToolResultInputItems(inputItems, toolResults); @@ -216,17 +221,166 @@ private ResponseInputItem.FunctionCallOutput toFunctionCallOutput(ToolCallResult if (result.id() != null) { b.callId(result.id()); } - final var content = result.content(); - if (content == null) { - b.output(ToolCallResult.CONTENT_NO_RESULT); - } else if (content instanceof String s) { - b.output(s); + final var inlineItems = toolResultMultimodalItems(result); + if (inlineItems != null) { + b.outputOfResponseFunctionCallOutputItemList(inlineItems); } else { - b.output(toJsonString(content)); + final var content = result.content(); + if (content == null) { + b.output(ToolCallResult.CONTENT_NO_RESULT); + } else if (content instanceof String s) { + b.output(s); + } else { + b.output(toJsonString(content)); + } } return b.build(); } + /** + * Builds the {@code [text, image, file]} list for a tool result whose {@code contentBlocks} were + * populated by the strategy with inline-supported documents. Returns null when no inline routing + * happened — caller falls back to the string content path. + * + *

      Shape: a single text item with the serialised tool-result content (JSON / string), followed + * by one image / file item per inline document. + */ + private List toolResultMultimodalItems(ToolCallResult result) { + if (result.contentBlocks() == null || result.contentBlocks().isEmpty()) { + return null; + } + final var items = new ArrayList(); + items.add( + ResponseFunctionCallOutputItem.ofInputText( + ResponseInputTextContent.builder().text(serializedToolResultText(result)).build())); + for (var block : result.contentBlocks()) { + if (!(block instanceof DocumentContent doc)) { + throw new IllegalArgumentException( + "Unsupported inline tool-result content block type: " + + block.getClass().getSimpleName()); + } + final var modality = DocumentModality.of(doc.document()); + switch (modality) { + case IMAGE -> + items.add( + ResponseFunctionCallOutputItem.ofInputImage( + ResponseInputImageContent.builder() + .imageUrl(toDataUrl(doc.document())) + .detail(ResponseInputImageContent.Detail.AUTO) + .build())); + case PDF -> + items.add( + ResponseFunctionCallOutputItem.ofInputFile( + ResponseInputFileContent.builder() + .fileData(toDataUrl(doc.document())) + .filename(safeFilename(doc.document())) + .build())); + default -> + throw new IllegalArgumentException( + "Document modality " + + modality + + " is not supported in OpenAI Responses tool-result content (only image + PDF " + + "emit natively); the strategy should have routed this document to a synthetic " + + "UserMessage."); + } + } + return items; + } + + private String serializedToolResultText(ToolCallResult result) { + final var content = result.content(); + if (content == null) { + return ToolCallResult.CONTENT_NO_RESULT; + } + return content instanceof String s ? s : toJsonString(content); + } + + /** + * Builds the {@link ResponseInputItem} for a user message. Pure-text messages keep the legacy + * {@code content(String)} path; messages with multimodal content blocks (image / PDF, validated + * by the strategy) emit a {@code List} on the same {@code + * EasyInputMessage}. + */ + private ResponseInputItem toUserInputItem(UserMessage user) { + final var content = user.content(); + if (!hasMultimodalContent(content)) { + return ResponseInputItem.ofEasyInputMessage( + EasyInputMessage.builder() + .role(EasyInputMessage.Role.USER) + .content(extractText(content)) + .build()); + } + final var items = new ArrayList(); + for (var c : content) { + switch (c) { + case TextContent t -> + items.add( + ResponseInputContent.ofInputText( + ResponseInputText.builder().text(t.text()).build())); + case ObjectContent o -> + items.add( + ResponseInputContent.ofInputText( + ResponseInputText.builder().text(String.valueOf(o.content())).build())); + case DocumentContent doc -> items.add(documentInputContent(doc.document())); + default -> + throw new IllegalArgumentException( + "Unsupported content block for OpenAI Responses user message: " + + c.getClass().getSimpleName()); + } + } + return ResponseInputItem.ofEasyInputMessage( + EasyInputMessage.builder() + .role(EasyInputMessage.Role.USER) + .contentOfResponseInputMessageContentList(items) + .build()); + } + + private static boolean hasMultimodalContent(List content) { + if (content == null) { + return false; + } + for (var c : content) { + if (c instanceof DocumentContent) { + return true; + } + } + return false; + } + + private static ResponseInputContent documentInputContent(Document document) { + final var modality = DocumentModality.of(document); + return switch (modality) { + case IMAGE -> + ResponseInputContent.ofInputImage( + ResponseInputImage.builder() + .imageUrl(toDataUrl(document)) + .detail(ResponseInputImage.Detail.AUTO) + .build()); + case PDF -> + ResponseInputContent.ofInputFile( + ResponseInputFile.builder() + .fileData(toDataUrl(document)) + .filename(safeFilename(document)) + .build()); + default -> + throw new IllegalArgumentException( + "Document modality " + + modality + + " is not supported in OpenAI Responses user message content (only image + PDF " + + "emit natively); the strategy should have rejected this user-message document."); + }; + } + + private static String toDataUrl(Document document) { + final var mimeType = document.metadata().getContentType(); + return "data:" + mimeType + ";base64," + document.asBase64(); + } + + private static String safeFilename(Document document) { + final var name = document.metadata() != null ? document.metadata().getFileName() : null; + return StringUtils.isNotBlank(name) ? name : "document"; + } + private String toJsonString(Object value) { try { return objectMapper.writeValueAsString(value); diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategy.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategy.java new file mode 100644 index 00000000000..22ea882038e --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategy.java @@ -0,0 +1,56 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.strategy; + +import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.model.message.UserMessage; +import java.util.List; + +/** + * Routes every {@code Document} found in a {@link ChatRequest} to one of two outcomes, driven by + * the resolved {@link ModelCapabilities}: + * + *

        + *
      • Tool-result documents are routed against {@link ModelCapabilities#toolResultModalities()}. + * Inline-supported modalities are appended to {@code ToolCallResult.contentBlocks} for the + * impl to emit natively. Unsupported modalities fall back to a synthetic {@link UserMessage} + * carrying the binary, anchored to the originating tool-result message (PR #6999 shape: + * header text + per-document XML correlation tag + {@code DocumentContent} block). + *
      • User-message and event-message documents are routed against {@link + * ModelCapabilities#userMessageModalities()}. Supported modalities stay inline; unsupported + * modalities fail loud with {@code ConnectorException} (no synthesis fallback for user + * messages, mirroring L4J's {@code DocumentConversionException} semantics). + *
      + * + *

      Implementations must be a pure function over {@code (request, capabilities)}. The returned + * {@link Result#request()} is the wire form to dispatch (synthetic context messages already + * interleaved at their anchor positions); {@link Result#syntheticContextMessages()} duplicates + * those synthetic messages so {@code ChatClientImpl} can persist them into {@code RuntimeMemory} at + * the matching positions. + * + *

      Part of the ADR-005 Phase E SPI scaffolding. + */ +public interface ToolCallResultStrategy { + + /** + * Single-pass walk over {@code request.messages()}. Routes every document; rebuilds tool-result + * messages with populated {@code contentBlocks} where inline; emits synthetic context messages + * for fallback documents; throws {@code ConnectorException} on user-message capability mismatch. + */ + Result apply(ChatRequest request, ModelCapabilities capabilities); + + /** + * @param request rewritten request — tool-result {@code contentBlocks} populated for inline + * documents; synthetic context messages interleaved at their anchor positions. + * @param syntheticContextMessages synthetic {@link UserMessage}s carrying fallback documents + * (with {@code METADATA_TOOL_CALL_DOCUMENTS=true}). Duplicates the entries already present in + * {@code request.messages()}; surfaced separately so the caller can insert them into {@code + * RuntimeMemory} at the matching positions. + */ + record Result(ChatRequest request, List syntheticContextMessages) {} +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategyImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategyImpl.java new file mode 100644 index 00000000000..e7aec95a845 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategyImpl.java @@ -0,0 +1,188 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.strategy; + +import static io.camunda.connector.agenticai.aiagent.agent.AgentErrorCodes.ERROR_CODE_FAILED_MODEL_CALL; +import static io.camunda.connector.agenticai.model.message.content.DocumentContent.documentContent; +import static io.camunda.connector.agenticai.model.message.content.TextContent.textContent; + +import io.camunda.connector.agenticai.aiagent.agent.ToolCallResultDocumentExtractor; +import io.camunda.connector.agenticai.aiagent.agent.ToolCallResultDocumentExtractor.ToolCallDocuments; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.multimodal.DocumentModality; +import io.camunda.connector.agenticai.model.message.DocumentXmlTag; +import io.camunda.connector.agenticai.model.message.Message; +import io.camunda.connector.agenticai.model.message.ToolCallResultMessage; +import io.camunda.connector.agenticai.model.message.UserMessage; +import io.camunda.connector.agenticai.model.message.content.Content; +import io.camunda.connector.agenticai.model.message.content.DocumentContent; +import io.camunda.connector.agenticai.model.tool.ToolCallResult; +import io.camunda.connector.api.document.Document; +import io.camunda.connector.api.error.ConnectorException; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ToolCallResultStrategyImpl implements ToolCallResultStrategy { + + private static final Logger LOGGER = LoggerFactory.getLogger(ToolCallResultStrategyImpl.class); + + private static final String FALLBACK_HEADER = "Documents extracted from tool call results:"; + + private final ToolCallResultDocumentExtractor documentExtractor; + + public ToolCallResultStrategyImpl(ToolCallResultDocumentExtractor documentExtractor) { + this.documentExtractor = documentExtractor; + } + + @Override + public Result apply(ChatRequest request, ModelCapabilities capabilities) { + final var rewrittenMessages = new ArrayList(request.messages().size()); + final var syntheticContextMessages = new ArrayList(); + + for (var message : request.messages()) { + switch (message) { + case ToolCallResultMessage toolCallResultMessage -> { + final var routed = routeToolCallResultMessage(toolCallResultMessage, capabilities); + rewrittenMessages.add(routed.message()); + if (routed.synthetic() != null) { + rewrittenMessages.add(routed.synthetic()); + syntheticContextMessages.add(routed.synthetic()); + } + } + case UserMessage userMessage -> { + validateUserMessageModalities(userMessage, capabilities); + rewrittenMessages.add(userMessage); + } + default -> rewrittenMessages.add(message); + } + } + + return new Result( + new ChatRequest(rewrittenMessages, request.toolDefinitions(), request.responseFormat()), + List.copyOf(syntheticContextMessages)); + } + + private record RoutedToolCallResultMessage( + ToolCallResultMessage message, UserMessage synthetic) {} + + private RoutedToolCallResultMessage routeToolCallResultMessage( + ToolCallResultMessage toolCallResultMessage, ModelCapabilities capabilities) { + + final var extractedByToolCall = + documentExtractor.extractDocuments(toolCallResultMessage.results()); + if (extractedByToolCall.isEmpty()) { + return new RoutedToolCallResultMessage(toolCallResultMessage, null); + } + + // Index extracted documents by tool call id for O(1) lookup while iterating results in order. + final var docsByToolCallId = new HashMap(); + for (var entry : extractedByToolCall) { + docsByToolCallId.put(entry.toolCallId(), entry); + } + + final var fallbackEntries = new ArrayList(); + final var rewrittenResults = + new ArrayList(toolCallResultMessage.results().size()); + + for (var result : toolCallResultMessage.results()) { + final var entry = docsByToolCallId.get(result.id()); + if (entry == null) { + rewrittenResults.add(result); + continue; + } + + final var inline = new ArrayList(); + final var fallback = new ArrayList(); + for (var doc : entry.documents()) { + final var modality = DocumentModality.of(doc); + if (capabilities.toolResultModalities().contains(modality)) { + inline.add(doc); + } else { + fallback.add(doc); + } + } + + rewrittenResults.add(inline.isEmpty() ? result : appendInlineContentBlocks(result, inline)); + if (!fallback.isEmpty()) { + fallbackEntries.add( + new ToolCallDocuments(entry.toolCallId(), entry.toolCallName(), fallback)); + } + } + + final var rewrittenMessage = + ToolCallResultMessage.builder() + .results(rewrittenResults) + .metadata(toolCallResultMessage.metadata()) + .build(); + final var synthetic = + fallbackEntries.isEmpty() ? null : buildSyntheticUserMessage(fallbackEntries); + + return new RoutedToolCallResultMessage(rewrittenMessage, synthetic); + } + + private static ToolCallResult appendInlineContentBlocks( + ToolCallResult result, List inlineDocs) { + final var blocks = new ArrayList(); + if (result.contentBlocks() != null) { + blocks.addAll(result.contentBlocks()); + } + for (var doc : inlineDocs) { + blocks.add(documentContent(doc)); + } + return result.withContentBlocks(blocks); + } + + private static UserMessage buildSyntheticUserMessage(List fallbackEntries) { + final var content = new ArrayList(); + content.add(textContent(FALLBACK_HEADER)); + for (var entry : fallbackEntries) { + for (var doc : entry.documents()) { + content.add( + textContent( + DocumentXmlTag.from(doc, entry.toolCallId(), entry.toolCallName()).toXml())); + content.add(documentContent(doc)); + } + } + + final Map metadata = new HashMap<>(); + metadata.put("timestamp", ZonedDateTime.now()); + metadata.put(UserMessage.METADATA_TOOL_CALL_DOCUMENTS, true); + + return UserMessage.builder().content(content).metadata(metadata).build(); + } + + private void validateUserMessageModalities( + UserMessage userMessage, ModelCapabilities capabilities) { + if (userMessage.content() == null) { + return; + } + for (var contentBlock : userMessage.content()) { + if (!(contentBlock instanceof DocumentContent documentContent)) { + continue; + } + final var doc = documentContent.document(); + final var modality = DocumentModality.of(doc); + if (!capabilities.userMessageModalities().contains(modality)) { + throw new ConnectorException( + ERROR_CODE_FAILED_MODEL_CALL, + "Document with reference '%s' has modality %s but the resolved model does not " + .formatted(doc.reference(), modality) + + "support that modality in user messages (supported: %s)." + .formatted(capabilities.userMessageModalities())); + } + } + LOGGER.trace( + "Validated user message modalities against capabilities {}", + capabilities.userMessageModalities()); + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java index 32ad93c0c13..45b58f6b470 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfiguration.java @@ -49,6 +49,8 @@ import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelHttpProxySupport; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.configuration.AgenticAiLangchain4JFrameworkConfiguration; import io.camunda.connector.agenticai.aiagent.framework.openai.OpenAiChatModelApiConfiguration; +import io.camunda.connector.agenticai.aiagent.framework.strategy.ToolCallResultStrategy; +import io.camunda.connector.agenticai.aiagent.framework.strategy.ToolCallResultStrategyImpl; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStore; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStoreRegistry; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStoreRegistryImpl; @@ -251,11 +253,8 @@ public ToolCallResultDocumentExtractor toolCallResultDocumentExtractor( @Bean @ConditionalOnMissingBean public AgentMessagesHandler aiAgentMessagesHandler( - GatewayToolHandlerRegistry gatewayToolHandlers, - SystemPromptComposer systemPromptComposer, - ToolCallResultDocumentExtractor documentExtractor) { - return new AgentMessagesHandlerImpl( - gatewayToolHandlers, systemPromptComposer, documentExtractor); + GatewayToolHandlerRegistry gatewayToolHandlers, SystemPromptComposer systemPromptComposer) { + return new AgentMessagesHandlerImpl(gatewayToolHandlers, systemPromptComposer); } @Bean @@ -274,8 +273,16 @@ public ChatModelApiRegistry aiAgentChatModelApiRegistry( @Bean @ConditionalOnMissingBean - public ChatClient aiAgentChatClient(ChatModelApiRegistry chatModelApiRegistry) { - return new ChatClientImpl(chatModelApiRegistry); + public ToolCallResultStrategy aiAgentToolCallResultStrategy( + ToolCallResultDocumentExtractor documentExtractor) { + return new ToolCallResultStrategyImpl(documentExtractor); + } + + @Bean + @ConditionalOnMissingBean + public ChatClient aiAgentChatClient( + ChatModelApiRegistry chatModelApiRegistry, ToolCallResultStrategy toolCallResultStrategy) { + return new ChatClientImpl(chatModelApiRegistry, toolCallResultStrategy); } @Bean diff --git a/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml b/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml index ed522a06856..c70579cd89c 100644 --- a/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml +++ b/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml @@ -105,7 +105,7 @@ camunda: openai-completions: defaults: input-modalities: - user-message: [text, image] + user-message: [text, image, pdf] tool-result: [text] output-modalities: assistant-message: [text] @@ -161,7 +161,7 @@ camunda: openai-responses: defaults: input-modalities: - user-message: [text, image] + user-message: [text, image, pdf] tool-result: [text, image, pdf] output-modalities: assistant-message: [text] diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java index 7b349f5ea0d..f2369061219 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/agent/AgentMessagesHandlerTest.java @@ -98,11 +98,7 @@ void setUp() { // No stub for handlerForToolDefinition: the mocked registry returns Optional.empty() by // default, so the extractor falls back to the generic content-tree walker for every result. // Individual tests can override this when a gateway-handler-specific extraction is needed. - messagesHandler = - new AgentMessagesHandlerImpl( - gatewayToolHandlers, - systemPromptComposer, - new ToolCallResultDocumentExtractor(gatewayToolHandlers)); + messagesHandler = new AgentMessagesHandlerImpl(gatewayToolHandlers, systemPromptComposer); runtimeMemory = spy(new DefaultRuntimeMemory()); } @@ -786,11 +782,13 @@ void interruptsToolCallsOnEventResultsWhenEventContentIsEmpty(Object eventConten } @Test - void createsDocumentUserMessageWhenToolResultsContainDocuments() { + void doesNotEmitSyntheticUserMessageEvenWhenToolResultsContainDocuments() { + // Document extraction is the responsibility of ToolCallResultStrategy at the ChatClient + // boundary (ADR-005 Phase E3). AgentMessagesHandlerImpl no longer creates synthetic + // UserMessages — it adds only the ToolCallResultMessage. Strategy-side assertions live in + // ToolCallResultStrategyImplTest. final var doc1 = createDocument("weather data", "text/plain", "weather.txt"); final var doc2 = createDocument("time report", "application/pdf", "report.pdf"); - final var shortId1 = documentShortId(doc1); - final var shortId2 = documentShortId(doc2); final var toolCallResultsWithDocs = List.of( @@ -820,45 +818,7 @@ void createsDocumentUserMessageWhenToolResultsContainDocuments() { userPromptWithDocuments, toolCallResultsWithDocs); - assertThat(addedMessages) - .hasSize(2) - .satisfiesExactly( - message -> assertThat(message).isInstanceOf(ToolCallResultMessage.class), - message -> - assertThat(message) - .isInstanceOfSatisfying( - UserMessage.class, - userMessage -> { - assertThat(userMessage.metadata()) - .containsEntry(UserMessage.METADATA_TOOL_CALL_DOCUMENTS, true) - .containsKey("timestamp"); - assertThat(userMessage.content()) - .hasSize(5) - .satisfiesExactly( - c -> - assertThat(c) - .isEqualTo( - textContent( - "Documents extracted from tool call results:")), - c -> - assertThat(c) - .isEqualTo( - textContent( - "" - .formatted(shortId1))), - c -> - assertThat(c) - .isEqualTo(DocumentContent.documentContent(doc1)), - c -> - assertThat(c) - .isEqualTo( - textContent( - "" - .formatted(shortId2))), - c -> - assertThat(c) - .isEqualTo(DocumentContent.documentContent(doc2))); - })); + assertThat(addedMessages).hasSize(1).first().isInstanceOf(ToolCallResultMessage.class); } @Test @@ -883,12 +843,15 @@ void doesNotCreateDocumentUserMessageWhenNoDocumentsInToolResults() { } @Test - void ordersDocumentUserMessageBetweenToolResultsAndEvents() { + void preservesToolResultThenEventOrderWithoutSyntheticUserMessage() { + // After ADR-005 E3, AgentMessagesHandler emits [ToolCallResultMessage, EventMessages] + // only. ToolCallResultStrategy (ChatClient boundary) inserts the synthetic + // toolCallDocuments=true UserMessage between them at chat time; that path is covered + // in ChatClientImplTest + ToolCallResultStrategyImplTest. when(executionContext.events()) .thenReturn(new EventHandlingConfiguration(WAIT_FOR_TOOL_CALL_RESULTS)); final var doc = createDocument("weather data", "text/plain", "weather.txt"); - final var shortId = documentShortId(doc); final var toolCallResultsWithDocsAndEvents = List.of( ToolCallResult.builder() @@ -914,33 +877,10 @@ void ordersDocumentUserMessageBetweenToolResultsAndEvents() { userPromptWithDocuments, toolCallResultsWithDocsAndEvents); - // order: ToolCallResultMessage -> document UserMessage -> event UserMessage assertThat(addedMessages) - .hasSize(3) + .hasSize(2) .satisfiesExactly( message -> assertThat(message).isInstanceOf(ToolCallResultMessage.class), - message -> - assertThat(message) - .isInstanceOfSatisfying( - UserMessage.class, - um -> - assertThat(um.content()) - .hasSize(3) - .satisfiesExactly( - c -> - assertThat(c) - .isEqualTo( - textContent( - "Documents extracted from tool call results:")), - c -> - assertThat(c) - .isEqualTo( - textContent( - "" - .formatted(shortId))), - c -> - assertThat(c) - .isEqualTo(DocumentContent.documentContent(doc)))), message -> assertThat(message) .isInstanceOfSatisfying( diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImplTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImplTest.java index 191ef611710..9af20b10f8e 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImplTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImplTest.java @@ -20,6 +20,9 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; +import io.camunda.connector.agenticai.aiagent.framework.strategy.ToolCallResultStrategy; import io.camunda.connector.agenticai.aiagent.memory.runtime.DefaultRuntimeMemory; import io.camunda.connector.agenticai.aiagent.memory.runtime.RuntimeMemory; import io.camunda.connector.agenticai.aiagent.model.AgentContext; @@ -67,9 +70,22 @@ class ChatClientImplTest { assistantMessage("hello world") .withUsage(TokenUsage.builder().inputTokenCount(10).outputTokenCount(20).build()); + private static final ModelCapabilities TEXT_ONLY_CAPABILITIES = + new ModelCapabilities( + List.of(Modality.TEXT), + List.of(Modality.TEXT), + List.of(Modality.TEXT), + false, + false, + false, + false, + null, + null); + @Mock private ChatModelApiRegistry registry; @Mock private ChatModelApi chatModelApi; @Mock private AgentExecutionContext executionContext; + @Mock private ToolCallResultStrategy strategy; @Captor private ArgumentCaptor requestCaptor; @Captor private ArgumentCaptor optionsCaptor; @@ -84,10 +100,13 @@ void setUp() { when(executionContext.provider()).thenReturn(PROVIDER_CONFIG); when(registry.resolve(PROVIDER_CONFIG)).thenReturn(chatModelApi); + when(chatModelApi.capabilities()).thenReturn(TEXT_ONLY_CAPABILITIES); when(chatModelApi.complete(requestCaptor.capture(), optionsCaptor.capture(), any())) .thenReturn(CompletableFuture.completedFuture(new ChatResponse(ASSISTANT_MESSAGE))); + when(strategy.apply(any(), any())) + .thenAnswer(inv -> new ToolCallResultStrategy.Result(inv.getArgument(0), List.of())); - chatClient = new ChatClientImpl(registry); + chatClient = new ChatClientImpl(registry, strategy); } @Test diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java index 025be699f4e..9240d72ed52 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java @@ -79,7 +79,15 @@ class AnthropicMessagesChatModelApiTest { void setUp() { when(client.messages()).thenReturn(messageService); api = - new AnthropicMessagesChatModelApi(client, MODEL_ID, CAPABILITIES, 1024L, null, null, null); + new AnthropicMessagesChatModelApi( + client, + MODEL_ID, + new com.fasterxml.jackson.databind.ObjectMapper(), + CAPABILITIES, + 1024L, + null, + null, + null); } @Test diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixOverrideTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixOverrideTest.java index 67002e340a8..68983398944 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixOverrideTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixOverrideTest.java @@ -70,7 +70,7 @@ void overrideAddsBrandNewModelEntryUnderExistingFamily() { assertThat(caps.userMessageModalities()) .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); assertThat(caps.toolResultModalities()) - .containsExactly(Modality.TEXT, Modality.IMAGE); + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); }); } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategyImplTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategyImplTest.java new file mode 100644 index 00000000000..0afc6d77918 --- /dev/null +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategyImplTest.java @@ -0,0 +1,270 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.strategy; + +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.assistantMessage; +import static io.camunda.connector.agenticai.aiagent.TestMessagesFixture.userMessage; +import static io.camunda.connector.agenticai.model.message.content.DocumentContent.documentContent; +import static io.camunda.connector.agenticai.model.message.content.TextContent.textContent; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import io.camunda.connector.agenticai.aiagent.agent.ToolCallResultDocumentExtractor; +import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities.Modality; +import io.camunda.connector.agenticai.aiagent.tool.GatewayToolHandlerRegistry; +import io.camunda.connector.agenticai.model.message.Message; +import io.camunda.connector.agenticai.model.message.ToolCallResultMessage; +import io.camunda.connector.agenticai.model.message.UserMessage; +import io.camunda.connector.agenticai.model.message.content.DocumentContent; +import io.camunda.connector.agenticai.model.tool.ToolCallResult; +import io.camunda.connector.agenticai.model.tool.ToolDefinition; +import io.camunda.connector.api.document.Document; +import io.camunda.connector.api.document.DocumentCreationRequest; +import io.camunda.connector.api.error.ConnectorException; +import io.camunda.connector.runtime.core.document.DocumentFactoryImpl; +import io.camunda.connector.runtime.core.document.store.InMemoryDocumentStore; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +class ToolCallResultStrategyImplTest { + + private static final InMemoryDocumentStore DOCUMENT_STORE = InMemoryDocumentStore.INSTANCE; + private static final DocumentFactoryImpl DOCUMENT_FACTORY = + new DocumentFactoryImpl(DOCUMENT_STORE); + + private static final ModelCapabilities INLINE_IMAGE_PDF = + capabilities( + List.of(Modality.TEXT, Modality.IMAGE, Modality.PDF), + List.of(Modality.TEXT, Modality.IMAGE, Modality.PDF)); + private static final ModelCapabilities INLINE_IMAGE_ONLY = + capabilities(List.of(Modality.TEXT, Modality.IMAGE), List.of(Modality.TEXT, Modality.IMAGE)); + private static final ModelCapabilities TEXT_ONLY = + capabilities(List.of(Modality.TEXT), List.of(Modality.TEXT)); + + private GatewayToolHandlerRegistry gatewayToolHandlers; + private ToolCallResultStrategyImpl strategy; + + @BeforeEach + void setUp() { + DOCUMENT_STORE.clear(); + gatewayToolHandlers = Mockito.mock(GatewayToolHandlerRegistry.class); + Mockito.when(gatewayToolHandlers.handlerForToolDefinition(Mockito.any())) + .thenReturn(java.util.Optional.empty()); + strategy = + new ToolCallResultStrategyImpl(new ToolCallResultDocumentExtractor(gatewayToolHandlers)); + } + + @Nested + class ToolResultRouting { + + @Test + void inlineImageStaysOnContentBlocksAndProducesNoSyntheticMessage() { + final var doc = createDocument("img-bytes", "image/png", "img.png"); + final var request = requestWithToolResultDocument("call-1", "lookup", doc); + + final var result = strategy.apply(request, INLINE_IMAGE_PDF); + + assertThat(result.syntheticContextMessages()).isEmpty(); + final var rewrittenResult = singleResult(result.request()); + assertThat(rewrittenResult.contentBlocks()).containsExactly(documentContent(doc)); + } + + @Test + void unsupportedPdfFallsBackToSyntheticUserMessage() { + final var doc = createDocument("pdf-bytes", "application/pdf", "report.pdf"); + final var request = requestWithToolResultDocument("call-1", "lookup", doc); + + final var result = strategy.apply(request, INLINE_IMAGE_ONLY); + + assertThat(result.syntheticContextMessages()).hasSize(1); + final var synthetic = result.syntheticContextMessages().getFirst(); + assertThat(synthetic.metadata()) + .containsEntry(UserMessage.METADATA_TOOL_CALL_DOCUMENTS, true); + assertThat(synthetic.content()) + .anySatisfy(c -> assertThat(c).isEqualTo(documentContent(doc))); + + final var rewrittenResult = singleResult(result.request()); + // No inline routing happened. + assertThat(rewrittenResult.contentBlocks()).isNullOrEmpty(); + } + + @Test + void mixedDocumentsAreSplitInOnePass() { + final var image = createDocument("img-bytes", "image/png", "img.png"); + final var pdf = createDocument("pdf-bytes", "application/pdf", "report.pdf"); + final var request = requestWithToolResultDocuments("call-1", "lookup", image, pdf); + + final var result = strategy.apply(request, INLINE_IMAGE_ONLY); + + // image inline, pdf in synthetic + final var rewrittenResult = singleResult(result.request()); + assertThat(rewrittenResult.contentBlocks()).containsExactly(documentContent(image)); + + assertThat(result.syntheticContextMessages()).hasSize(1); + final var synthetic = result.syntheticContextMessages().getFirst(); + assertThat(synthetic.content()) + .filteredOn(DocumentContent.class::isInstance) + .extracting(c -> ((DocumentContent) c).document()) + .containsExactly(pdf); + } + + @Test + void syntheticMessageIsInsertedRightAfterToolResultMessageInRequest() { + final var pdf = createDocument("pdf-bytes", "application/pdf", "r.pdf"); + final var toolResultMessage = toolResultMessage("call-1", "lookup", pdf); + final var followingUserMessage = userMessage("follow-up event"); + final var initial = + new ChatRequest( + List.of(assistantMessage("call lookup"), toolResultMessage, followingUserMessage), + List.of(), + null); + + final var result = strategy.apply(initial, INLINE_IMAGE_ONLY); + + // Order: AssistantMessage, ToolCallResultMessage, syntheticUM, followingUserMessage + assertThat(result.request().messages()) + .satisfiesExactly( + m -> + assertThat(m) + .isInstanceOf( + io.camunda.connector.agenticai.model.message.AssistantMessage.class), + m -> assertThat(m).isInstanceOf(ToolCallResultMessage.class), + m -> { + assertThat(m).isInstanceOf(UserMessage.class); + assertThat(((UserMessage) m).metadata()) + .containsEntry(UserMessage.METADATA_TOOL_CALL_DOCUMENTS, true); + }, + m -> assertThat(m).isEqualTo(followingUserMessage)); + } + + @Test + void noDocumentsLeavesRequestUnchangedAndProducesNoSynthetic() { + final var request = + new ChatRequest( + List.of( + assistantMessage("call lookup"), + ToolCallResultMessage.builder() + .results( + List.of( + ToolCallResult.builder() + .id("call-1") + .name("lookup") + .content("plain text result") + .build())) + .metadata(Map.of()) + .build()), + List.of(), + null); + + final var result = strategy.apply(request, INLINE_IMAGE_PDF); + + assertThat(result.syntheticContextMessages()).isEmpty(); + assertThat(singleResult(result.request()).contentBlocks()).isNullOrEmpty(); + } + } + + @Nested + class UserMessageValidation { + + @Test + void supportedDocumentsPassThroughUnchanged() { + final var img = createDocument("img-bytes", "image/png", "img.png"); + final var userMsg = + UserMessage.builder() + .content(List.of(textContent("look at this"), documentContent(img))) + .build(); + final var request = new ChatRequest(List.of(userMsg), List.of(), null); + + final var result = strategy.apply(request, INLINE_IMAGE_PDF); + + assertThat(result.request().messages()).containsExactly(userMsg); + } + + @Test + void unsupportedDocumentInUserMessageThrows() { + final var pdf = createDocument("pdf-bytes", "application/pdf", "report.pdf"); + final var userMsg = + UserMessage.builder().content(List.of(textContent("look"), documentContent(pdf))).build(); + final var request = new ChatRequest(List.of(userMsg), List.of(), null); + + assertThatThrownBy(() -> strategy.apply(request, INLINE_IMAGE_ONLY)) + .isInstanceOf(ConnectorException.class) + .hasMessageContaining("PDF") + .hasMessageContaining("does not"); + } + + @Test + void textOnlyUserMessageOnTextOnlyModelPassesThrough() { + final var userMsg = userMessage("hello"); + final var request = new ChatRequest(List.of(userMsg), List.of(), null); + + final var result = strategy.apply(request, TEXT_ONLY); + + assertThat(result.request().messages()).containsExactly(userMsg); + assertThat(result.syntheticContextMessages()).isEmpty(); + } + } + + // ------- helpers ------- + + private static ModelCapabilities capabilities( + List userMessage, List toolResult) { + return new ModelCapabilities( + userMessage, toolResult, List.of(Modality.TEXT), false, false, false, false, null, null); + } + + private static Document createDocument(String content, String contentType, String filename) { + return DOCUMENT_FACTORY.create( + DocumentCreationRequest.from(content.getBytes(StandardCharsets.UTF_8)) + .contentType(contentType) + .fileName(filename) + .build()); + } + + private static ChatRequest requestWithToolResultDocument( + String callId, String toolName, Document doc) { + return requestWithToolResultDocuments(callId, toolName, doc); + } + + private static ChatRequest requestWithToolResultDocuments( + String callId, String toolName, Document... docs) { + return new ChatRequest( + List.of(assistantMessage("call " + toolName), toolResultMessage(callId, toolName, docs)), + List.of(), + null); + } + + private static ToolCallResultMessage toolResultMessage( + String callId, String toolName, Document... docs) { + final Object content = + docs.length == 1 + ? Map.of("file", docs[0]) + : Map.of("files", List.of((Object[]) docs)); + return ToolCallResultMessage.builder() + .results( + List.of(ToolCallResult.builder().id(callId).name(toolName).content(content).build())) + .metadata(Map.of()) + .build(); + } + + private static ToolCallResult singleResult(ChatRequest request) { + for (Message m : request.messages()) { + if (m instanceof ToolCallResultMessage trm) { + assertThat(trm.results()).hasSize(1); + return trm.results().getFirst(); + } + } + throw new AssertionError("No ToolCallResultMessage in request"); + } +} From 0bd7b900cd8fb6797cae038b149662d4e1134083 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 20:35:24 +0200 Subject: [PATCH 61/81] refactor(agentic-ai): rename Modality.PDF to Modality.DOCUMENT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The capability-matrix vocabulary now uses `document` instead of `pdf` for the document-shaped modality. The MIME-level mapping (application/pdf → DOCUMENT) is unchanged, but the enum name no longer ties the modality to one specific document format. Touches the matrix YAML, the SPI enum + classifier, all three native emitters, ADR-005 + plan, and tests. No behavior change. --- .../docs/adr-005-implementation-plan.md | 2 +- .../adr/005-replace-langchain4j-framework.md | 8 ++++---- .../AnthropicMessagesChatModelApi.java | 20 +++++++++---------- .../framework/api/ModelCapabilities.java | 4 ++-- .../multimodal/DocumentModality.java | 6 +++--- .../OpenAiChatCompletionsChatModelApi.java | 12 +++++------ .../openai/OpenAiResponsesChatModelApi.java | 19 +++++++++--------- .../capabilities/model-capabilities.yaml | 12 +++++------ .../AnthropicMessagesChatModelApiTest.java | 2 +- .../BundledCapabilityMatrixTest.java | 12 +++++------ .../CapabilityMatrixOverrideTest.java | 6 +++--- .../ModelCapabilitiesResolverTest.java | 6 +++--- .../OpenAiResponsesChatModelApiTest.java | 2 +- .../ToolCallResultStrategyImplTest.java | 6 +++--- 14 files changed, 59 insertions(+), 58 deletions(-) diff --git a/connectors/agentic-ai/docs/adr-005-implementation-plan.md b/connectors/agentic-ai/docs/adr-005-implementation-plan.md index 028ce30dd4e..beeb45ee46f 100644 --- a/connectors/agentic-ai/docs/adr-005-implementation-plan.md +++ b/connectors/agentic-ai/docs/adr-005-implementation-plan.md @@ -424,7 +424,7 @@ tests stay green without modification. **Tests to add**: - `ToolCallResultStrategyImplTest` — pure-function table-driven cases. Covers: - - Tool-result image with `tool-result: [text, image, pdf]` → `INLINE`, no synthetic. + - Tool-result image with `tool-result: [text, image, document]` → `INLINE`, no synthetic. - Tool-result PDF with `tool-result: [text, image]` → `FALLBACK`, one synthetic UM with one DocumentContent; XML placeholder substituted in tool result body. - Mixed result (one image + one PDF in same tool result, capability `[text, image]`) → diff --git a/connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md b/connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md index f16572e478e..508595a60b2 100644 --- a/connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md +++ b/connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md @@ -301,8 +301,8 @@ camunda.connector.agenticai.aiagent.framework.capabilities: anthropic-messages: defaults: input-modalities: - user-message: [text, image, pdf] - tool-result: [text, image, pdf] + user-message: [text, image, document] + tool-result: [text, image, document] output-modalities: assistant-message: [text] supports-reasoning: false @@ -340,7 +340,7 @@ Each api family carries: so glob patterns always live in the `pattern` field while the map key stays a stable override identifier. -Modality vocabulary: `text | image | pdf | audio | video`. Modality lists per location +Modality vocabulary: `text | image | document | audio | video`. Modality lists per location (`user-message`, `tool-result`, `assistant-message`) are symmetric — every modality at every location has an explicit answer for each model. @@ -411,7 +411,7 @@ camunda.connector.agenticai.aiagent.framework.capabilities: Map-key reuse means a consumer override deep-merges into the bundled entry; a new map key adds a new entry. Modality lists replace wholesale (Spring Boot list semantics) — overriding -`tool-result: [text]` discards the bundled `[text, image, pdf]`. To add a modality, restate +`tool-result: [text]` discards the bundled `[text, image, document]`. To add a modality, restate the full list including the inherited entries. ## Tool Call Result Routing diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java index ea5c0240730..cddf291a3e8 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java @@ -197,7 +197,7 @@ private MessageParam toMessageParam(ToolCallResultMessage message) { } /** - * User-message blocks: text + multimodal documents (image, PDF). The {@link + * User-message blocks: text + multimodal documents (image, document/PDF). The {@link * io.camunda.connector.agenticai.aiagent.framework.strategy.ToolCallResultStrategy} has already * validated user-message documents against the model's {@code userMessageModalities}, so any * {@link DocumentContent} reaching this point is known to be supported. @@ -218,14 +218,14 @@ private static ContentBlockParam messageContentBlock(Content content) { final var modality = DocumentModality.of(doc.document()); return switch (modality) { case IMAGE -> ContentBlockParam.ofImage(imageBlockParam(doc.document())); - case PDF -> ContentBlockParam.ofDocument(pdfBlockParam(doc.document())); + case DOCUMENT -> ContentBlockParam.ofDocument(documentBlockParam(doc.document())); default -> throw new IllegalArgumentException( "Document modality " + modality + " is not supported in Anthropic user/tool messages " - + "(only image + PDF emit natively); the strategy should have routed this " - + "document to a synthetic UserMessage or rejected it."); + + "(only image + document emit natively); the strategy should have routed " + + "this document to a synthetic UserMessage or rejected it."); }; } return textOnlyBlock(content); @@ -270,7 +270,7 @@ private static ImageBlockParam imageBlockParam(Document document) { .build(); } - private static DocumentBlockParam pdfBlockParam(Document document) { + private static DocumentBlockParam documentBlockParam(Document document) { return DocumentBlockParam.builder() .source(Base64PdfSource.builder().data(document.asBase64()).build()) .build(); @@ -353,16 +353,16 @@ private List toolResultContentBlocks(ToolCal switch (modality) { case IMAGE -> blocks.add(ToolResultBlockParam.Content.Block.ofImage(imageBlockParam(doc.document()))); - case PDF -> + case DOCUMENT -> blocks.add( - ToolResultBlockParam.Content.Block.ofDocument(pdfBlockParam(doc.document()))); + ToolResultBlockParam.Content.Block.ofDocument(documentBlockParam(doc.document()))); default -> throw new IllegalArgumentException( "Document modality " + modality - + " is not supported in Anthropic tool result blocks (only image + PDF emit " - + "natively); the strategy should have routed this document to a synthetic " - + "UserMessage."); + + " is not supported in Anthropic tool result blocks (only image + document " + + "emit natively); the strategy should have routed this document to a " + + "synthetic UserMessage."); } } return blocks; diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java index f58e37e0ab7..b2188a2cbf3 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/api/ModelCapabilities.java @@ -35,8 +35,8 @@ public enum Modality { TEXT, @JsonProperty("image") IMAGE, - @JsonProperty("pdf") - PDF, + @JsonProperty("document") + DOCUMENT, @JsonProperty("audio") AUDIO, @JsonProperty("video") diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/multimodal/DocumentModality.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/multimodal/DocumentModality.java index f0347ca3f8e..d82fbd1ff9c 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/multimodal/DocumentModality.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/multimodal/DocumentModality.java @@ -27,8 +27,8 @@ *

      Coverage parity with the LangChain4j path ({@code DocumentToContentConverterImpl}): {@code * text/*}, {@code application/json}, {@code application/xml}, {@code application/yaml} → {@link * Modality#TEXT}; the four common image MIME types → {@link Modality#IMAGE}; {@code - * application/pdf} → {@link Modality#PDF}. Audio + video MIME types map for completeness but no - * emitter consumes them yet (Phase G+). + * application/pdf} → {@link Modality#DOCUMENT}. Audio + video MIME types map for completeness but + * no emitter consumes them yet (Phase G+). */ public final class DocumentModality { @@ -74,7 +74,7 @@ public static Optional classify(String mimeType) { return Optional.of(Modality.IMAGE); } if ("application/pdf".equals(bareType)) { - return Optional.of(Modality.PDF); + return Optional.of(Modality.DOCUMENT); } if (bareType.startsWith("audio/")) { return Optional.of(Modality.AUDIO); diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java index f37a94b78e1..258e637f907 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java @@ -160,8 +160,8 @@ private static String systemPrompt(SystemMessage system) { /** * Routes a {@link UserMessage} onto either the legacy text-only {@code addUserMessage(String)} * path or the multimodal {@code addUserMessageOfArrayOfContentParts(...)} path. The latter is - * used when the message contains at least one {@link DocumentContent} (image / PDF, validated - * upstream by {@code ToolCallResultStrategy}). + * used when the message contains at least one {@link DocumentContent} (image / document, + * validated upstream by {@code ToolCallResultStrategy}). */ private static void addUserMessage(ChatCompletionCreateParams.Builder builder, UserMessage user) { final var content = user.content(); @@ -216,7 +216,7 @@ private static ChatCompletionContentPart documentPart(Document document) { .detail(ChatCompletionContentPartImage.ImageUrl.Detail.AUTO) .build()) .build()); - case PDF -> + case DOCUMENT -> ChatCompletionContentPart.ofFile( ChatCompletionContentPart.File.builder() .file( @@ -229,9 +229,9 @@ private static ChatCompletionContentPart documentPart(Document document) { throw new IllegalArgumentException( "Document modality " + modality - + " is not supported in OpenAI Chat Completions user-message content (only image " - + "+ PDF emit natively); the strategy should have rejected this user-message " - + "document."); + + " is not supported in OpenAI Chat Completions user-message content (only " + + "image + document emit natively); the strategy should have rejected this " + + "user-message document."); }; } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApi.java index 92d05d41ce9..d747d1fa70c 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApi.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApi.java @@ -268,7 +268,7 @@ private List toolResultMultimodalItems(ToolCallR .imageUrl(toDataUrl(doc.document())) .detail(ResponseInputImageContent.Detail.AUTO) .build())); - case PDF -> + case DOCUMENT -> items.add( ResponseFunctionCallOutputItem.ofInputFile( ResponseInputFileContent.builder() @@ -279,9 +279,9 @@ private List toolResultMultimodalItems(ToolCallR throw new IllegalArgumentException( "Document modality " + modality - + " is not supported in OpenAI Responses tool-result content (only image + PDF " - + "emit natively); the strategy should have routed this document to a synthetic " - + "UserMessage."); + + " is not supported in OpenAI Responses tool-result content (only image + " + + "document emit natively); the strategy should have routed this document to " + + "a synthetic UserMessage."); } } return items; @@ -297,8 +297,8 @@ private String serializedToolResultText(ToolCallResult result) { /** * Builds the {@link ResponseInputItem} for a user message. Pure-text messages keep the legacy - * {@code content(String)} path; messages with multimodal content blocks (image / PDF, validated - * by the strategy) emit a {@code List} on the same {@code + * {@code content(String)} path; messages with multimodal content blocks (image / document, + * validated by the strategy) emit a {@code List} on the same {@code * EasyInputMessage}. */ private ResponseInputItem toUserInputItem(UserMessage user) { @@ -356,7 +356,7 @@ private static ResponseInputContent documentInputContent(Document document) { .imageUrl(toDataUrl(document)) .detail(ResponseInputImage.Detail.AUTO) .build()); - case PDF -> + case DOCUMENT -> ResponseInputContent.ofInputFile( ResponseInputFile.builder() .fileData(toDataUrl(document)) @@ -366,8 +366,9 @@ private static ResponseInputContent documentInputContent(Document document) { throw new IllegalArgumentException( "Document modality " + modality - + " is not supported in OpenAI Responses user message content (only image + PDF " - + "emit natively); the strategy should have rejected this user-message document."); + + " is not supported in OpenAI Responses user message content (only image + " + + "document emit natively); the strategy should have rejected this user-message " + + "document."); }; } diff --git a/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml b/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml index c70579cd89c..4a2611f0f17 100644 --- a/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml +++ b/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml @@ -37,7 +37,7 @@ # 3. Pattern (longest-match wins) # 4. Conservative defaults (text-only, all flags false) # -# Modality vocabulary: text | image | pdf | audio | video. Modality lists per +# Modality vocabulary: text | image | document | audio | video. Modality lists per # location (user-message, tool-result, assistant-message) are symmetric. camunda: @@ -50,8 +50,8 @@ camunda: anthropic-messages: defaults: input-modalities: - user-message: [text, image, pdf] - tool-result: [text, image, pdf] + user-message: [text, image, document] + tool-result: [text, image, document] output-modalities: assistant-message: [text] supports-reasoning: false @@ -105,7 +105,7 @@ camunda: openai-completions: defaults: input-modalities: - user-message: [text, image, pdf] + user-message: [text, image, document] tool-result: [text] output-modalities: assistant-message: [text] @@ -161,8 +161,8 @@ camunda: openai-responses: defaults: input-modalities: - user-message: [text, image, pdf] - tool-result: [text, image, pdf] + user-message: [text, image, document] + tool-result: [text, image, document] output-modalities: assistant-message: [text] supports-reasoning: false diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java index 9240d72ed52..ea1abfccfe4 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java @@ -58,7 +58,7 @@ class AnthropicMessagesChatModelApiTest { private static final ModelCapabilities CAPABILITIES = new ModelCapabilities( - List.of(Modality.TEXT, Modality.IMAGE, Modality.PDF), + List.of(Modality.TEXT, Modality.IMAGE, Modality.DOCUMENT), List.of(Modality.TEXT, Modality.IMAGE), List.of(Modality.TEXT), true, diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/BundledCapabilityMatrixTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/BundledCapabilityMatrixTest.java index f90ed4f156f..65351d741ac 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/BundledCapabilityMatrixTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/BundledCapabilityMatrixTest.java @@ -58,9 +58,9 @@ void claudeSonnet4ResolvesToFullCapabilities() { final var caps = resolve(context, "anthropic-messages", "claude-sonnet-4-6"); assertThat(caps.userMessageModalities()) - .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.DOCUMENT); assertThat(caps.toolResultModalities()) - .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.DOCUMENT); assertThat(caps.supportsReasoning()).isTrue(); assertThat(caps.supportsReasoningSignatureRoundtrip()).isTrue(); assertThat(caps.supportsPromptCaching()).isTrue(); @@ -77,7 +77,7 @@ void claudeHaiku4InheritsToolResultButOverridesUserMessage() { assertThat(caps.userMessageModalities()).containsExactly(Modality.TEXT, Modality.IMAGE); assertThat(caps.toolResultModalities()) - .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.DOCUMENT); assertThat(caps.supportsReasoning()).isFalse(); }); } @@ -89,7 +89,7 @@ void unknownClaudeModelFallsThroughToFamilyCatchAll() { final var caps = resolve(context, "anthropic-messages", "claude-some-future-model"); assertThat(caps.userMessageModalities()) - .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.DOCUMENT); assertThat(caps.supportsPromptCaching()).isTrue(); }); } @@ -118,7 +118,7 @@ void gpt5OnResponsesHasReasoningRoundtripAndMultimodalToolResults() { assertThat(caps.supportsReasoning()).isTrue(); assertThat(caps.supportsReasoningSignatureRoundtrip()).isTrue(); assertThat(caps.toolResultModalities()) - .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.DOCUMENT); }); } @@ -132,7 +132,7 @@ void gpt4oAddsAudioToUserMessageButKeepsToolResultFromDefaults() { assertThat(caps.userMessageModalities()) .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.AUDIO); assertThat(caps.toolResultModalities()) - .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.DOCUMENT); assertThat(caps.supportsReasoning()).isFalse(); }); } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixOverrideTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixOverrideTest.java index 68983398944..962a6b1c4e0 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixOverrideTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixOverrideTest.java @@ -48,7 +48,7 @@ void overrideTunesScalarFieldsOnExistingPattern() { // Other bundled fields untouched: assertThat(caps.supportsReasoning()).isTrue(); assertThat(caps.userMessageModalities()) - .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.DOCUMENT); }); } @@ -68,9 +68,9 @@ void overrideAddsBrandNewModelEntryUnderExistingFamily() { assertThat(caps.supportsReasoning()).isTrue(); // Inherited from anthropic-messages defaults: assertThat(caps.userMessageModalities()) - .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.DOCUMENT); assertThat(caps.toolResultModalities()) - .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.DOCUMENT); }); } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesResolverTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesResolverTest.java index dfa853ec377..d3821aeaca2 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesResolverTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/ModelCapabilitiesResolverTest.java @@ -89,11 +89,11 @@ void overrideShortCircuitsResolution() { @Test void exactIdMatchInheritsDefaultsAndAppliesOverrides() { - // defaults: PDF in user_message, max_output_tokens=8192, no reasoning + // defaults: DOCUMENT in user_message, max_output_tokens=8192, no reasoning // override on claude-opus-4-7: enable reasoning, max_output_tokens=32000 final var defaults = fullDefaults( - List.of(Modality.TEXT, Modality.IMAGE, Modality.PDF), + List.of(Modality.TEXT, Modality.IMAGE, Modality.DOCUMENT), List.of(Modality.TEXT, Modality.IMAGE)); final var override = new ModelCapabilitiesYaml(null, null, true, true, null, null, null, 32000); @@ -111,7 +111,7 @@ void exactIdMatchInheritsDefaultsAndAppliesOverrides() { final var caps = resolver.resolve("anthropic-messages", "claude-opus-4-7", Optional.empty()); assertThat(caps.userMessageModalities()) - .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.PDF); + .containsExactly(Modality.TEXT, Modality.IMAGE, Modality.DOCUMENT); assertThat(caps.toolResultModalities()).containsExactly(Modality.TEXT, Modality.IMAGE); assertThat(caps.supportsReasoning()).isTrue(); assertThat(caps.supportsReasoningSignatureRoundtrip()).isTrue(); diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApiTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApiTest.java index b8b7fb67d70..655aa5aa509 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApiTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApiTest.java @@ -54,7 +54,7 @@ class OpenAiResponsesChatModelApiTest { private static final ModelCapabilities CAPABILITIES = new ModelCapabilities( List.of(Modality.TEXT, Modality.IMAGE), - List.of(Modality.TEXT, Modality.IMAGE, Modality.PDF), + List.of(Modality.TEXT, Modality.IMAGE, Modality.DOCUMENT), List.of(Modality.TEXT), true, true, diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategyImplTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategyImplTest.java index 0afc6d77918..2f3f05573c1 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategyImplTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategyImplTest.java @@ -45,8 +45,8 @@ class ToolCallResultStrategyImplTest { private static final ModelCapabilities INLINE_IMAGE_PDF = capabilities( - List.of(Modality.TEXT, Modality.IMAGE, Modality.PDF), - List.of(Modality.TEXT, Modality.IMAGE, Modality.PDF)); + List.of(Modality.TEXT, Modality.IMAGE, Modality.DOCUMENT), + List.of(Modality.TEXT, Modality.IMAGE, Modality.DOCUMENT)); private static final ModelCapabilities INLINE_IMAGE_ONLY = capabilities(List.of(Modality.TEXT, Modality.IMAGE), List.of(Modality.TEXT, Modality.IMAGE)); private static final ModelCapabilities TEXT_ONLY = @@ -200,7 +200,7 @@ void unsupportedDocumentInUserMessageThrows() { assertThatThrownBy(() -> strategy.apply(request, INLINE_IMAGE_ONLY)) .isInstanceOf(ConnectorException.class) - .hasMessageContaining("PDF") + .hasMessageContaining("DOCUMENT") .hasMessageContaining("does not"); } From 5e99c3d7c9e5a3bcd5916080ff23a781bfc2d78e Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 21:00:30 +0200 Subject: [PATCH 62/81] fix(agentic-ai): serialise ObjectContent as JSON in native impls The native ChatModelApi impls rendered ObjectContent values via String.valueOf(...), which emits Java toString output for Maps / POJOs ({key=value}) instead of JSON. The LangChain4j bridge has always used the document-aware ObjectMapper to produce {"key":"value"}, and tool results commonly arrive as Maps from MCP / A2A / process variables, so the native path silently sent the model malformed payloads. Route all text-only emission paths in the Anthropic Messages, OpenAI Chat Completions, and OpenAI Responses impls through a new ContentTextSerializer helper that mirrors ContentConverterImpl: a TextContent passes through verbatim; an ObjectContent returns its inner String as-is or otherwise Jackson-serialises against the @ConnectorsObjectMapper (which has JacksonModuleDocumentSerializer registered, so nested Documents serialise to references rather than throwing). --- .../AnthropicMessagesChatModelApi.java | 22 ++-- .../content/ContentTextSerializer.java | 55 +++++++++ .../OpenAiChatCompletionsChatModelApi.java | 46 +++----- .../openai/OpenAiResponsesChatModelApi.java | 36 ++---- .../content/ContentTextSerializerTest.java | 104 ++++++++++++++++++ 5 files changed, 190 insertions(+), 73 deletions(-) create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/content/ContentTextSerializer.java create mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/content/ContentTextSerializerTest.java diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java index cddf291a3e8..ced23d007e1 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java @@ -33,6 +33,7 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.content.ContentTextSerializer; import io.camunda.connector.agenticai.aiagent.framework.multimodal.DocumentModality; import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.agenticai.model.message.AssistantMessage; @@ -43,7 +44,6 @@ import io.camunda.connector.agenticai.model.message.UserMessage; import io.camunda.connector.agenticai.model.message.content.Content; import io.camunda.connector.agenticai.model.message.content.DocumentContent; -import io.camunda.connector.agenticai.model.message.content.ObjectContent; import io.camunda.connector.agenticai.model.message.content.TextContent; import io.camunda.connector.agenticai.model.tool.ToolCall; import io.camunda.connector.agenticai.model.tool.ToolCallResult; @@ -202,7 +202,7 @@ private MessageParam toMessageParam(ToolCallResultMessage message) { * validated user-message documents against the model's {@code userMessageModalities}, so any * {@link DocumentContent} reaching this point is known to be supported. */ - private static List messageContentBlocks(List content) { + private List messageContentBlocks(List content) { if (content == null) { return List.of(); } @@ -213,7 +213,7 @@ private static List messageContentBlocks(List conten return blocks; } - private static ContentBlockParam messageContentBlock(Content content) { + private ContentBlockParam messageContentBlock(Content content) { if (content instanceof DocumentContent doc) { final var modality = DocumentModality.of(doc.document()); return switch (modality) { @@ -235,7 +235,7 @@ private static ContentBlockParam messageContentBlock(Content content) { * Assistant-message blocks: text only (assistant turns we send back to the model don't carry * documents). Used by the assistant-message conversion path. */ - private static List textOnlyBlocks(List content) { + private List textOnlyBlocks(List content) { if (content == null) { return List.of(); } @@ -246,17 +246,9 @@ private static List textOnlyBlocks(List content) { return blocks; } - private static ContentBlockParam textOnlyBlock(Content content) { - if (content instanceof TextContent text) { - return ContentBlockParam.ofText(TextBlockParam.builder().text(text.text()).build()); - } - if (content instanceof ObjectContent object) { - return ContentBlockParam.ofText( - TextBlockParam.builder().text(String.valueOf(object.content())).build()); - } - throw new IllegalArgumentException( - "Unsupported content block for text-only Anthropic Messages API: " - + content.getClass().getSimpleName()); + private ContentBlockParam textOnlyBlock(Content content) { + return ContentBlockParam.ofText( + TextBlockParam.builder().text(ContentTextSerializer.toText(content, objectMapper)).build()); } private static ImageBlockParam imageBlockParam(Document document) { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/content/ContentTextSerializer.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/content/ContentTextSerializer.java new file mode 100644 index 00000000000..2e3f7beb183 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/content/ContentTextSerializer.java @@ -0,0 +1,55 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.content; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.connector.agenticai.model.message.content.Content; +import io.camunda.connector.agenticai.model.message.content.ObjectContent; +import io.camunda.connector.agenticai.model.message.content.TextContent; + +/** + * Renders a {@link Content} block to a text representation suitable for provider-native APIs that + * only accept text (system prompts, assistant content, the text portion of multimodal user + * messages, the text fallback for unsupported tool-result modalities). + * + *

      Mirrors the LangChain4j bridge's {@code ContentConverterImpl}: a {@link TextContent} returns + * its text verbatim; an {@link ObjectContent} returns its inner value as-is when it's already a + * {@code String}, otherwise as Jackson-serialised JSON. Other content types must be handled by the + * caller before reaching this helper. + * + *

      Callers are expected to pass the {@code @ConnectorsObjectMapper} so nested {@link + * io.camunda.connector.api.document.Document}s serialise to their reference shape rather than + * throwing. + */ +public final class ContentTextSerializer { + + private ContentTextSerializer() {} + + public static String toText(Content content, ObjectMapper objectMapper) { + if (content instanceof TextContent text) { + return text.text(); + } + if (content instanceof ObjectContent object) { + return objectContentToText(object, objectMapper); + } + throw new IllegalArgumentException( + "Unsupported content block for text serialization: " + content.getClass().getSimpleName()); + } + + public static String objectContentToText(ObjectContent content, ObjectMapper objectMapper) { + final var value = content.content(); + if (value instanceof String s) { + return s; + } + try { + return objectMapper.writeValueAsString(value); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Failed to serialize ObjectContent value to JSON", e); + } + } +} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java index 258e637f907..861695e3b59 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatCompletionsChatModelApi.java @@ -27,6 +27,7 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.content.ContentTextSerializer; import io.camunda.connector.agenticai.aiagent.framework.multimodal.DocumentModality; import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.agenticai.model.message.AssistantMessage; @@ -36,7 +37,6 @@ import io.camunda.connector.agenticai.model.message.UserMessage; import io.camunda.connector.agenticai.model.message.content.Content; import io.camunda.connector.agenticai.model.message.content.DocumentContent; -import io.camunda.connector.agenticai.model.message.content.ObjectContent; import io.camunda.connector.agenticai.model.message.content.TextContent; import io.camunda.connector.agenticai.model.tool.ToolCall; import io.camunda.connector.agenticai.model.tool.ToolCallResult; @@ -153,7 +153,7 @@ private Long resolveMaxCompletionTokens(ChatOptions options) { return configuredMaxCompletionTokens; } - private static String systemPrompt(SystemMessage system) { + private String systemPrompt(SystemMessage system) { return extractText(system.content()); } @@ -163,7 +163,7 @@ private static String systemPrompt(SystemMessage system) { * used when the message contains at least one {@link DocumentContent} (image / document, * validated upstream by {@code ToolCallResultStrategy}). */ - private static void addUserMessage(ChatCompletionCreateParams.Builder builder, UserMessage user) { + private void addUserMessage(ChatCompletionCreateParams.Builder builder, UserMessage user) { final var content = user.content(); if (!hasMultimodalContent(content)) { builder.addUserMessage(extractText(content)); @@ -171,22 +171,14 @@ private static void addUserMessage(ChatCompletionCreateParams.Builder builder, U } final var parts = new ArrayList(); for (var c : content) { - switch (c) { - case TextContent t -> - parts.add( - ChatCompletionContentPart.ofText( - ChatCompletionContentPartText.builder().text(t.text()).build())); - case ObjectContent o -> - parts.add( - ChatCompletionContentPart.ofText( - ChatCompletionContentPartText.builder() - .text(String.valueOf(o.content())) - .build())); - case DocumentContent doc -> parts.add(documentPart(doc.document())); - default -> - throw new IllegalArgumentException( - "Unsupported content block for OpenAI Chat Completions user message: " - + c.getClass().getSimpleName()); + if (c instanceof DocumentContent doc) { + parts.add(documentPart(doc.document())); + } else { + parts.add( + ChatCompletionContentPart.ofText( + ChatCompletionContentPartText.builder() + .text(ContentTextSerializer.toText(c, objectMapper)) + .build())); } } builder.addUserMessageOfArrayOfContentParts(parts); @@ -245,29 +237,17 @@ private static String safeFilename(Document document) { return StringUtils.isNotBlank(name) ? name : "document"; } - private static String extractText(List content) { + private String extractText(List content) { if (content == null || content.isEmpty()) { return ""; } final var sb = new StringBuilder(); for (var c : content) { - sb.append(textOf(c)); + sb.append(ContentTextSerializer.toText(c, objectMapper)); } return sb.toString(); } - private static String textOf(Content content) { - if (content instanceof TextContent text) { - return text.text(); - } - if (content instanceof ObjectContent object) { - return String.valueOf(object.content()); - } - throw new IllegalArgumentException( - "Unsupported content block for text-only OpenAI Chat Completions API: " - + content.getClass().getSimpleName()); - } - private ChatCompletionAssistantMessageParam toAssistantParam(AssistantMessage message) { final var builder = ChatCompletionAssistantMessageParam.builder(); final var text = message.content() != null ? extractText(message.content()) : ""; diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApi.java index d747d1fa70c..aa44689c044 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApi.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiResponsesChatModelApi.java @@ -33,6 +33,7 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatResponse; import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; import io.camunda.connector.agenticai.aiagent.framework.api.ModelCapabilities; +import io.camunda.connector.agenticai.aiagent.framework.content.ContentTextSerializer; import io.camunda.connector.agenticai.aiagent.framework.multimodal.DocumentModality; import io.camunda.connector.agenticai.aiagent.model.AgentMetrics; import io.camunda.connector.agenticai.model.message.AssistantMessage; @@ -42,7 +43,6 @@ import io.camunda.connector.agenticai.model.message.UserMessage; import io.camunda.connector.agenticai.model.message.content.Content; import io.camunda.connector.agenticai.model.message.content.DocumentContent; -import io.camunda.connector.agenticai.model.message.content.ObjectContent; import io.camunda.connector.agenticai.model.message.content.TextContent; import io.camunda.connector.agenticai.model.tool.ToolCall; import io.camunda.connector.agenticai.model.tool.ToolCallResult; @@ -163,21 +163,13 @@ private Long resolveMaxOutputTokens(ChatOptions options) { return configuredMaxOutputTokens; } - private static String extractText(List content) { + private String extractText(List content) { if (content == null || content.isEmpty()) { return ""; } final var sb = new StringBuilder(); for (var c : content) { - if (c instanceof TextContent t) { - sb.append(t.text()); - } else if (c instanceof ObjectContent o) { - sb.append(String.valueOf(o.content())); - } else { - throw new IllegalArgumentException( - "Unsupported content block for text-only OpenAI Responses API: " - + c.getClass().getSimpleName()); - } + sb.append(ContentTextSerializer.toText(c, objectMapper)); } return sb.toString(); } @@ -312,20 +304,14 @@ private ResponseInputItem toUserInputItem(UserMessage user) { } final var items = new ArrayList(); for (var c : content) { - switch (c) { - case TextContent t -> - items.add( - ResponseInputContent.ofInputText( - ResponseInputText.builder().text(t.text()).build())); - case ObjectContent o -> - items.add( - ResponseInputContent.ofInputText( - ResponseInputText.builder().text(String.valueOf(o.content())).build())); - case DocumentContent doc -> items.add(documentInputContent(doc.document())); - default -> - throw new IllegalArgumentException( - "Unsupported content block for OpenAI Responses user message: " - + c.getClass().getSimpleName()); + if (c instanceof DocumentContent doc) { + items.add(documentInputContent(doc.document())); + } else { + items.add( + ResponseInputContent.ofInputText( + ResponseInputText.builder() + .text(ContentTextSerializer.toText(c, objectMapper)) + .build())); } } return ResponseInputItem.ofEasyInputMessage( diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/content/ContentTextSerializerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/content/ContentTextSerializerTest.java new file mode 100644 index 00000000000..437d3622ab2 --- /dev/null +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/content/ContentTextSerializerTest.java @@ -0,0 +1,104 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.content; + +import static io.camunda.connector.agenticai.model.message.content.ObjectContent.objectContent; +import static io.camunda.connector.agenticai.model.message.content.ReasoningContent.reasoningContent; +import static io.camunda.connector.agenticai.model.message.content.TextContent.textContent; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.connector.agenticai.model.message.content.DocumentContent; +import io.camunda.connector.api.document.Document; +import io.camunda.connector.api.document.DocumentCreationRequest; +import io.camunda.connector.api.document.DocumentFactory; +import io.camunda.connector.document.jackson.JacksonModuleDocumentSerializer; +import io.camunda.connector.runtime.core.document.DocumentFactoryImpl; +import io.camunda.connector.runtime.core.document.store.InMemoryDocumentStore; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class ContentTextSerializerTest { + + private final ObjectMapper objectMapper = + new ObjectMapper().registerModule(new JacksonModuleDocumentSerializer()); + + @Test + void textContentReturnsRawText() { + assertThat(ContentTextSerializer.toText(textContent("hello"), objectMapper)).isEqualTo("hello"); + } + + @Test + void objectContentWithStringReturnsRawString() { + assertThat(ContentTextSerializer.toText(objectContent("plain string"), objectMapper)) + .isEqualTo("plain string"); + } + + @Test + void objectContentWithMapReturnsJson() { + final var content = new LinkedHashMap(); + content.put("key", "value"); + content.put("count", 42); + + assertThat(ContentTextSerializer.toText(objectContent(content), objectMapper)) + .isEqualTo("{\"key\":\"value\",\"count\":42}"); + } + + @Test + void objectContentWithListReturnsJson() { + assertThat(ContentTextSerializer.toText(objectContent(List.of("a", "b", "c")), objectMapper)) + .isEqualTo("[\"a\",\"b\",\"c\"]"); + } + + @Test + void objectContentWithDocumentSerialisesViaDocumentModule() { + final var documentStore = InMemoryDocumentStore.INSTANCE; + documentStore.clear(); + final DocumentFactory factory = new DocumentFactoryImpl(documentStore); + final Document document = + factory.create( + DocumentCreationRequest.from("hello".getBytes(StandardCharsets.UTF_8)) + .contentType("text/plain") + .fileName("greeting.txt") + .build()); + + final var nested = Map.of("attachment", document); + + final var json = ContentTextSerializer.toText(objectContent(nested), objectMapper); + + assertThat(json).contains("\"camunda.document.type\":\"camunda\""); + assertThat(json).doesNotContain("hello"); + } + + @Test + void unsupportedContentTypeThrows() { + final var documentStore = InMemoryDocumentStore.INSTANCE; + documentStore.clear(); + final DocumentFactory factory = new DocumentFactoryImpl(documentStore); + final Document document = + factory.create( + DocumentCreationRequest.from("x".getBytes(StandardCharsets.UTF_8)) + .contentType("text/plain") + .build()); + + assertThatThrownBy( + () -> + ContentTextSerializer.toText( + DocumentContent.documentContent(document), objectMapper)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("DocumentContent"); + + assertThatThrownBy( + () -> ContentTextSerializer.toText(reasoningContent("thinking"), objectMapper)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("ReasoningContent"); + } +} From 72cd95eecff58252982c489896933cf4e5a285ca Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 21:10:32 +0200 Subject: [PATCH 63/81] fix(agentic-ai): serialise Anthropic tool-result content via ConnectorsObjectMapper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Anthropic impl built tool-result content via ToolResultBlockParam.contentAsJson(content), which delegates to the SDK's internal ObjectMapper. That mapper has no document serialiser registered, so a CamundaDocument anywhere in the content tree blew up with InvalidDefinitionException ("No serializer found for class CamundaDocument..."). Always pre-serialise via the existing serializedToolResultText helper — which uses our @ConnectorsObjectMapper (with JacksonModuleDocumentSerializer registered) — and pass the result as content(String). Brings Anthropic in line with the OpenAI Chat Completions and Responses impls, which were already routing through our mapper. --- .../AnthropicMessagesChatModelApi.java | 12 ++-- .../AnthropicMessagesChatModelApiTest.java | 67 +++++++++++++++++++ 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java index ced23d007e1..e349238a4c5 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApi.java @@ -300,14 +300,10 @@ private ContentBlockParam toolResultBlock(ToolCallResult result) { if (inlineBlocks != null) { b.contentOfBlocks(inlineBlocks); } else { - final var content = result.content(); - if (content == null) { - b.content(ToolCallResult.CONTENT_NO_RESULT); - } else if (content instanceof String s) { - b.content(s); - } else { - b.contentAsJson(content); - } + // Always serialise via our @ConnectorsObjectMapper (document-aware) instead of the SDK's + // internal mapper — `contentAsJson(content)` would route through Anthropic's own + // ObjectMapper which can't serialise CamundaDocument and friends. + b.content(serializedToolResultText(result)); } final var interrupted = result.properties() != null diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java index ea1abfccfe4..2bf191ea188 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiTest.java @@ -27,6 +27,7 @@ import com.anthropic.models.messages.ToolUseBlock; import com.anthropic.models.messages.Usage; import com.anthropic.services.blocking.MessageService; +import com.fasterxml.jackson.databind.ObjectMapper; import io.camunda.connector.agenticai.aiagent.framework.api.ChatOptions; import io.camunda.connector.agenticai.aiagent.framework.api.ChatRequest; import io.camunda.connector.agenticai.aiagent.framework.api.ChatStreamListener; @@ -35,7 +36,13 @@ import io.camunda.connector.agenticai.model.tool.ToolCall; import io.camunda.connector.agenticai.model.tool.ToolCallResult; import io.camunda.connector.agenticai.model.tool.ToolDefinition; +import io.camunda.connector.api.document.Document; +import io.camunda.connector.api.document.DocumentCreationRequest; import io.camunda.connector.api.error.ConnectorException; +import io.camunda.connector.document.jackson.JacksonModuleDocumentSerializer; +import io.camunda.connector.runtime.core.document.DocumentFactoryImpl; +import io.camunda.connector.runtime.core.document.store.InMemoryDocumentStore; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.util.concurrent.CompletionException; @@ -186,6 +193,66 @@ void priorAssistantToolCallsAndResultsRoundTripIntoParams() { assertThat(params.messages().get(2).role()).isEqualTo(MessageParam.Role.USER); } + @Test + void toolResultContentWithCamundaDocumentSerialisesViaConnectorsObjectMapper() { + // Regression: previously the impl called ToolResultBlockParam.contentAsJson(content), which + // delegated to the Anthropic SDK's internal ObjectMapper. That mapper has no document + // serialiser registered, so a CamundaDocument inside the content tree threw + // InvalidDefinitionException. The fix routes everything through serializedToolResultText, + // which uses our @ConnectorsObjectMapper (with JacksonModuleDocumentSerializer registered). + final var documentStore = InMemoryDocumentStore.INSTANCE; + documentStore.clear(); + final var documentFactory = new DocumentFactoryImpl(documentStore); + final Document document = + documentFactory.create( + DocumentCreationRequest.from("hello".getBytes(StandardCharsets.UTF_8)) + .contentType("text/plain") + .fileName("greeting.txt") + .build()); + + final var documentAwareApi = + new AnthropicMessagesChatModelApi( + client, + MODEL_ID, + new ObjectMapper().registerModule(new JacksonModuleDocumentSerializer()), + CAPABILITIES, + 1024L, + null, + null, + null); + + when(messageService.create((MessageCreateParams) paramsCaptor.capture())) + .thenReturn(textOnlyResponse("ack")); + + final var prior = + assistantMessage( + "let me check", + List.of(ToolCall.builder().id("abc").name("getDocument").arguments(Map.of()).build())); + final var results = + toolCallResultMessage( + List.of( + ToolCallResult.builder() + .id("abc") + .name("getDocument") + .content(Map.of("attachment", document)) + .build())); + + documentAwareApi + .complete( + new ChatRequest(List.of(userMessage("show it"), prior, results), tools(), null), + defaultOptions(), + ChatStreamListener.NOOP) + .join(); + + final var params = paramsCaptor.getValue(); + final var toolResultMessage = params.messages().get(2); + final var blocks = toolResultMessage.content().asBlockParams(); + final var toolResultBlock = blocks.getFirst().asToolResult(); + final var contentString = toolResultBlock.content().get().asString(); + assertThat(contentString).contains("\"camunda.document.type\":\"camunda\""); + assertThat(contentString).contains("greeting.txt"); + } + @Test void wrapsSdkExceptionInConnectorException() { when(messageService.create(any(MessageCreateParams.class))) From f6352963f422aa8ba8e814de989ae5b9f59fddad Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 21:20:25 +0200 Subject: [PATCH 64/81] TMP TMP: Update BPMN example to return an image --- .../ai-agent-chat-with-tools.bpmn | 61 ++++++++++++++++--- 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/connectors/agentic-ai/examples/ai-agent/ad-hoc-sub-process/ai-agent-chat-with-tools/ai-agent-chat-with-tools.bpmn b/connectors/agentic-ai/examples/ai-agent/ad-hoc-sub-process/ai-agent-chat-with-tools/ai-agent-chat-with-tools.bpmn index 2c4e813e866..076ae564e5e 100644 --- a/connectors/agentic-ai/examples/ai-agent/ad-hoc-sub-process/ai-agent-chat-with-tools/ai-agent-chat-with-tools.bpmn +++ b/connectors/agentic-ai/examples/ai-agent/ad-hoc-sub-process/ai-agent-chat-with-tools/ai-agent-chat-with-tools.bpmn @@ -1,5 +1,5 @@ - + @@ -39,17 +39,14 @@ =userSatisfied - + - - - - - - + + + @@ -66,7 +63,7 @@ - + @@ -248,9 +245,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -278,6 +315,9 @@ + + + @@ -326,6 +366,9 @@ + + + From 92a58f5abd722c007f5872fb617936c69f2effae Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 21:34:12 +0200 Subject: [PATCH 65/81] fix(agentic-ai): load capability matrix YAML in @Configuration setEnvironment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bundled `model-capabilities.yaml` was registered as a low-precedence Spring PropertySource via an `EnvironmentPostProcessor` discovered through `META-INF/spring/...EnvironmentPostProcessor.imports`. That discovery only runs during a vanilla Spring Boot application startup. When the agentic-ai jar is loaded after the host context has already started — as happens in some Camunda Connector Runtime deployments — the post-processor never fires, `AgenticAiFrameworkProperties.capabilities` binds to an empty map, and `ModelCapabilitiesResolver` falls through to its conservative defaults (`tool-result: [text]`). The `ToolCallResultStrategy` then routes every tool-result document — image or otherwise — into a synthetic UserMessage instead of inlining it. Move the loading into `AgenticAiCapabilitiesConfiguration.setEnvironment`, which Spring invokes when the configuration bean itself is instantiated — robust to whatever discovery path the host runtime uses, and still register the source via `addLast(...)` so consumer overrides under the same prefix continue to win. Drop the dedicated post-processor class and its `META-INF/spring/...imports` registration. Also add DEBUG log lines in `ToolCallResultStrategyImpl` printing the resolved tool-result modalities and the inline/fallback decision per tool-result, so future capability-resolution mismatches surface from plain logs rather than requiring a debugger. --- .../AgenticAiCapabilitiesConfiguration.java | 47 +++++++++++++++-- .../AgenticAiFrameworkProperties.java | 3 +- ...abilityMatrixEnvironmentPostProcessor.java | 50 ------------------- .../strategy/ToolCallResultStrategyImpl.java | 18 +++++++ ....boot.env.EnvironmentPostProcessor.imports | 1 - .../capabilities/model-capabilities.yaml | 9 ++-- .../BundledCapabilityMatrixTest.java | 16 +++--- .../CapabilityMatrixOverrideTest.java | 11 ++-- 8 files changed, 77 insertions(+), 78 deletions(-) delete mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixEnvironmentPostProcessor.java delete mode 100644 connectors/agentic-ai/src/main/resources/META-INF/spring/org.springframework.boot.env.EnvironmentPostProcessor.imports diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/AgenticAiCapabilitiesConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/AgenticAiCapabilitiesConfiguration.java index 92d2eda0c6e..a1b6480295e 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/AgenticAiCapabilitiesConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/AgenticAiCapabilitiesConfiguration.java @@ -8,20 +8,57 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.camunda.connector.runtime.annotation.ConnectorsObjectMapper; +import java.io.IOException; +import java.util.List; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.env.YamlPropertySourceLoader; +import org.springframework.context.EnvironmentAware; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.Environment; +import org.springframework.core.env.PropertySource; +import org.springframework.core.io.ClassPathResource; /** - * Spring configuration for the model capability matrix. Bundled defaults from {@code - * resources/capabilities/model-capabilities.yaml} are loaded as a low-precedence property source by - * {@link CapabilityMatrixEnvironmentPostProcessor}; library-consumer overrides under the same - * {@code camunda.connector.agenticai.aiagent.framework.capabilities.*} prefix land on top. + * Spring configuration for the model capability matrix. + * + *

      Bundled defaults from {@code resources/capabilities/model-capabilities.yaml} are loaded as a + * low-precedence {@link PropertySource} via {@link #setEnvironment(Environment)} — invoked when + * Spring instantiates this configuration bean and therefore robust against runtimes that skip + * Spring Boot's {@code EnvironmentPostProcessor} discovery (e.g. when the connector jar is loaded + * after the host application context has already started). Library-consumer overrides under the + * same {@code camunda.connector.agenticai.aiagent.framework.capabilities.*} prefix land on top. */ @Configuration @EnableConfigurationProperties(AgenticAiFrameworkProperties.class) -public class AgenticAiCapabilitiesConfiguration { +public class AgenticAiCapabilitiesConfiguration implements EnvironmentAware { + + private static final String BUNDLED_RESOURCE = "capabilities/model-capabilities.yaml"; + private static final String PROPERTY_SOURCE_NAME = "agentic-ai-bundled-capability-matrix"; + + @Override + public void setEnvironment(Environment environment) { + if (!(environment instanceof ConfigurableEnvironment configurable)) { + return; + } + if (configurable.getPropertySources().contains(PROPERTY_SOURCE_NAME)) { + return; + } + final var resource = new ClassPathResource(BUNDLED_RESOURCE); + if (!resource.exists()) { + return; + } + try { + final List> sources = + new YamlPropertySourceLoader().load(PROPERTY_SOURCE_NAME, resource); + sources.forEach(configurable.getPropertySources()::addLast); + } catch (IOException e) { + throw new IllegalStateException( + "Failed to load bundled capability matrix from classpath:" + BUNDLED_RESOURCE, e); + } + } @Bean @ConditionalOnMissingBean diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/AgenticAiFrameworkProperties.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/AgenticAiFrameworkProperties.java index 189c9d5e2be..204cc792ec4 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/AgenticAiFrameworkProperties.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/AgenticAiFrameworkProperties.java @@ -18,7 +18,8 @@ *

        *
      1. Bundled defaults: {@code resources/capabilities/model-capabilities.yaml}, registered as a * low-precedence {@link org.springframework.core.env.PropertySource} by {@link - * CapabilityMatrixEnvironmentPostProcessor} at startup. + * AgenticAiCapabilitiesConfiguration#setEnvironment} when the configuration bean is + * instantiated. *
      2. Application overrides: any property under {@code * camunda.connector.agenticai.aiagent.framework.capabilities.*} declared by the library * consumer (typically in their own {@code application.yml}). Library consumers can override diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixEnvironmentPostProcessor.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixEnvironmentPostProcessor.java deleted file mode 100644 index 98e3e5048d0..00000000000 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixEnvironmentPostProcessor.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH - * under one or more contributor license agreements. Licensed under a proprietary license. - * See the License.txt file for more information. You may not use this file - * except in compliance with the proprietary license. - */ -package io.camunda.connector.agenticai.aiagent.framework.capabilities; - -import java.io.IOException; -import java.util.List; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.env.EnvironmentPostProcessor; -import org.springframework.boot.env.YamlPropertySourceLoader; -import org.springframework.core.env.ConfigurableEnvironment; -import org.springframework.core.env.PropertySource; -import org.springframework.core.io.ClassPathResource; -import org.springframework.core.io.Resource; - -/** - * Loads the bundled model capability matrix YAML as a low-precedence {@link PropertySource} so - * library consumers can override any value via their own {@code application.yml}. - * - *

        The bundled file lives at {@code classpath:capabilities/model-capabilities.yaml} and is - * structured under the {@code camunda.connector.agenticai.aiagent.framework.capabilities} prefix. - * It is registered with {@code addLast(...)} so any user-supplied source — including {@code - * application.yml}, environment variables and command-line arguments — wins. - */ -public class CapabilityMatrixEnvironmentPostProcessor implements EnvironmentPostProcessor { - - private static final String BUNDLED_RESOURCE = "capabilities/model-capabilities.yaml"; - private static final String PROPERTY_SOURCE_NAME = "agentic-ai-bundled-capability-matrix"; - - private final YamlPropertySourceLoader loader = new YamlPropertySourceLoader(); - - @Override - public void postProcessEnvironment( - ConfigurableEnvironment environment, SpringApplication application) { - final Resource resource = new ClassPathResource(BUNDLED_RESOURCE); - if (!resource.exists()) { - return; - } - try { - final List> sources = loader.load(PROPERTY_SOURCE_NAME, resource); - sources.forEach(environment.getPropertySources()::addLast); - } catch (IOException e) { - throw new IllegalStateException( - "Failed to load bundled capability matrix from classpath:" + BUNDLED_RESOURCE, e); - } - } -} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategyImpl.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategyImpl.java index e7aec95a845..5967e0256a4 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategyImpl.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/strategy/ToolCallResultStrategyImpl.java @@ -46,6 +46,13 @@ public ToolCallResultStrategyImpl(ToolCallResultDocumentExtractor documentExtrac @Override public Result apply(ChatRequest request, ModelCapabilities capabilities) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "Applying tool-call-result strategy with resolved capabilities — " + + "userMessageModalities={}, toolResultModalities={}", + capabilities.userMessageModalities(), + capabilities.toolResultModalities()); + } final var rewrittenMessages = new ArrayList(request.messages().size()); final var syntheticContextMessages = new ArrayList(); @@ -111,6 +118,17 @@ private RoutedToolCallResultMessage routeToolCallResultMessage( fallback.add(doc); } } + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "Routing tool result id={} name={}: documents={}, inlined={}, fallback={}, " + + "toolResultModalities={}", + result.id(), + result.name(), + entry.documents().size(), + inline.size(), + fallback.size(), + capabilities.toolResultModalities()); + } rewrittenResults.add(inline.isEmpty() ? result : appendInlineContentBlocks(result, inline)); if (!fallback.isEmpty()) { diff --git a/connectors/agentic-ai/src/main/resources/META-INF/spring/org.springframework.boot.env.EnvironmentPostProcessor.imports b/connectors/agentic-ai/src/main/resources/META-INF/spring/org.springframework.boot.env.EnvironmentPostProcessor.imports deleted file mode 100644 index a03e83065b0..00000000000 --- a/connectors/agentic-ai/src/main/resources/META-INF/spring/org.springframework.boot.env.EnvironmentPostProcessor.imports +++ /dev/null @@ -1 +0,0 @@ -io.camunda.connector.agenticai.aiagent.framework.capabilities.CapabilityMatrixEnvironmentPostProcessor diff --git a/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml b/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml index 4a2611f0f17..72150a419ba 100644 --- a/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml +++ b/connectors/agentic-ai/src/main/resources/capabilities/model-capabilities.yaml @@ -1,9 +1,10 @@ # Model capability matrix for the agentic-ai connector. # -# This file is loaded as a Spring Boot PropertySource at startup (lowest -# precedence) by `CapabilityMatrixEnvironmentPostProcessor`. Library consumers -# can override or extend any value via `application.yml` using the same -# property prefix, no full-file replacement required: +# This file is loaded as a low-precedence Spring PropertySource by +# `AgenticAiCapabilitiesConfiguration#setEnvironment` when the configuration +# bean is instantiated. Library consumers can override or extend any value via +# `application.yml` using the same property prefix, no full-file replacement +# required: # # camunda.connector.agenticai.aiagent.framework.capabilities: # anthropic-messages: diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/BundledCapabilityMatrixTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/BundledCapabilityMatrixTest.java index 65351d741ac..c6bed6e1ccc 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/BundledCapabilityMatrixTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/BundledCapabilityMatrixTest.java @@ -20,21 +20,17 @@ import org.springframework.context.annotation.Configuration; /** - * Spring-Boot-style integration test for the bundled capability matrix. Runs the {@link - * CapabilityMatrixEnvironmentPostProcessor} against the test {@link - * org.springframework.context.ConfigurableApplicationContext} so the bundled YAML is loaded as a - * {@link org.springframework.core.env.PropertySource}, then exercises the full {@link - * AgenticAiFrameworkProperties} → {@link CapabilityMatrixFactory} → {@link - * ModelCapabilitiesResolver} pipeline. + * Spring-Boot-style integration test for the bundled capability matrix. {@link + * AgenticAiCapabilitiesConfiguration} loads the bundled YAML as a {@link + * org.springframework.core.env.PropertySource} during its own {@code setEnvironment(...)} callback, + * so importing the configuration class is enough — no manual environment post-processing needed. + * The test then exercises the full {@link AgenticAiFrameworkProperties} → {@link + * CapabilityMatrixFactory} → {@link ModelCapabilitiesResolver} pipeline. */ class BundledCapabilityMatrixTest { private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() - .withInitializer( - context -> - new CapabilityMatrixEnvironmentPostProcessor() - .postProcessEnvironment(context.getEnvironment(), null)) .withUserConfiguration(TestObjectMapperConfig.class) .withUserConfiguration(AgenticAiCapabilitiesConfiguration.class); diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixOverrideTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixOverrideTest.java index 962a6b1c4e0..b8cf2377a51 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixOverrideTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/capabilities/CapabilityMatrixOverrideTest.java @@ -17,18 +17,15 @@ /** * Verifies that consumer-supplied properties (higher precedence than the bundled YAML) deep-merge - * into the capability matrix. Uses {@link ApplicationContextRunner} with both the bundled {@link - * CapabilityMatrixEnvironmentPostProcessor} and {@code withPropertyValues(...)} overrides — exactly - * the layering a library consumer gets when adding entries to their {@code application.yml}. + * into the capability matrix. {@link AgenticAiCapabilitiesConfiguration} loads the bundled YAML + * itself during {@code setEnvironment(...)}, so importing it plus {@code withPropertyValues(...)} + * mirrors the layering a library consumer gets when adding entries to their {@code + * application.yml}. */ class CapabilityMatrixOverrideTest { private final ApplicationContextRunner baseRunner = new ApplicationContextRunner() - .withInitializer( - context -> - new CapabilityMatrixEnvironmentPostProcessor() - .postProcessEnvironment(context.getEnvironment(), null)) .withUserConfiguration(TestObjectMapperConfig.class) .withUserConfiguration(AgenticAiCapabilitiesConfiguration.class); From 9935fe054120ccef4f0e75b3fc2622e9f2fa2f29 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Thu, 7 May 2026 21:40:40 +0200 Subject: [PATCH 66/81] refactor(agentic-ai): drop L4J factory + provider beans for migrated providers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Anthropic, OpenAI (Chat Completions + Responses), and OpenAi-Compatible have native ChatModelApi factories now (ADR-005 phases B/C/D). The L4J bridge factory beans for the same discriminators were still being declared — both configurations registered a bean named `langchain4JChatModelApiFactory` with mutual `@ConditionalOnMissingBean(name = …)` gates, so whichever auto-config Spring evaluated first won and the other quietly skipped. That made it ambiguous which factory the e2e tests were exercising. Drop the three L4J factory `@Bean` methods and the three matching `ChatModelProvider<…>` `@Bean` methods. The bridge factories for `bedrock`, `azureopenai` and `googlevertexai` stay (Phase G will replace them). The L4J converters and registry stay too — the remaining bridge factories still depend on them. Tests: drop the three migrated providers from `LANGCHAIN4J_BEANS` and remove their `ChatModelProvider` override scenarios from `AgenticAiConnectorsAutoConfigurationTest` — overriding a bean that no longer exists is not a meaningful test case. --- ...icAiLangchain4JChatModelConfiguration.java | 32 ------ ...icAiLangchain4JFrameworkConfiguration.java | 53 ---------- ...nticAiConnectorsAutoConfigurationTest.java | 100 ++---------------- 3 files changed, 7 insertions(+), 178 deletions(-) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JChatModelConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JChatModelConfiguration.java index 60660112813..f971a1f4200 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JChatModelConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JChatModelConfiguration.java @@ -9,20 +9,14 @@ import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelFactory; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelFactoryImpl; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelHttpProxySupport; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.AnthropicChatModelProvider; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.AzureOpenAiChatModelProvider; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.BedrockChatModelProvider; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProvider; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProviderRegistry; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.GoogleVertexAiChatModelProvider; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.OpenAiChatModelProvider; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.OpenAiCompatibleChatModelProvider; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsConfigurationProperties; import io.camunda.connector.agenticai.common.AgenticAiHttpProxySupport; import java.util.List; @@ -42,14 +36,6 @@ public ChatModelHttpProxySupport langchain4JChatModelHttpProxySupport( httpProxySupport.getJdkHttpClientProxyConfigurator()); } - @Bean - @ConditionalOnMissingBean - public ChatModelProvider langchain4JAnthropicChatModelProvider( - AgenticAiConnectorsConfigurationProperties config, - ChatModelHttpProxySupport chatModelHttpProxySupport) { - return new AnthropicChatModelProvider(config.aiagent().chatModel(), chatModelHttpProxySupport); - } - @Bean @ConditionalOnMissingBean public ChatModelProvider @@ -75,24 +61,6 @@ public ChatModelProvider langchain4JBedrockChatMod return new GoogleVertexAiChatModelProvider(); } - @Bean - @ConditionalOnMissingBean - public ChatModelProvider langchain4JOpenAiChatModelProvider( - AgenticAiConnectorsConfigurationProperties config, - ChatModelHttpProxySupport chatModelHttpProxySupport) { - return new OpenAiChatModelProvider(config.aiagent().chatModel(), chatModelHttpProxySupport); - } - - @Bean - @ConditionalOnMissingBean - public ChatModelProvider - langchain4JOpenAiCompatibleChatModelProvider( - AgenticAiConnectorsConfigurationProperties config, - ChatModelHttpProxySupport chatModelHttpProxySupport) { - return new OpenAiCompatibleChatModelProvider( - config.aiagent().chatModel(), chatModelHttpProxySupport); - } - @Bean @ConditionalOnMissingBean public ChatModelProviderRegistry langchain4JChatModelProviderRegistry( diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java index e3139903c78..07180e96c3f 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java @@ -21,12 +21,9 @@ import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolCallConverterImpl; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverterImpl; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; import io.camunda.connector.runtime.annotation.ConnectorsObjectMapper; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -84,23 +81,6 @@ public ChatMessageConverter langchain4JChatMessageConverter( return new ChatMessageConverterImpl(contentConverter, toolCallConverter); } - @Bean - @ConditionalOnMissingBean(name = "langchain4JAnthropicChatModelApiFactory") - public ChatModelApiFactory - langchain4JAnthropicChatModelApiFactory( - ChatModelProvider provider, - ChatMessageConverter chatMessageConverter, - ToolSpecificationConverter toolSpecificationConverter, - JsonSchemaConverter jsonSchemaConverter) { - return new Langchain4JChatModelApiFactory<>( - AnthropicProviderConfiguration.ANTHROPIC_ID, - AnthropicProviderConfiguration.class, - provider, - chatMessageConverter, - toolSpecificationConverter, - jsonSchemaConverter); - } - @Bean @ConditionalOnMissingBean(name = "langchain4JBedrockChatModelApiFactory") public ChatModelApiFactory langchain4JBedrockChatModelApiFactory( @@ -150,37 +130,4 @@ public ChatModelApiFactory langchain4JBedrockChatM toolSpecificationConverter, jsonSchemaConverter); } - - @Bean - @ConditionalOnMissingBean(name = "langchain4JOpenAiChatModelApiFactory") - public ChatModelApiFactory langchain4JOpenAiChatModelApiFactory( - ChatModelProvider provider, - ChatMessageConverter chatMessageConverter, - ToolSpecificationConverter toolSpecificationConverter, - JsonSchemaConverter jsonSchemaConverter) { - return new Langchain4JChatModelApiFactory<>( - OpenAiProviderConfiguration.OPENAI_ID, - OpenAiProviderConfiguration.class, - provider, - chatMessageConverter, - toolSpecificationConverter, - jsonSchemaConverter); - } - - @Bean - @ConditionalOnMissingBean(name = "langchain4JOpenAiCompatibleChatModelApiFactory") - public ChatModelApiFactory - langchain4JOpenAiCompatibleChatModelApiFactory( - ChatModelProvider provider, - ChatMessageConverter chatMessageConverter, - ToolSpecificationConverter toolSpecificationConverter, - JsonSchemaConverter jsonSchemaConverter) { - return new Langchain4JChatModelApiFactory<>( - OpenAiCompatibleProviderConfiguration.OPENAI_COMPATIBLE_ID, - OpenAiCompatibleProviderConfiguration.class, - provider, - chatMessageConverter, - toolSpecificationConverter, - jsonSchemaConverter); - } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java index a50e2c4e179..535e436ee6b 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java @@ -36,14 +36,11 @@ import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ContentConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.document.DocumentToContentConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.jsonschema.JsonSchemaConverter; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.AnthropicChatModelProvider; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.AzureOpenAiChatModelProvider; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.BedrockChatModelProvider; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProvider; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProviderRegistry; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.GoogleVertexAiChatModelProvider; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.OpenAiChatModelProvider; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.OpenAiCompatibleChatModelProvider; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolCallConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverter; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStoreRegistry; @@ -51,20 +48,14 @@ import io.camunda.connector.agenticai.aiagent.memory.conversation.awsagentcore.mapping.AwsAgentCoreConversationMapper; import io.camunda.connector.agenticai.aiagent.memory.conversation.document.CamundaDocumentConversationStore; import io.camunda.connector.agenticai.aiagent.memory.conversation.inprocess.InProcessConversationStore; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.ProviderConfiguration; import io.camunda.connector.agenticai.aiagent.tool.GatewayToolHandlerRegistry; -import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsAutoConfigurationTest.CustomChatModelProviderOverrides.CustomAnthropicProviderConfig.CustomAnthropicChatModelProvider; import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsAutoConfigurationTest.CustomChatModelProviderOverrides.CustomAzureOpenAiProviderConfig.CustomAzureOpenAiChatModelProvider; import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsAutoConfigurationTest.CustomChatModelProviderOverrides.CustomBedrockProviderConfig.CustomBedrockChatModelProvider; import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsAutoConfigurationTest.CustomChatModelProviderOverrides.CustomGoogleVertexAiProviderConfig.CustomGoogleVertexAiChatModelProvider; -import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsAutoConfigurationTest.CustomChatModelProviderOverrides.CustomOpenAiCompatibleProviderConfig.CustomOpenAiCompatibleChatModelProvider; -import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsAutoConfigurationTest.CustomChatModelProviderOverrides.CustomOpenAiProviderConfig.CustomOpenAiChatModelProvider; import io.camunda.connector.agenticai.common.AgenticAiHttpProxySupport; import io.camunda.connector.http.client.proxy.EnvironmentProxyConfiguration; import java.util.List; @@ -109,15 +100,15 @@ class AgenticAiConnectorsAutoConfigurationTest { JobWorkerAgentRequestHandler.class, AiAgentJobWorker.class); + // L4J factory + provider beans for `anthropic`, `openai`, `openaiCompatible` were dropped when + // those providers landed native in ADR-005 Phase B/C/D — only the still-bridged providers + // (Bedrock, Azure OpenAI, Google Vertex AI) keep an L4J `ChatModelProvider` bean. private static final List> LANGCHAIN4J_BEANS = List.of( ChatModelHttpProxySupport.class, - AnthropicChatModelProvider.class, AzureOpenAiChatModelProvider.class, BedrockChatModelProvider.class, GoogleVertexAiChatModelProvider.class, - OpenAiChatModelProvider.class, - OpenAiCompatibleChatModelProvider.class, ChatModelProviderRegistry.class, ChatModelFactory.class, DocumentToContentConverter.class, @@ -309,12 +300,10 @@ void userProvidedProviderBeanOverridesDefault(ProviderOverrideCase override) { } static Stream providerOverrideCases() { + // Only providers still backed by the L4J bridge declare a `ChatModelProvider` bean and + // therefore support overriding it. `anthropic`, `openai`, and `openaiCompatible` have native + // factories now and no L4J `ChatModelProvider` to replace. return Stream.of( - new ProviderOverrideCase( - CustomAnthropicProviderConfig.class, - "customAnthropicChatModelProvider", - AnthropicProviderConfiguration.class, - CustomAnthropicChatModelProvider.class), new ProviderOverrideCase( CustomAzureOpenAiProviderConfig.class, "customAzureOpenAiChatModelProvider", @@ -329,17 +318,7 @@ static Stream providerOverrideCases() { CustomGoogleVertexAiProviderConfig.class, "customGoogleVertexAiChatModelProvider", GoogleVertexAiProviderConfiguration.class, - CustomGoogleVertexAiChatModelProvider.class), - new ProviderOverrideCase( - CustomOpenAiProviderConfig.class, - "customOpenAiChatModelProvider", - OpenAiProviderConfiguration.class, - CustomOpenAiChatModelProvider.class), - new ProviderOverrideCase( - CustomOpenAiCompatibleProviderConfig.class, - "customOpenAiCompatibleChatModelProvider", - OpenAiCompatibleProviderConfiguration.class, - CustomOpenAiCompatibleChatModelProvider.class)); + CustomGoogleVertexAiChatModelProvider.class)); } record ProviderOverrideCase( @@ -354,27 +333,6 @@ public String toString() { } } - static class CustomAnthropicProviderConfig { - @Bean - ChatModelProvider customAnthropicChatModelProvider() { - return new CustomAnthropicChatModelProvider(); - } - - static class CustomAnthropicChatModelProvider - implements ChatModelProvider { - - @Override - public String type() { - return AnthropicProviderConfiguration.ANTHROPIC_ID; - } - - @Override - public ChatModel createChatModel(AnthropicProviderConfiguration providerConfiguration) { - return mock(ChatModel.class); - } - } - } - static class CustomAzureOpenAiProviderConfig { @Bean ChatModelProvider customAzureOpenAiChatModelProvider() { @@ -439,50 +397,6 @@ public ChatModel createChatModel( } } } - - static class CustomOpenAiProviderConfig { - @Bean - ChatModelProvider customOpenAiChatModelProvider() { - return new CustomOpenAiChatModelProvider(); - } - - static class CustomOpenAiChatModelProvider - implements ChatModelProvider { - - @Override - public String type() { - return OpenAiProviderConfiguration.OPENAI_ID; - } - - @Override - public ChatModel createChatModel(OpenAiProviderConfiguration providerConfiguration) { - return mock(ChatModel.class); - } - } - } - - static class CustomOpenAiCompatibleProviderConfig { - @Bean - ChatModelProvider - customOpenAiCompatibleChatModelProvider() { - return new CustomOpenAiCompatibleChatModelProvider(); - } - - static class CustomOpenAiCompatibleChatModelProvider - implements ChatModelProvider { - - @Override - public String type() { - return OpenAiCompatibleProviderConfiguration.OPENAI_COMPATIBLE_ID; - } - - @Override - public ChatModel createChatModel( - OpenAiCompatibleProviderConfiguration providerConfiguration) { - return mock(ChatModel.class); - } - } - } } private Predicate> notAnyOf(Class... classes) { From 041074b31b251f8132e48c254969276160f69ce3 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Fri, 8 May 2026 09:55:41 +0200 Subject: [PATCH 67/81] docs(agentic-ai): wire Phase F deserializer via @JsonDeserialize, not a Module The connector runtime composes `@ConnectorsObjectMapper` with a hard-coded module list in `ConnectorsAutoConfiguration` and exposes no `Module` bean discovery, so the original "register via Jackson Module" approach isn't reachable from the agentic-ai module. Type-level `@JsonDeserialize(using = ProviderConfigurationDeserializer.class)` on the abstract `ProviderConfiguration` works on any `ObjectMapper`, doesn't contribute to a global module list, and leaves serialization on the standard `@JsonTypeInfo` / `@JsonSubTypes` path. Update the Phase F plan accordingly: switch the registration mechanism note, document the dispatch pattern (`mapper.treeToValue(...)` doesn't re-enter the deserializer), and add a test that round-trips a `ProviderConfiguration` with a `Document` field through the actual `@ConnectorsObjectMapper` to confirm the type-level deserializer doesn't collide with the FEEL / document modules already layered on it. --- .../docs/adr-005-implementation-plan.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/connectors/agentic-ai/docs/adr-005-implementation-plan.md b/connectors/agentic-ai/docs/adr-005-implementation-plan.md index beeb45ee46f..a6d46f8f2e1 100644 --- a/connectors/agentic-ai/docs/adr-005-implementation-plan.md +++ b/connectors/agentic-ai/docs/adr-005-implementation-plan.md @@ -513,7 +513,16 @@ Element template version bump 11 → 12 (after D's bump). (forwarding to the Anthropic factory if it is). - New `ProviderConfigurationDeserializer extends StdDeserializer`: pre-processes JSON node before delegating. Migration rules per ADR table. Pattern: - `JsonSchemaElementDeserializer.java:52` (tree-walking dispatch). Register via Jackson `Module`. + `JsonSchemaElementDeserializer.java:52` (tree-walking dispatch). **Wire via type-level + `@JsonDeserialize(using = ProviderConfigurationDeserializer.class)` on the abstract + `ProviderConfiguration` itself — not via a Jackson `Module`.** The connector runtime composes + `@ConnectorsObjectMapper` in `ConnectorsAutoConfiguration` with a hard-coded module list and + exposes no `Module` bean discovery, so module-based registration from agentic-ai isn't possible. + Type-level `@JsonDeserialize` works on any `ObjectMapper` without contributing to a global module + list. Serialization continues to use the standard `@JsonTypeInfo` / `@JsonSubTypes` mechanism on + the new shape — `@JsonDeserialize` only affects the read path. Inside the deserializer, dispatch + to concrete subtypes via `mapper.treeToValue(migrated, SubType.class)`; that doesn't re-enter the + custom deserializer because Jackson's `BeanDeserializer` for the resolved subtype takes over. - `element-templates/agenticai-aiagent-outbound-connector.json`: bump 11 → 12; add conditional UI groups for `backend`. Maven regenerates the versioned snapshot + job-worker template. - `element-templates/README.md`: replace top row again (or insert new row if Camunda min version @@ -521,7 +530,12 @@ Element template version bump 11 → 12 (after D's bump). **Tests to add**: - `ProviderConfigurationDeserializerTest` — every row of the migration table round-trips to new - shape; forward serialization writes new shape. + shape; forward serialization writes new shape; idempotence (deserialize old → re-serialize → + re-deserialize gives the same canonical instance). +- One test that round-trips a `ProviderConfiguration` containing a `Document`-typed field through + the actual `@ConnectorsObjectMapper` (with `JacksonModuleDocumentDeserializer` and + `JacksonModuleFeelFunction` registered) to confirm the type-level `@JsonDeserialize` doesn't + collide with the runtime modules already layered on the mapper. **Verification**: `mvn clean install -pl connectors/agentic-ai`; manual inspect `versioned/agenticai-aiagent-outbound-connector-11.json` created; deserialization smoke test From 04150588fdaa546c42b30488774e0729c7b6c8dc Mon Sep 17 00:00:00 2001 From: Dmitri Nikonov Date: Fri, 8 May 2026 10:25:59 +0200 Subject: [PATCH 68/81] refactor(agentic-ai): add AnthropicBackend + sealed AnthropicAuthentication (Phase F) - Add AnthropicBackend enum (DIRECT, BEDROCK, VERTEX, FOUNDRY) with lowercase Jackson serialisation - Replace flat AnthropicAuthentication record with sealed interface + AnthropicApiKeyAuthentication / AnthropicClientCredentialsAuthentication variants (mirrors AzureAuthentication pattern) - Add backend field to AnthropicConnection with DIRECT default via compact constructor - Add @AssertFalse validation rejecting clientCredentials on non-FOUNDRY backends - Update AnthropicMessagesChatModelApiFactory and AnthropicChatModelProvider to use pattern matching on the sealed interface - Update all affected tests to use the new constructor signatures and auth subtypes --- .../AnthropicMessagesChatModelApiFactory.java | 9 +- .../provider/AnthropicChatModelProvider.java | 8 +- .../AnthropicProviderConfiguration.java | 128 ++++++++++++++++-- .../aiagent/framework/ChatClientImplTest.java | 5 +- .../ChatModelApiRegistryImplTest.java | 5 +- .../langchain4j/ChatModelFactoryTest.java | 5 +- .../AnthropicChatModelProviderTest.java | 14 +- .../ChatModelProviderRegistryTest.java | 5 +- .../request/ProviderConfigurationTest.java | 52 ++++++- 9 files changed, 199 insertions(+), 32 deletions(-) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java index 153261b5557..654250145fd 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/anthropic/AnthropicMessagesChatModelApiFactory.java @@ -13,6 +13,7 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; import io.camunda.connector.agenticai.aiagent.framework.capabilities.ModelCapabilitiesResolver; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication.AnthropicApiKeyAuthentication; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicConnection; import java.time.Duration; import java.util.Optional; @@ -82,8 +83,12 @@ public ChatModelApi create(AnthropicProviderConfiguration configuration) { } private AnthropicClient buildClient(AnthropicConnection connection) { - final var builder = - AnthropicOkHttpClient.builder().apiKey(connection.authentication().apiKey()); + if (!(connection.authentication() instanceof AnthropicApiKeyAuthentication apiKeyAuth)) { + throw new IllegalArgumentException( + "Unsupported authentication type for DIRECT backend: " + + connection.authentication().getClass().getSimpleName()); + } + final var builder = AnthropicOkHttpClient.builder().apiKey(apiKeyAuth.apiKey()); if (StringUtils.isNotBlank(connection.endpoint())) { builder.baseUrl(normalizeBaseUrl(connection.endpoint())); diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/AnthropicChatModelProvider.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/AnthropicChatModelProvider.java index 85e0897618d..4faf37d032c 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/AnthropicChatModelProvider.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/AnthropicChatModelProvider.java @@ -12,6 +12,7 @@ import dev.langchain4j.model.chat.ChatModel; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelHttpProxySupport; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication.AnthropicApiKeyAuthentication; import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsConfigurationProperties.ChatModelProperties; import java.util.Optional; import org.slf4j.Logger; @@ -40,9 +41,14 @@ public String type() { public ChatModel createChatModel(AnthropicProviderConfiguration anthropic) { final var connection = anthropic.anthropic(); + if (!(connection.authentication() instanceof AnthropicApiKeyAuthentication apiKeyAuth)) { + throw new IllegalArgumentException( + "Unsupported authentication type for Anthropic LangChain4j provider: " + + connection.authentication().getClass().getSimpleName()); + } final var builder = AnthropicChatModel.builder() - .apiKey(connection.authentication().apiKey()) + .apiKey(apiKeyAuth.apiKey()) .modelName(connection.model().model()) .timeout( deriveTimeoutSetting("Anthropic model call", config, connection.timeouts(), LOGGER)) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/AnthropicProviderConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/AnthropicProviderConfiguration.java index addfce9d304..b26c9b05aca 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/AnthropicProviderConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/AnthropicProviderConfiguration.java @@ -8,12 +8,17 @@ import static io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.ANTHROPIC_ID; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.camunda.connector.agenticai.aiagent.model.request.provider.shared.HttpUrl; import io.camunda.connector.agenticai.aiagent.model.request.provider.shared.TimeoutConfiguration; import io.camunda.connector.generator.java.annotation.FeelMode; +import io.camunda.connector.generator.java.annotation.TemplateDiscriminatorProperty; import io.camunda.connector.generator.java.annotation.TemplateProperty; import io.camunda.connector.generator.java.annotation.TemplateSubType; import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertFalse; import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; @@ -30,6 +35,20 @@ public String providerType() { return ANTHROPIC_ID; } + public enum AnthropicBackend { + @JsonProperty("direct") + DIRECT, + + @JsonProperty("bedrock") + BEDROCK, + + @JsonProperty("vertex") + VERTEX, + + @JsonProperty("foundry") + FOUNDRY + } + public record AnthropicConnection( @HttpUrl @TemplateProperty( @@ -39,23 +58,106 @@ public record AnthropicConnection( feel = FeelMode.optional, optional = true) String endpoint, + @TemplateProperty(ignore = true) AnthropicBackend backend, @Valid @NotNull AnthropicAuthentication authentication, @Valid TimeoutConfiguration timeouts, - @Valid @NotNull AnthropicModel model) {} + @Valid @NotNull AnthropicModel model) { - public record AnthropicAuthentication( - @NotBlank - @TemplateProperty( - group = "provider", - label = "Anthropic API key", - type = TemplateProperty.PropertyType.String, - feel = FeelMode.optional, - constraints = @TemplateProperty.PropertyConstraints(notEmpty = true)) - String apiKey) { + public AnthropicConnection { + if (backend == null) { + backend = AnthropicBackend.DIRECT; + } + } + + @AssertFalse( + message = "Client credentials authentication is only supported for the FOUNDRY backend") + public boolean isClientCredentialsUsedWithNonFoundryBackend() { + return authentication + instanceof AnthropicAuthentication.AnthropicClientCredentialsAuthentication + && backend != AnthropicBackend.FOUNDRY; + } + } + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") + @JsonSubTypes({ + @JsonSubTypes.Type( + value = AnthropicAuthentication.AnthropicApiKeyAuthentication.class, + name = "apiKey"), + @JsonSubTypes.Type( + value = AnthropicAuthentication.AnthropicClientCredentialsAuthentication.class, + name = "clientCredentials") + }) + @TemplateDiscriminatorProperty( + label = "Authentication", + group = "provider", + name = "type", + defaultValue = "apiKey", + description = "Specify the Anthropic authentication strategy.") + public sealed interface AnthropicAuthentication { + + @TemplateSubType(id = "apiKey", label = "API key") + record AnthropicApiKeyAuthentication( + @NotBlank + @TemplateProperty( + group = "provider", + label = "Anthropic API key", + type = TemplateProperty.PropertyType.String, + feel = FeelMode.optional, + constraints = @TemplateProperty.PropertyConstraints(notEmpty = true)) + String apiKey) + implements AnthropicAuthentication { + + @Override + public String toString() { + return "AnthropicApiKeyAuthentication{apiKey=[REDACTED]}"; + } + } + + @TemplateSubType(id = "clientCredentials", label = "Client credentials") + record AnthropicClientCredentialsAuthentication( + @NotBlank + @TemplateProperty( + group = "provider", + label = "Client ID", + description = "ID of a Microsoft Entra application", + type = TemplateProperty.PropertyType.String, + feel = FeelMode.optional, + constraints = @TemplateProperty.PropertyConstraints(notEmpty = true)) + String clientId, + @NotBlank + @TemplateProperty( + group = "provider", + label = "Client secret", + description = "Secret of a Microsoft Entra application", + type = TemplateProperty.PropertyType.String, + feel = FeelMode.optional, + constraints = @TemplateProperty.PropertyConstraints(notEmpty = true)) + String clientSecret, + @NotBlank + @TemplateProperty( + group = "provider", + label = "Tenant ID", + description = + "ID of a Microsoft Entra tenant. Details in the documentation.", + type = TemplateProperty.PropertyType.String, + feel = FeelMode.optional) + String tenantId, + @TemplateProperty( + group = "provider", + label = "Authority host", + description = + "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", + type = TemplateProperty.PropertyType.String, + feel = FeelMode.optional, + optional = true) + String authorityHost) + implements AnthropicAuthentication { - @Override - public String toString() { - return "AnthropicAuthentication{apiKey=[REDACTED]}"; + @Override + public String toString() { + return "AnthropicClientCredentialsAuthentication{clientId=%s, clientSecret=[REDACTED], tenantId=%s, authorityHost=%s}" + .formatted(clientId, tenantId, authorityHost); + } } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImplTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImplTest.java index 9af20b10f8e..5d43db636af 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImplTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatClientImplTest.java @@ -35,7 +35,7 @@ import io.camunda.connector.agenticai.aiagent.model.request.ResponseFormatConfiguration.JsonResponseFormatConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.ResponseFormatConfiguration.TextResponseFormatConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication.AnthropicApiKeyAuthentication; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicConnection; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicModel; import io.camunda.connector.agenticai.model.message.AssistantMessage; @@ -62,7 +62,8 @@ class ChatClientImplTest { new AnthropicProviderConfiguration( new AnthropicConnection( null, - new AnthropicAuthentication("api-key"), + null, + new AnthropicApiKeyAuthentication("api-key"), null, new AnthropicModel("claude", null))); diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImplTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImplTest.java index 7374ee62652..01df1e570ad 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImplTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/ChatModelApiRegistryImplTest.java @@ -15,7 +15,7 @@ import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication.AnthropicApiKeyAuthentication; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicConnection; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicModel; import io.camunda.connector.agenticai.aiagent.model.request.provider.ProviderConfiguration; @@ -71,7 +71,8 @@ private static AnthropicProviderConfiguration validAnthropicConfig() { return new AnthropicProviderConfiguration( new AnthropicConnection( null, - new AnthropicAuthentication("api-key"), + null, + new AnthropicApiKeyAuthentication("api-key"), null, new AnthropicModel("claude", null))); } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatModelFactoryTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatModelFactoryTest.java index 94b05b336c7..be86f456816 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatModelFactoryTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/ChatModelFactoryTest.java @@ -15,7 +15,7 @@ import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProvider; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProviderRegistry; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication.AnthropicApiKeyAuthentication; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicConnection; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicModel; import io.camunda.connector.agenticai.aiagent.model.request.provider.ProviderConfiguration; @@ -36,7 +36,8 @@ void delegatesToProviderResolvedFromRegistry() { new AnthropicProviderConfiguration( new AnthropicConnection( null, - new AnthropicAuthentication("api-key"), + null, + new AnthropicApiKeyAuthentication("api-key"), null, new AnthropicModel("claude", null))); final var expectedChatModel = mock(ChatModel.class); diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/AnthropicChatModelProviderTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/AnthropicChatModelProviderTest.java index e74d7c3d8ab..acb2aca7f2f 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/AnthropicChatModelProviderTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/AnthropicChatModelProviderTest.java @@ -23,7 +23,7 @@ import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelHttpProxySupport; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProviderTestSupport.ResultCaptor; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication.AnthropicApiKeyAuthentication; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicConnection; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicModel; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicModel.AnthropicModelParameters; @@ -66,7 +66,8 @@ void createsAnthropicChatModel() { new AnthropicProviderConfiguration( new AnthropicConnection( null, - new AnthropicAuthentication(ANTHROPIC_API_KEY), + null, + new AnthropicApiKeyAuthentication(ANTHROPIC_API_KEY), MODEL_TIMEOUT, new AnthropicModel(ANTHROPIC_MODEL, DEFAULT_MODEL_PARAMETERS))); @@ -90,7 +91,8 @@ void createsAnthropicChatModelWithCustomEndpoint() { new AnthropicProviderConfiguration( new AnthropicConnection( "https://my-custom-endpoint.local", - new AnthropicAuthentication(ANTHROPIC_API_KEY), + null, + new AnthropicApiKeyAuthentication(ANTHROPIC_API_KEY), MODEL_TIMEOUT, new AnthropicModel(ANTHROPIC_MODEL, DEFAULT_MODEL_PARAMETERS))); @@ -109,7 +111,8 @@ void createsAnthropicChatModelWithNullModelParameters(AnthropicModelParameters m new AnthropicProviderConfiguration( new AnthropicConnection( null, - new AnthropicAuthentication(ANTHROPIC_API_KEY), + null, + new AnthropicApiKeyAuthentication(ANTHROPIC_API_KEY), MODEL_TIMEOUT, new AnthropicModel(ANTHROPIC_MODEL, modelParameters))); @@ -132,7 +135,8 @@ void createsAnthropicChatModelWithUnspecifiedTimeouts(TimeoutConfiguration timeo new AnthropicProviderConfiguration( new AnthropicConnection( null, - new AnthropicAuthentication(ANTHROPIC_API_KEY), + null, + new AnthropicApiKeyAuthentication(ANTHROPIC_API_KEY), timeouts, new AnthropicModel(ANTHROPIC_MODEL, null))); diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/ChatModelProviderRegistryTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/ChatModelProviderRegistryTest.java index da581451e58..e805bec7c1d 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/ChatModelProviderRegistryTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/ChatModelProviderRegistryTest.java @@ -12,7 +12,7 @@ import static org.mockito.Mockito.when; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication.AnthropicApiKeyAuthentication; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicConnection; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicModel; import io.camunda.connector.agenticai.aiagent.model.request.provider.ProviderConfiguration; @@ -92,7 +92,8 @@ private static AnthropicProviderConfiguration validAnthropicConfig() { return new AnthropicProviderConfiguration( new AnthropicConnection( null, - new AnthropicAuthentication("api-key"), + null, + new AnthropicApiKeyAuthentication("api-key"), null, new AnthropicModel("claude", null))); } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java index cde2d8a156d..d5285867799 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java @@ -10,7 +10,9 @@ import io.camunda.connector.agenticai.aiagent.model.request.MemoryStorageConfiguration.AwsAgentCoreAuthentication; import io.camunda.connector.agenticai.aiagent.model.request.MemoryStorageConfiguration.AwsAgentCoreMemoryStorageConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication.AnthropicApiKeyAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication.AnthropicClientCredentialsAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicBackend; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicConnection; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicModel; import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration.AzureAuthentication; @@ -150,7 +152,8 @@ void shouldAcceptValidEndpoint(String endpoint) { var connection = new AnthropicConnection( endpoint, - new AnthropicAuthentication("key"), + null, + new AnthropicApiKeyAuthentication("key"), TIMEOUT, new AnthropicModel("model", null)); assertThat(validator.validate(connection)).isEmpty(); @@ -163,13 +166,56 @@ void shouldRejectInvalidEndpoint(String endpoint) { var connection = new AnthropicConnection( endpoint, - new AnthropicAuthentication("key"), + null, + new AnthropicApiKeyAuthentication("key"), TIMEOUT, new AnthropicModel("model", null)); assertThat(validator.validate(connection)) .extracting(ConstraintViolation::getMessage) .contains(HTTP_URL_VALIDATION_MESSAGE); } + + @Test + void validationShouldSucceed_WhenApiKeyAuthenticationUsed() { + var connection = + new AnthropicConnection( + null, + null, + new AnthropicApiKeyAuthentication("my-api-key"), + TIMEOUT, + new AnthropicModel("model", null)); + assertThat(validator.validate(connection)).isEmpty(); + } + + @Test + void validationShouldSucceed_WhenClientCredentialsAuthUsedWithFoundryBackend() { + var connection = + new AnthropicConnection( + null, + AnthropicBackend.FOUNDRY, + new AnthropicClientCredentialsAuthentication( + "client-id", "client-secret", "tenant-id", null), + TIMEOUT, + new AnthropicModel("model", null)); + assertThat(validator.validate(connection)).isEmpty(); + } + + @Test + void validationShouldFail_WhenClientCredentialsAuthUsedWithDirectBackend() { + var connection = + new AnthropicConnection( + null, + AnthropicBackend.DIRECT, + new AnthropicClientCredentialsAuthentication( + "client-id", "client-secret", "tenant-id", null), + TIMEOUT, + new AnthropicModel("model", null)); + assertThat(validator.validate(connection)) + .hasSize(1) + .extracting(ConstraintViolation::getMessage) + .containsExactly( + "Client credentials authentication is only supported for the FOUNDRY backend"); + } } @Nested From 5ae3a8f7ade24abad1f2d6cd5e0aa608d6ced709 Mon Sep 17 00:00:00 2001 From: Dmitri Nikonov Date: Fri, 8 May 2026 10:31:09 +0200 Subject: [PATCH 69/81] refactor(agentic-ai): address review nits on AnthropicAuthentication (Phase F) - Add @NotNull to AnthropicApiKeyAuthentication.toString() to match AzureApiKeyAuthentication - Remove blank line between AnthropicAuthentication opening brace and first @TemplateSubType - Expand @AssertFalse test to cover BEDROCK and VERTEX backends via @EnumSource parameterized test --- .../provider/AnthropicProviderConfiguration.java | 3 +-- .../model/request/ProviderConfigurationTest.java | 11 ++++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/AnthropicProviderConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/AnthropicProviderConfiguration.java index b26c9b05aca..611b003ae33 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/AnthropicProviderConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/AnthropicProviderConfiguration.java @@ -94,7 +94,6 @@ public boolean isClientCredentialsUsedWithNonFoundryBackend() { defaultValue = "apiKey", description = "Specify the Anthropic authentication strategy.") public sealed interface AnthropicAuthentication { - @TemplateSubType(id = "apiKey", label = "API key") record AnthropicApiKeyAuthentication( @NotBlank @@ -108,7 +107,7 @@ record AnthropicApiKeyAuthentication( implements AnthropicAuthentication { @Override - public String toString() { + public @NotNull String toString() { return "AnthropicApiKeyAuthentication{apiKey=[REDACTED]}"; } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java index d5285867799..6367a8e0d32 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java @@ -40,6 +40,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.NullAndEmptySource; import org.junit.jupiter.params.provider.NullSource; @@ -200,12 +201,16 @@ void validationShouldSucceed_WhenClientCredentialsAuthUsedWithFoundryBackend() { assertThat(validator.validate(connection)).isEmpty(); } - @Test - void validationShouldFail_WhenClientCredentialsAuthUsedWithDirectBackend() { + @ParameterizedTest + @EnumSource( + value = AnthropicBackend.class, + names = {"DIRECT", "BEDROCK", "VERTEX"}) + void validationShouldFail_WhenClientCredentialsAuthUsedWithNonFoundryBackend( + AnthropicBackend backend) { var connection = new AnthropicConnection( null, - AnthropicBackend.DIRECT, + backend, new AnthropicClientCredentialsAuthentication( "client-id", "client-secret", "tenant-id", null), TIMEOUT, From 748f087bb555badcd603e5e9697e8ec38ca1eba7 Mon Sep 17 00:00:00 2001 From: Dmitri Nikonov Date: Fri, 8 May 2026 10:35:37 +0200 Subject: [PATCH 70/81] =?UTF-8?q?refactor(agentic-ai):=20rename=20GoogleVe?= =?UTF-8?q?rtexAi=20=E2=86=92=20GoogleGenAi=20+=20add=20GoogleBackend=20(P?= =?UTF-8?q?hase=20F)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renames GoogleVertexAiProviderConfiguration to GoogleGenAiProviderConfiguration with id "googleGenAi", adds GoogleBackend{DEVELOPER_API,VERTEX} enum with @JsonProperty values, and adds a backend field to GoogleGenAiConnection (null→VERTEX migration default). Updates all callsites: ProviderConfiguration sealed interface, Langchain4J provider and factory wiring, and all test files. --- ...icAiLangchain4JChatModelConfiguration.java | 4 +- ...icAiLangchain4JFrameworkConfiguration.java | 10 ++-- .../GoogleVertexAiChatModelProvider.java | 12 ++--- ... => GoogleGenAiProviderConfiguration.java} | 52 ++++++++++++------- .../provider/ProviderConfiguration.java | 6 +-- .../GoogleVertexAiChatModelProviderTest.java | 45 ++++++++-------- .../request/ProviderConfigurationTest.java | 30 ++++++----- ...nticAiConnectorsAutoConfigurationTest.java | 28 +++++----- 8 files changed, 103 insertions(+), 84 deletions(-) rename connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/{GoogleVertexAiProviderConfiguration.java => GoogleGenAiProviderConfiguration.java} (81%) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JChatModelConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JChatModelConfiguration.java index f971a1f4200..da3b2812e14 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JChatModelConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JChatModelConfiguration.java @@ -16,7 +16,7 @@ import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.GoogleVertexAiChatModelProvider; import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration; import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsConfigurationProperties; import io.camunda.connector.agenticai.common.AgenticAiHttpProxySupport; import java.util.List; @@ -56,7 +56,7 @@ public ChatModelProvider langchain4JBedrockChatMod @Bean @ConditionalOnMissingBean - public ChatModelProvider + public ChatModelProvider langchain4JGoogleVertexAiChatModelProvider() { return new GoogleVertexAiChatModelProvider(); } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java index 07180e96c3f..d82df7ccde9 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java @@ -23,7 +23,7 @@ import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverterImpl; import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration; import io.camunda.connector.runtime.annotation.ConnectorsObjectMapper; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -116,15 +116,15 @@ public ChatModelApiFactory langchain4JBedrockChatM @Bean @ConditionalOnMissingBean(name = "langchain4JGoogleVertexAiChatModelApiFactory") - public ChatModelApiFactory + public ChatModelApiFactory langchain4JGoogleVertexAiChatModelApiFactory( - ChatModelProvider provider, + ChatModelProvider provider, ChatMessageConverter chatMessageConverter, ToolSpecificationConverter toolSpecificationConverter, JsonSchemaConverter jsonSchemaConverter) { return new Langchain4JChatModelApiFactory<>( - GoogleVertexAiProviderConfiguration.GOOGLE_VERTEX_AI_ID, - GoogleVertexAiProviderConfiguration.class, + GoogleGenAiProviderConfiguration.GOOGLE_GENAI_ID, + GoogleGenAiProviderConfiguration.class, provider, chatMessageConverter, toolSpecificationConverter, diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/GoogleVertexAiChatModelProvider.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/GoogleVertexAiChatModelProvider.java index 0c7ddc3a31c..1052c678c6d 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/GoogleVertexAiChatModelProvider.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/GoogleVertexAiChatModelProvider.java @@ -9,8 +9,8 @@ import com.google.auth.oauth2.ServiceAccountCredentials; import dev.langchain4j.model.chat.ChatModel; import dev.langchain4j.model.vertexai.gemini.VertexAiGeminiChatModel; -import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration.GoogleVertexAiAuthentication.ServiceAccountCredentialsAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GoogleGenAiAuthentication.ServiceAccountCredentialsAuthentication; import io.camunda.connector.api.error.ConnectorInputException; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -20,19 +20,19 @@ import org.slf4j.LoggerFactory; public class GoogleVertexAiChatModelProvider - implements ChatModelProvider { + implements ChatModelProvider { private static final Logger LOGGER = LoggerFactory.getLogger(GoogleVertexAiChatModelProvider.class); @Override public String type() { - return GoogleVertexAiProviderConfiguration.GOOGLE_VERTEX_AI_ID; + return GoogleGenAiProviderConfiguration.GOOGLE_GENAI_ID; } @Override - public ChatModel createChatModel(GoogleVertexAiProviderConfiguration vertexAi) { - final var connection = vertexAi.googleVertexAi(); + public ChatModel createChatModel(GoogleGenAiProviderConfiguration vertexAi) { + final var connection = vertexAi.googleGenAi(); final var builder = VertexAiGeminiChatModel.builder() .project(connection.projectId()) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/GoogleVertexAiProviderConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/GoogleGenAiProviderConfiguration.java similarity index 81% rename from connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/GoogleVertexAiProviderConfiguration.java rename to connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/GoogleGenAiProviderConfiguration.java index e4fbcc9203e..d179cebd5b1 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/GoogleVertexAiProviderConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/GoogleGenAiProviderConfiguration.java @@ -6,8 +6,9 @@ */ package io.camunda.connector.agenticai.aiagent.model.request.provider; -import static io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration.GOOGLE_VERTEX_AI_ID; +import static io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GOOGLE_GENAI_ID; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.camunda.connector.agenticai.util.ConnectorUtils; @@ -21,19 +22,27 @@ import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; -@TemplateSubType(id = GOOGLE_VERTEX_AI_ID, label = "Google Vertex AI") -public record GoogleVertexAiProviderConfiguration( - @Valid @NotNull GoogleVertexAiConnection googleVertexAi) implements ProviderConfiguration { +@TemplateSubType(id = GOOGLE_GENAI_ID, label = "Google GenAI") +public record GoogleGenAiProviderConfiguration(@Valid @NotNull GoogleGenAiConnection googleGenAi) + implements ProviderConfiguration { @TemplateProperty(ignore = true) - public static final String GOOGLE_VERTEX_AI_ID = "google-vertex-ai"; + public static final String GOOGLE_GENAI_ID = "googleGenAi"; @Override public String providerType() { - return GOOGLE_VERTEX_AI_ID; + return GOOGLE_GENAI_ID; } - public record GoogleVertexAiConnection( + public enum GoogleBackend { + @JsonProperty("developer-api") + DEVELOPER_API, + + @JsonProperty("vertex") + VERTEX + } + + public record GoogleGenAiConnection( @NotBlank @TemplateProperty( group = "provider", @@ -52,24 +61,31 @@ public record GoogleVertexAiConnection( feel = FeelMode.optional, constraints = @TemplateProperty.PropertyConstraints(notEmpty = true)) String region, - @Valid @NotNull GoogleVertexAiAuthentication authentication, - @Valid @NotNull GoogleVertexAiProviderConfiguration.GoogleVertexAiModel model) { + @Valid @NotNull GoogleGenAiAuthentication authentication, + @Valid @NotNull GoogleGenAiModel model, + @TemplateProperty(ignore = true) GoogleBackend backend) { + + public GoogleGenAiConnection { + if (backend == null) { + backend = GoogleBackend.VERTEX; + } + } @AssertFalse(message = "Google Vertex AI is not supported on SaaS") public boolean isUsedInSaaS() { return ConnectorUtils.isSaaS() && authentication - instanceof GoogleVertexAiAuthentication.ApplicationDefaultCredentialsAuthentication; + instanceof GoogleGenAiAuthentication.ApplicationDefaultCredentialsAuthentication; } } @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes({ @JsonSubTypes.Type( - value = GoogleVertexAiAuthentication.ServiceAccountCredentialsAuthentication.class, + value = GoogleGenAiAuthentication.ServiceAccountCredentialsAuthentication.class, name = "serviceAccountCredentials"), @JsonSubTypes.Type( - value = GoogleVertexAiAuthentication.ApplicationDefaultCredentialsAuthentication.class, + value = GoogleGenAiAuthentication.ApplicationDefaultCredentialsAuthentication.class, name = "applicationDefaultCredentials"), }) @TemplateDiscriminatorProperty( @@ -78,7 +94,7 @@ public boolean isUsedInSaaS() { name = "type", defaultValue = "serviceAccountCredentials", description = "Specify the Google Vertex AI authentication strategy.") - public sealed interface GoogleVertexAiAuthentication { + public sealed interface GoogleGenAiAuthentication { @TemplateSubType(id = "serviceAccountCredentials", label = "Service account credentials") record ServiceAccountCredentialsAuthentication( @NotBlank @@ -89,7 +105,7 @@ record ServiceAccountCredentialsAuthentication( feel = FeelMode.optional, constraints = @TemplateProperty.PropertyConstraints(notEmpty = true)) String jsonKey) - implements GoogleVertexAiAuthentication { + implements GoogleGenAiAuthentication { @Override public String toString() { return "ServiceAccountCredentialsAuthentication{jsonKey=[REDACTED]}"; @@ -99,10 +115,10 @@ public String toString() { @TemplateSubType( id = "applicationDefaultCredentials", label = "Application default credentials (Hybrid/Self-Managed only)") - record ApplicationDefaultCredentialsAuthentication() implements GoogleVertexAiAuthentication {} + record ApplicationDefaultCredentialsAuthentication() implements GoogleGenAiAuthentication {} } - public record GoogleVertexAiModel( + public record GoogleGenAiModel( @NotBlank @TemplateProperty( group = "model", @@ -113,9 +129,9 @@ public record GoogleVertexAiModel( feel = FeelMode.optional, constraints = @TemplateProperty.PropertyConstraints(notEmpty = true)) String model, - @Valid GoogleVertexAiModelParameters parameters) { + @Valid GoogleGenAiModelParameters parameters) { - public record GoogleVertexAiModelParameters( + public record GoogleGenAiModelParameters( @Min(0) @TemplateProperty( group = "model", diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/ProviderConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/ProviderConfiguration.java index e95076773a4..c3b5c5d279f 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/ProviderConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/ProviderConfiguration.java @@ -9,7 +9,7 @@ import static io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.ANTHROPIC_ID; import static io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration.AZURE_OPENAI_ID; import static io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration.BEDROCK_ID; -import static io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration.GOOGLE_VERTEX_AI_ID; +import static io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GOOGLE_GENAI_ID; import static io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration.OPENAI_COMPATIBLE_ID; import static io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OPENAI_ID; @@ -22,7 +22,7 @@ @JsonSubTypes.Type(value = AnthropicProviderConfiguration.class, name = ANTHROPIC_ID), @JsonSubTypes.Type(value = BedrockProviderConfiguration.class, name = BEDROCK_ID), @JsonSubTypes.Type(value = AzureOpenAiProviderConfiguration.class, name = AZURE_OPENAI_ID), - @JsonSubTypes.Type(value = GoogleVertexAiProviderConfiguration.class, name = GOOGLE_VERTEX_AI_ID), + @JsonSubTypes.Type(value = GoogleGenAiProviderConfiguration.class, name = GOOGLE_GENAI_ID), @JsonSubTypes.Type(value = OpenAiProviderConfiguration.class, name = OPENAI_ID), @JsonSubTypes.Type( value = OpenAiCompatibleProviderConfiguration.class, @@ -38,7 +38,7 @@ public sealed interface ProviderConfiguration permits AnthropicProviderConfiguration, BedrockProviderConfiguration, AzureOpenAiProviderConfiguration, - GoogleVertexAiProviderConfiguration, + GoogleGenAiProviderConfiguration, OpenAiProviderConfiguration, OpenAiCompatibleProviderConfiguration { diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/GoogleVertexAiChatModelProviderTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/GoogleVertexAiChatModelProviderTest.java index 317048c09b1..cdd52cfb25c 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/GoogleVertexAiChatModelProviderTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/GoogleVertexAiChatModelProviderTest.java @@ -23,12 +23,12 @@ import dev.langchain4j.model.vertexai.gemini.VertexAiGeminiChatModel; import dev.langchain4j.model.vertexai.gemini.VertexAiGeminiChatModel.VertexAiGeminiChatModelBuilder; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProviderTestSupport.ResultCaptor; -import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration.GoogleVertexAiAuthentication.ApplicationDefaultCredentialsAuthentication; -import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration.GoogleVertexAiAuthentication.ServiceAccountCredentialsAuthentication; -import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration.GoogleVertexAiConnection; -import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration.GoogleVertexAiModel; -import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration.GoogleVertexAiModel.GoogleVertexAiModelParameters; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GoogleGenAiAuthentication.ApplicationDefaultCredentialsAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GoogleGenAiAuthentication.ServiceAccountCredentialsAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GoogleGenAiConnection; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GoogleGenAiModel; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GoogleGenAiModel.GoogleGenAiModelParameters; import java.util.stream.Stream; import org.assertj.core.api.ThrowingConsumer; import org.junit.jupiter.api.Test; @@ -47,20 +47,21 @@ class GoogleVertexAiChatModelProviderTest { private static final String REGION = "us-central1"; private static final String MODEL = "gemini-2.5-pro"; - private static final GoogleVertexAiModelParameters DEFAULT_MODEL_PARAMETERS = - new GoogleVertexAiModelParameters(10, 1.0F, 0.8F, 100); + private static final GoogleGenAiModelParameters DEFAULT_MODEL_PARAMETERS = + new GoogleGenAiModelParameters(10, 1.0F, 0.8F, 100); private final GoogleVertexAiChatModelProvider provider = new GoogleVertexAiChatModelProvider(); @Test void createsGoogleVertexAiChatModel() { final var providerConfig = - new GoogleVertexAiProviderConfiguration( - new GoogleVertexAiConnection( + new GoogleGenAiProviderConfiguration( + new GoogleGenAiConnection( PROJECT_ID, REGION, new ApplicationDefaultCredentialsAuthentication(), - new GoogleVertexAiModel(MODEL, DEFAULT_MODEL_PARAMETERS))); + new GoogleGenAiModel(MODEL, DEFAULT_MODEL_PARAMETERS), + null)); testGoogleVertexAiChatModelBuilder( providerConfig, @@ -79,14 +80,15 @@ void createsGoogleVertexAiChatModel() { @NullSource @MethodSource("nullModelParameters") void createsGoogleVertexAiChatModelWithNullModelParameters( - GoogleVertexAiModelParameters modelParameters) { + GoogleGenAiModelParameters modelParameters) { final var providerConfig = - new GoogleVertexAiProviderConfiguration( - new GoogleVertexAiConnection( + new GoogleGenAiProviderConfiguration( + new GoogleGenAiConnection( PROJECT_ID, REGION, new ApplicationDefaultCredentialsAuthentication(), - new GoogleVertexAiModel(MODEL, modelParameters))); + new GoogleGenAiModel(MODEL, modelParameters), + null)); testGoogleVertexAiChatModelBuilder( providerConfig, @@ -101,12 +103,13 @@ void createsGoogleVertexAiChatModelWithNullModelParameters( @Test void createsGoogleVertexAiChatModelWithServiceAccountCredential() { final var providerConfig = - new GoogleVertexAiProviderConfiguration( - new GoogleVertexAiConnection( + new GoogleGenAiProviderConfiguration( + new GoogleGenAiConnection( PROJECT_ID, REGION, new ServiceAccountCredentialsAuthentication("{}"), - new GoogleVertexAiModel(MODEL, DEFAULT_MODEL_PARAMETERS))); + new GoogleGenAiModel(MODEL, DEFAULT_MODEL_PARAMETERS), + null)); try (final var staticMockedSac = mockStatic(ServiceAccountCredentials.class)) { final var mockedSac = mock(ServiceAccountCredentials.class); @@ -131,7 +134,7 @@ void createsGoogleVertexAiChatModelWithServiceAccountCredential() { } private void testGoogleVertexAiChatModelBuilder( - GoogleVertexAiProviderConfiguration providerConfig, + GoogleGenAiProviderConfiguration providerConfig, ThrowingConsumer builderAssertions) { final var chatModelBuilder = spy(VertexAiGeminiChatModel.builder()); final var chatModelResultCaptor = new ResultCaptor(); @@ -149,7 +152,7 @@ private void testGoogleVertexAiChatModelBuilder( } } - static Stream nullModelParameters() { - return Stream.of(new GoogleVertexAiModelParameters(null, null, null, null)); + static Stream nullModelParameters() { + return Stream.of(new GoogleGenAiModelParameters(null, null, null, null)); } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java index 6367a8e0d32..ab1dff5d672 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java @@ -21,11 +21,11 @@ import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration.AwsAuthentication; import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration.BedrockConnection; -import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration.GoogleVertexAiAuthentication.ApplicationDefaultCredentialsAuthentication; -import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration.GoogleVertexAiAuthentication.ServiceAccountCredentialsAuthentication; -import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration.GoogleVertexAiConnection; -import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration.GoogleVertexAiModel; -import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration.GoogleVertexAiModel.GoogleVertexAiModelParameters; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GoogleGenAiAuthentication.ApplicationDefaultCredentialsAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GoogleGenAiAuthentication.ServiceAccountCredentialsAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GoogleGenAiConnection; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GoogleGenAiModel; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GoogleGenAiModel.GoogleGenAiModelParameters; import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration.OpenAiCompatibleAuthentication; import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration.OpenAiCompatibleConnection; import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration.OpenAiCompatibleModel; @@ -322,7 +322,7 @@ void shouldRejectBlankEndpoint(String endpoint) { } @Nested - class GoogleVertexAiConnectionTest { + class GoogleGenAiConnectionTest { @Test void validationShouldSucceed_WhenNotSaaS() { @@ -353,22 +353,24 @@ void validationShouldSucceed_WhenNotSaaSAndServiceAccountCredentialsUsed() { assertThat(validator.validate(connection)).isEmpty(); } - private static GoogleVertexAiConnection createConnectionWithApplicationDefaultCredentials() { - return new GoogleVertexAiConnection( + private static GoogleGenAiConnection createConnectionWithApplicationDefaultCredentials() { + return new GoogleGenAiConnection( "my-project-id", "us-central1", new ApplicationDefaultCredentialsAuthentication(), - new GoogleVertexAiModel( - "gemini-1.5-flash", new GoogleVertexAiModelParameters(null, null, null, null))); + new GoogleGenAiModel( + "gemini-1.5-flash", new GoogleGenAiModelParameters(null, null, null, null)), + null); } - private static GoogleVertexAiConnection createConnectionWithServiceAccountCredentials() { - return new GoogleVertexAiConnection( + private static GoogleGenAiConnection createConnectionWithServiceAccountCredentials() { + return new GoogleGenAiConnection( "my-project-id", "us-central1", new ServiceAccountCredentialsAuthentication("{}"), - new GoogleVertexAiModel( - "gemini-1.5-flash", new GoogleVertexAiModelParameters(null, null, null, null))); + new GoogleGenAiModel( + "gemini-1.5-flash", new GoogleGenAiModelParameters(null, null, null, null)), + null); } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java index 535e436ee6b..59bed337e57 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java @@ -50,12 +50,12 @@ import io.camunda.connector.agenticai.aiagent.memory.conversation.inprocess.InProcessConversationStore; import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleVertexAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.ProviderConfiguration; import io.camunda.connector.agenticai.aiagent.tool.GatewayToolHandlerRegistry; import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsAutoConfigurationTest.CustomChatModelProviderOverrides.CustomAzureOpenAiProviderConfig.CustomAzureOpenAiChatModelProvider; import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsAutoConfigurationTest.CustomChatModelProviderOverrides.CustomBedrockProviderConfig.CustomBedrockChatModelProvider; -import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsAutoConfigurationTest.CustomChatModelProviderOverrides.CustomGoogleVertexAiProviderConfig.CustomGoogleVertexAiChatModelProvider; +import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsAutoConfigurationTest.CustomChatModelProviderOverrides.CustomGoogleGenAiProviderConfig.CustomGoogleGenAiChatModelProvider; import io.camunda.connector.agenticai.common.AgenticAiHttpProxySupport; import io.camunda.connector.http.client.proxy.EnvironmentProxyConfiguration; import java.util.List; @@ -315,10 +315,10 @@ static Stream providerOverrideCases() { BedrockProviderConfiguration.class, CustomBedrockChatModelProvider.class), new ProviderOverrideCase( - CustomGoogleVertexAiProviderConfig.class, - "customGoogleVertexAiChatModelProvider", - GoogleVertexAiProviderConfiguration.class, - CustomGoogleVertexAiChatModelProvider.class)); + CustomGoogleGenAiProviderConfig.class, + "customGoogleGenAiChatModelProvider", + GoogleGenAiProviderConfiguration.class, + CustomGoogleGenAiChatModelProvider.class)); } record ProviderOverrideCase( @@ -375,24 +375,22 @@ public ChatModel createChatModel(BedrockProviderConfiguration providerConfigurat } } - static class CustomGoogleVertexAiProviderConfig { + static class CustomGoogleGenAiProviderConfig { @Bean - ChatModelProvider - customGoogleVertexAiChatModelProvider() { - return new CustomGoogleVertexAiChatModelProvider(); + ChatModelProvider customGoogleGenAiChatModelProvider() { + return new CustomGoogleGenAiChatModelProvider(); } - static class CustomGoogleVertexAiChatModelProvider - implements ChatModelProvider { + static class CustomGoogleGenAiChatModelProvider + implements ChatModelProvider { @Override public String type() { - return GoogleVertexAiProviderConfiguration.GOOGLE_VERTEX_AI_ID; + return GoogleGenAiProviderConfiguration.GOOGLE_GENAI_ID; } @Override - public ChatModel createChatModel( - GoogleVertexAiProviderConfiguration providerConfiguration) { + public ChatModel createChatModel(GoogleGenAiProviderConfiguration providerConfiguration) { return mock(ChatModel.class); } } From 36fd7c963b0f96612381beda618a86bbaf7a1d9b Mon Sep 17 00:00:00 2001 From: Dmitri Nikonov Date: Fri, 8 May 2026 10:39:27 +0200 Subject: [PATCH 71/81] refactor(agentic-ai): update stale SaaS validation message in GoogleGenAiConnection --- .../request/provider/GoogleGenAiProviderConfiguration.java | 2 +- .../aiagent/model/request/ProviderConfigurationTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/GoogleGenAiProviderConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/GoogleGenAiProviderConfiguration.java index d179cebd5b1..1aa51c2ccda 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/GoogleGenAiProviderConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/GoogleGenAiProviderConfiguration.java @@ -71,7 +71,7 @@ public record GoogleGenAiConnection( } } - @AssertFalse(message = "Google Vertex AI is not supported on SaaS") + @AssertFalse(message = "Google GenAI is not supported on SaaS") public boolean isUsedInSaaS() { return ConnectorUtils.isSaaS() && authentication diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java index ab1dff5d672..a1433388087 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java @@ -337,7 +337,7 @@ void validationShouldFail_WhenSaaS() { assertThat(validator.validate(connection)) .hasSize(1) .extracting(ConstraintViolation::getMessage) - .containsExactly("Google Vertex AI is not supported on SaaS"); + .containsExactly("Google GenAI is not supported on SaaS"); } @Test From 554a52ebee06c66ed9e70c964f336f9768d76073 Mon Sep 17 00:00:00 2001 From: Dmitri Nikonov Date: Fri, 8 May 2026 10:42:10 +0200 Subject: [PATCH 72/81] refactor(agentic-ai): add Bedrock non-Anthropic model validation (Phase F) --- .../BedrockProviderConfiguration.java | 8 ++++ .../request/ProviderConfigurationTest.java | 37 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/BedrockProviderConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/BedrockProviderConfiguration.java index 7ad415ae2d8..2e72f428e92 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/BedrockProviderConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/BedrockProviderConfiguration.java @@ -63,6 +63,14 @@ public boolean isDefaultCredentialsChainUsedInSaaS() { return ConnectorUtils.isSaaS() && authentication instanceof AwsAuthentication.AwsDefaultCredentialsChainAuthentication; } + + @AssertFalse( + message = + "Anthropic models must be configured via the Anthropic provider with backend = BEDROCK") + @SuppressWarnings("unused") + public boolean isAnthropicModelUsed() { + return model != null && model.model() != null && model.model().startsWith("anthropic."); + } } @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java index a1433388087..90024ab693d 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java @@ -129,6 +129,30 @@ void shouldRejectInvalidEndpoint(String endpoint) { .contains(HTTP_URL_VALIDATION_MESSAGE); } + @Test + void validationShouldFail_WhenAnthropicModelUsedWithBedrock() { + var connection = + createConnection( + null, + new AwsAuthentication.AwsStaticCredentialsAuthentication("key", "secret"), + "anthropic.claude-sonnet-4-5"); + assertThat(validator.validate(connection)) + .hasSize(1) + .extracting(ConstraintViolation::getMessage) + .containsExactly( + "Anthropic models must be configured via the Anthropic provider with backend = BEDROCK"); + } + + @Test + void validationShouldSucceed_WhenNonAnthropicModelUsedWithBedrock() { + var connection = + createConnection( + null, + new AwsAuthentication.AwsStaticCredentialsAuthentication("key", "secret"), + "amazon.nova-pro-v1:0"); + assertThat(validator.validate(connection)).isEmpty(); + } + private BedrockConnection createConnection(String endpoint, AwsAuthentication authentication) { return new BedrockConnection( "eu-central-1", @@ -140,6 +164,19 @@ private BedrockConnection createConnection(String endpoint, AwsAuthentication au new BedrockProviderConfiguration.BedrockModel.BedrockModelParameters( null, null, null))); } + + private BedrockConnection createConnection( + String endpoint, AwsAuthentication authentication, String modelId) { + return new BedrockConnection( + "eu-central-1", + endpoint, + authentication, + TIMEOUT, + new BedrockProviderConfiguration.BedrockModel( + modelId, + new BedrockProviderConfiguration.BedrockModel.BedrockModelParameters( + null, null, null))); + } } @Nested From 8747dd44ff3f85c2155d2c59a95917029ed37cc5 Mon Sep 17 00:00:00 2001 From: Dmitri Nikonov Date: Fri, 8 May 2026 10:59:21 +0200 Subject: [PATCH 73/81] refactor(agentic-ai): consolidate OpenAI configs into single OpenAiProviderConfiguration (Phase F) Introduce OpenAiDispatchingChatModelProvider to route OPENAI/FOUNDRY/CUSTOM backends to OpenAiChatModelProvider, AzureOpenAiChatModelProvider, and OpenAiCompatibleChatModelProvider respectively. Wire it in AgenticAiLangchain4JChatModelConfiguration as the single ChatModelProvider bean replacing the previous AzureOpenAiChatModelProvider-only registration. --- ...icAiLangchain4JChatModelConfiguration.java | 19 ++++--- .../OpenAiDispatchingChatModelProvider.java | 57 +++++++++++++++++++ ...nticAiConnectorsAutoConfigurationTest.java | 40 ++++++------- 3 files changed, 90 insertions(+), 26 deletions(-) create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiDispatchingChatModelProvider.java diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JChatModelConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JChatModelConfiguration.java index da3b2812e14..aac815a4f8a 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JChatModelConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JChatModelConfiguration.java @@ -14,9 +14,12 @@ import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProvider; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProviderRegistry; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.GoogleVertexAiChatModelProvider; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.OpenAiChatModelProvider; +import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.OpenAiCompatibleChatModelProvider; +import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.OpenAiDispatchingChatModelProvider; import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsConfigurationProperties; import io.camunda.connector.agenticai.common.AgenticAiHttpProxySupport; import java.util.List; @@ -38,12 +41,14 @@ public ChatModelHttpProxySupport langchain4JChatModelHttpProxySupport( @Bean @ConditionalOnMissingBean - public ChatModelProvider - langchain4JAzureOpenAiChatModelProvider( - AgenticAiConnectorsConfigurationProperties config, - ChatModelHttpProxySupport chatModelHttpProxySupport) { - return new AzureOpenAiChatModelProvider( - config.aiagent().chatModel(), chatModelHttpProxySupport); + public ChatModelProvider langchain4JAzureOpenAiChatModelProvider( + AgenticAiConnectorsConfigurationProperties config, + ChatModelHttpProxySupport chatModelHttpProxySupport) { + final var chatModelProperties = config.aiagent().chatModel(); + return new OpenAiDispatchingChatModelProvider( + new OpenAiChatModelProvider(chatModelProperties, chatModelHttpProxySupport), + new AzureOpenAiChatModelProvider(chatModelProperties, chatModelHttpProxySupport), + new OpenAiCompatibleChatModelProvider(chatModelProperties, chatModelHttpProxySupport)); } @Bean diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiDispatchingChatModelProvider.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiDispatchingChatModelProvider.java new file mode 100644 index 00000000000..46b15a1e1d4 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiDispatchingChatModelProvider.java @@ -0,0 +1,57 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider; + +import dev.langchain4j.model.chat.ChatModel; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiBackend; + +/** + * Dispatching {@link ChatModelProvider} for the unified {@link OpenAiProviderConfiguration}. Routes + * each request to the appropriate backend-specific provider: + * + *

          + *
        • {@link OpenAiBackend#OPENAI} → {@link OpenAiChatModelProvider} + *
        • {@link OpenAiBackend#FOUNDRY} → {@link AzureOpenAiChatModelProvider} + *
        • {@link OpenAiBackend#CUSTOM} → {@link OpenAiCompatibleChatModelProvider} + *
        + * + *

        This bean is registered as the single {@code ChatModelProvider} + * in the Spring context and acts as the LangChain4j fallback when the native OpenAI SDK factory is + * not on the classpath (or is excluded). + */ +public class OpenAiDispatchingChatModelProvider + implements ChatModelProvider { + + private final OpenAiChatModelProvider openAiProvider; + private final AzureOpenAiChatModelProvider azureOpenAiProvider; + private final OpenAiCompatibleChatModelProvider openAiCompatibleProvider; + + public OpenAiDispatchingChatModelProvider( + OpenAiChatModelProvider openAiProvider, + AzureOpenAiChatModelProvider azureOpenAiProvider, + OpenAiCompatibleChatModelProvider openAiCompatibleProvider) { + this.openAiProvider = openAiProvider; + this.azureOpenAiProvider = azureOpenAiProvider; + this.openAiCompatibleProvider = openAiCompatibleProvider; + } + + @Override + public String type() { + return OpenAiProviderConfiguration.OPENAI_ID; + } + + @Override + public ChatModel createChatModel(OpenAiProviderConfiguration providerConfiguration) { + final var backend = providerConfiguration.openai().backend(); + return switch (backend) { + case OPENAI -> openAiProvider.createChatModel(providerConfiguration); + case FOUNDRY -> azureOpenAiProvider.createChatModel(providerConfiguration); + case CUSTOM -> openAiCompatibleProvider.createChatModel(providerConfiguration); + }; + } +} diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java index 59bed337e57..7a27a25f28f 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/autoconfigure/AgenticAiConnectorsAutoConfigurationTest.java @@ -36,11 +36,11 @@ import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ContentConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.document.DocumentToContentConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.jsonschema.JsonSchemaConverter; -import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.AzureOpenAiChatModelProvider; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.BedrockChatModelProvider; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProvider; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProviderRegistry; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.GoogleVertexAiChatModelProvider; +import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.OpenAiDispatchingChatModelProvider; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolCallConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverter; import io.camunda.connector.agenticai.aiagent.memory.conversation.ConversationStoreRegistry; @@ -48,13 +48,13 @@ import io.camunda.connector.agenticai.aiagent.memory.conversation.awsagentcore.mapping.AwsAgentCoreConversationMapper; import io.camunda.connector.agenticai.aiagent.memory.conversation.document.CamundaDocumentConversationStore; import io.camunda.connector.agenticai.aiagent.memory.conversation.inprocess.InProcessConversationStore; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.ProviderConfiguration; import io.camunda.connector.agenticai.aiagent.tool.GatewayToolHandlerRegistry; -import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsAutoConfigurationTest.CustomChatModelProviderOverrides.CustomAzureOpenAiProviderConfig.CustomAzureOpenAiChatModelProvider; import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsAutoConfigurationTest.CustomChatModelProviderOverrides.CustomBedrockProviderConfig.CustomBedrockChatModelProvider; +import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsAutoConfigurationTest.CustomChatModelProviderOverrides.CustomFoundryProviderConfig.CustomFoundryChatModelProvider; import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsAutoConfigurationTest.CustomChatModelProviderOverrides.CustomGoogleGenAiProviderConfig.CustomGoogleGenAiChatModelProvider; import io.camunda.connector.agenticai.common.AgenticAiHttpProxySupport; import io.camunda.connector.http.client.proxy.EnvironmentProxyConfiguration; @@ -100,13 +100,14 @@ class AgenticAiConnectorsAutoConfigurationTest { JobWorkerAgentRequestHandler.class, AiAgentJobWorker.class); - // L4J factory + provider beans for `anthropic`, `openai`, `openaiCompatible` were dropped when - // those providers landed native in ADR-005 Phase B/C/D — only the still-bridged providers - // (Bedrock, Azure OpenAI, Google Vertex AI) keep an L4J `ChatModelProvider` bean. + // L4J factory + provider beans for `anthropic` were dropped when those providers landed native in + // ADR-005 Phase B/C — the `openai` discriminator is still served by the LangChain4j bridge via + // `OpenAiDispatchingChatModelProvider` (which internally routes OPENAI/FOUNDRY/CUSTOM backends), + // unless overridden by the native SDK factory. private static final List> LANGCHAIN4J_BEANS = List.of( ChatModelHttpProxySupport.class, - AzureOpenAiChatModelProvider.class, + OpenAiDispatchingChatModelProvider.class, BedrockChatModelProvider.class, GoogleVertexAiChatModelProvider.class, ChatModelProviderRegistry.class, @@ -301,14 +302,15 @@ void userProvidedProviderBeanOverridesDefault(ProviderOverrideCase override) { static Stream providerOverrideCases() { // Only providers still backed by the L4J bridge declare a `ChatModelProvider` bean and - // therefore support overriding it. `anthropic`, `openai`, and `openaiCompatible` have native + // therefore support overriding it. `anthropic`, `openai` (OPENAI/CUSTOM backends) have native // factories now and no L4J `ChatModelProvider` to replace. + // The FOUNDRY backend of `openai` still goes through the L4J bridge (AzureOpenAiChatModel). return Stream.of( new ProviderOverrideCase( - CustomAzureOpenAiProviderConfig.class, - "customAzureOpenAiChatModelProvider", - AzureOpenAiProviderConfiguration.class, - CustomAzureOpenAiChatModelProvider.class), + CustomFoundryProviderConfig.class, + "customFoundryChatModelProvider", + OpenAiProviderConfiguration.class, + CustomFoundryChatModelProvider.class), new ProviderOverrideCase( CustomBedrockProviderConfig.class, "customBedrockChatModelProvider", @@ -333,22 +335,22 @@ public String toString() { } } - static class CustomAzureOpenAiProviderConfig { + static class CustomFoundryProviderConfig { @Bean - ChatModelProvider customAzureOpenAiChatModelProvider() { - return new CustomAzureOpenAiChatModelProvider(); + ChatModelProvider customFoundryChatModelProvider() { + return new CustomFoundryChatModelProvider(); } - static class CustomAzureOpenAiChatModelProvider - implements ChatModelProvider { + static class CustomFoundryChatModelProvider + implements ChatModelProvider { @Override public String type() { - return AzureOpenAiProviderConfiguration.AZURE_OPENAI_ID; + return OpenAiProviderConfiguration.OPENAI_ID; } @Override - public ChatModel createChatModel(AzureOpenAiProviderConfiguration providerConfiguration) { + public ChatModel createChatModel(OpenAiProviderConfiguration providerConfiguration) { return mock(ChatModel.class); } } From ba4fb32e4715cf5b5144ac5cd08d6687e92fd18e Mon Sep 17 00:00:00 2001 From: Dmitri Nikonov Date: Fri, 8 May 2026 11:03:00 +0200 Subject: [PATCH 74/81] refactor(agentic-ai): rename stale Azure bean name to langchain4JOpenAiChatModelProvider --- .../AgenticAiLangchain4JChatModelConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JChatModelConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JChatModelConfiguration.java index aac815a4f8a..8b7bbadccba 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JChatModelConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JChatModelConfiguration.java @@ -41,7 +41,7 @@ public ChatModelHttpProxySupport langchain4JChatModelHttpProxySupport( @Bean @ConditionalOnMissingBean - public ChatModelProvider langchain4JAzureOpenAiChatModelProvider( + public ChatModelProvider langchain4JOpenAiChatModelProvider( AgenticAiConnectorsConfigurationProperties config, ChatModelHttpProxySupport chatModelHttpProxySupport) { final var chatModelProperties = config.aiagent().chatModel(); From de0b0b30bea65100028ba3ee4e4a5bc9c7f900e8 Mon Sep 17 00:00:00 2001 From: Dmitri Nikonov Date: Fri, 8 May 2026 11:06:36 +0200 Subject: [PATCH 75/81] refactor(agentic-ai): update OpenAiChatModelApiFactory for backend branches + remove compat factory (Phase F) Split buildClient() into three backend-specific methods (OPENAI, FOUNDRY, CUSTOM) and remove the OpenAiCompatibleChatModelApiFactory bean from OpenAiChatModelApiConfiguration now that all backends are handled by the single openai discriminator. --- .../OpenAiChatModelApiConfiguration.java | 23 +----- .../openai/OpenAiChatModelApiFactory.java | 81 ++++++++++++++++--- 2 files changed, 75 insertions(+), 29 deletions(-) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiConfiguration.java index bcfd1e49fbe..a29fad867f5 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiConfiguration.java @@ -10,7 +10,6 @@ import com.openai.client.OpenAIClient; import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; import io.camunda.connector.agenticai.aiagent.framework.capabilities.ModelCapabilitiesResolver; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsConfigurationProperties; import io.camunda.connector.runtime.annotation.ConnectorsObjectMapper; @@ -20,11 +19,10 @@ import org.springframework.context.annotation.Configuration; /** - * Registers the native OpenAI factories under the same Spring bean names as the LangChain4j bridge - * factories ({@code langchain4JOpenAiChatModelApiFactory} and {@code - * langchain4JOpenAiCompatibleChatModelApiFactory}). The bridge configuration uses - * {@code @ConditionalOnMissingBean(name = ...)}, so these native beans take over whenever the - * OpenAI SDK is on the classpath. Azure OpenAI stays on the bridge for now (Phase G). + * Registers the native OpenAI factory under the same Spring bean name as the LangChain4j bridge + * factory ({@code langchain4JOpenAiChatModelApiFactory}). The bridge configuration uses + * {@code @ConditionalOnMissingBean(name = ...)}, so this native bean takes over whenever the OpenAI + * SDK is on the classpath. */ @Configuration @ConditionalOnClass(OpenAIClient.class) @@ -41,17 +39,4 @@ public ChatModelApiFactory openAiChatModelApiFactor capabilitiesResolver, properties.aiagent().chatModel().api().defaultTimeout()); } - - @Bean(name = "langchain4JOpenAiCompatibleChatModelApiFactory") - @ConditionalOnMissingBean(name = "langchain4JOpenAiCompatibleChatModelApiFactory") - public ChatModelApiFactory - openAiCompatibleChatModelApiFactory( - @ConnectorsObjectMapper ObjectMapper objectMapper, - ModelCapabilitiesResolver capabilitiesResolver, - AgenticAiConnectorsConfigurationProperties properties) { - return new OpenAiCompatibleChatModelApiFactory( - objectMapper, - capabilitiesResolver, - properties.aiagent().chatModel().api().defaultTimeout()); - } } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java index 181be912e94..25ec0db941f 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiChatModelApiFactory.java @@ -14,6 +14,8 @@ import io.camunda.connector.agenticai.aiagent.framework.capabilities.ModelCapabilitiesResolver; import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.ApiFamily; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiAuthentication.OpenAiApiKeyAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiAuthentication.OpenAiClientCredentialsAuthentication; import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiConnection; import java.time.Duration; import java.util.Optional; @@ -22,12 +24,11 @@ /** * Native OpenAI factory for the {@code openai} discriminator. Builds an {@link OpenAIClient} - * (OkHttp transport) from {@link OpenAiProviderConfiguration} and instantiates an {@link - * OpenAiChatCompletionsChatModelApi}. + * (OkHttp transport) from {@link OpenAiProviderConfiguration} and dispatches to the appropriate + * backend-specific client builder ({@code OPENAI}, {@code FOUNDRY}, or {@code CUSTOM}). * - *

        Phase D will add the {@code apiFamily} branch: when the config selects {@code apiFamily = - * RESPONSES}, the factory will build {@code OpenAiResponsesChatModelApi} instead, sharing this - * client construction. + *

        The {@code create()} method branches on {@code apiFamily} to pick {@link + * OpenAiResponsesChatModelApi} vs {@link OpenAiChatCompletionsChatModelApi}. */ public class OpenAiChatModelApiFactory implements ChatModelApiFactory { @@ -102,17 +103,77 @@ public ChatModelApi create(OpenAiProviderConfiguration configuration) { } private OpenAIClient buildClient(OpenAiConnection connection) { + return switch (connection.backend()) { + case OPENAI -> buildOpenAiClient(connection); + case FOUNDRY -> buildFoundryClient(connection); + case CUSTOM -> buildCustomClient(connection); + }; + } + + private OpenAIClient buildOpenAiClient(OpenAiConnection connection) { final var builder = OpenAIOkHttpClient.builder(); - builder.apiKey(connection.authentication().apiKey()); + + if (connection.authentication() instanceof OpenAiApiKeyAuthentication apiKeyAuth) { + builder.apiKey(apiKeyAuth.apiKey()); + if (StringUtils.isNotBlank(apiKeyAuth.organizationId())) { + builder.organization(apiKeyAuth.organizationId()); + } + if (StringUtils.isNotBlank(apiKeyAuth.projectId())) { + builder.project(apiKeyAuth.projectId()); + } + } else { + builder.apiKey("no-key"); + } if (StringUtils.isNotBlank(connection.endpoint())) { builder.baseUrl(connection.endpoint()); } - if (StringUtils.isNotBlank(connection.authentication().organizationId())) { - builder.organization(connection.authentication().organizationId()); + + final var timeout = resolveTimeout(connection); + if (timeout != null) { + builder.timeout(timeout); + } + + return builder.build(); + } + + private OpenAIClient buildFoundryClient(OpenAiConnection connection) { + if (connection.authentication() instanceof OpenAiClientCredentialsAuthentication) { + throw new UnsupportedOperationException( + "Client credentials for FOUNDRY backend requires Phase G Azure SDK integration"); + } + + final var builder = OpenAIOkHttpClient.builder().baseUrl(connection.endpoint()); + + if (connection.authentication() instanceof OpenAiApiKeyAuthentication apiKeyAuth) { + builder.apiKey(apiKeyAuth.apiKey()); + } else { + builder.apiKey("no-key"); + } + + final var timeout = resolveTimeout(connection); + if (timeout != null) { + builder.timeout(timeout); + } + + return builder.build(); + } + + private OpenAIClient buildCustomClient(OpenAiConnection connection) { + final var builder = OpenAIOkHttpClient.builder().baseUrl(connection.endpoint()); + + final var apiKey = + connection.authentication() instanceof OpenAiApiKeyAuthentication apiKeyAuth + && StringUtils.isNotBlank(apiKeyAuth.apiKey()) + ? apiKeyAuth.apiKey() + : "no-key"; + builder.apiKey(apiKey); + + if (connection.headers() != null && !connection.headers().isEmpty()) { + connection.headers().forEach(builder::putHeader); } - if (StringUtils.isNotBlank(connection.authentication().projectId())) { - builder.project(connection.authentication().projectId()); + if (connection.queryParameters() != null && !connection.queryParameters().isEmpty()) { + connection.queryParameters().forEach(builder::putQueryParam); } final var timeout = resolveTimeout(connection); From fd675ede9576498c439c9d2f333e35a98ef78e5f Mon Sep 17 00:00:00 2001 From: Dmitri Nikonov Date: Fri, 8 May 2026 11:10:34 +0200 Subject: [PATCH 76/81] refactor(agentic-ai): require endpoint for FOUNDRY and CUSTOM OpenAI backends (Phase F) --- .../provider/OpenAiProviderConfiguration.java | 198 +++++++++++++--- .../request/ProviderConfigurationTest.java | 215 +++++++++++++----- 2 files changed, 321 insertions(+), 92 deletions(-) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiProviderConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiProviderConfiguration.java index 13e8c2b2ae8..27d1e7c3308 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiProviderConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiProviderConfiguration.java @@ -8,15 +8,22 @@ import static io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OPENAI_ID; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.camunda.connector.agenticai.aiagent.model.request.provider.shared.HttpUrl; import io.camunda.connector.agenticai.aiagent.model.request.provider.shared.TimeoutConfiguration; +import io.camunda.connector.api.annotation.FEEL; import io.camunda.connector.generator.java.annotation.FeelMode; +import io.camunda.connector.generator.java.annotation.TemplateDiscriminatorProperty; import io.camunda.connector.generator.java.annotation.TemplateProperty; import io.camunda.connector.generator.java.annotation.TemplateSubType; import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertFalse; import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; +import java.util.Map; @TemplateSubType(id = OPENAI_ID, label = "OpenAI") public record OpenAiProviderConfiguration(@Valid @NotNull OpenAiConnection openai) @@ -30,7 +37,19 @@ public String providerType() { return OPENAI_ID; } + public enum OpenAiBackend { + @JsonProperty("openai") + OPENAI, + + @JsonProperty("foundry") + FOUNDRY, + + @JsonProperty("custom") + CUSTOM + } + public record OpenAiConnection( + @TemplateProperty(ignore = true) OpenAiBackend backend, @Valid @NotNull OpenAiAuthentication authentication, @Valid TimeoutConfiguration timeouts, @Valid @NotNull OpenAiModel model, @@ -54,24 +73,57 @@ public record OpenAiConnection( @HttpUrl @TemplateProperty( group = "provider", - label = "Custom API endpoint", + label = "Endpoint", description = "Optional. Override the default OpenAI base URL (e.g. for an OpenAI proxy or gateway). Leave blank to use the SDK default.", type = TemplateProperty.PropertyType.String, feel = FeelMode.optional, optional = true) - String endpoint) { + String endpoint, + @FEEL + @TemplateProperty( + group = "provider", + label = "Headers", + description = "Map of HTTP headers to add to the request.", + feel = FeelMode.required, + optional = true) + Map headers, + @FEEL + @TemplateProperty( + group = "provider", + label = "Query Parameters", + description = "Map of query parameters to add to the request URL.", + feel = FeelMode.required, + optional = true) + @Valid + Map<@NotBlank String, String> queryParameters) { public OpenAiConnection { + if (backend == null) { + backend = OpenAiBackend.OPENAI; + } if (apiFamily == null) { apiFamily = ApiFamily.COMPLETIONS; } } - /** Convenience constructor used by existing call sites that pre-date the apiFamily field. */ + /** Convenience constructor used by existing call sites that pre-date the backend field. */ public OpenAiConnection( OpenAiAuthentication authentication, TimeoutConfiguration timeouts, OpenAiModel model) { - this(authentication, timeouts, model, ApiFamily.COMPLETIONS, null); + this(null, authentication, timeouts, model, ApiFamily.COMPLETIONS, null, null, null); + } + + @AssertFalse( + message = "Client credentials authentication is only supported for the FOUNDRY backend") + public boolean isClientCredentialsUsedWithNonFoundryBackend() { + return authentication instanceof OpenAiAuthentication.OpenAiClientCredentialsAuthentication + && backend != OpenAiBackend.FOUNDRY; + } + + @AssertFalse(message = "Endpoint is required for FOUNDRY and CUSTOM backends") + public boolean isEndpointMissingForBackendThatRequiresIt() { + return (backend == OpenAiBackend.FOUNDRY || backend == OpenAiBackend.CUSTOM) + && (endpoint == null || endpoint.isBlank()); } } @@ -82,38 +134,104 @@ public enum ApiFamily { RESPONSES } - public record OpenAiAuthentication( - @NotBlank - @TemplateProperty( - group = "provider", - label = "OpenAI API key", - type = TemplateProperty.PropertyType.String, - feel = FeelMode.optional, - constraints = @TemplateProperty.PropertyConstraints(notEmpty = true)) - String apiKey, - @TemplateProperty( - group = "provider", - label = "Organization ID", - description = - "For members of multiple organizations. Details in the documentation.", - type = TemplateProperty.PropertyType.String, - feel = FeelMode.optional, - optional = true) - String organizationId, - @TemplateProperty( - group = "provider", - label = "Project ID", - description = - "For accounts with multiple projects. Details in the documentation.", - type = TemplateProperty.PropertyType.String, - feel = FeelMode.optional, - optional = true) - String projectId) { + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") + @JsonSubTypes({ + @JsonSubTypes.Type( + value = OpenAiAuthentication.OpenAiApiKeyAuthentication.class, + name = "apiKey"), + @JsonSubTypes.Type( + value = OpenAiAuthentication.OpenAiClientCredentialsAuthentication.class, + name = "clientCredentials") + }) + @TemplateDiscriminatorProperty( + label = "Authentication", + group = "provider", + name = "type", + defaultValue = "apiKey", + description = "Specify the OpenAI authentication strategy.") + public sealed interface OpenAiAuthentication { + + @TemplateSubType(id = "apiKey", label = "API key") + record OpenAiApiKeyAuthentication( + @TemplateProperty( + group = "provider", + label = "API key", + type = TemplateProperty.PropertyType.String, + feel = FeelMode.optional, + optional = true) + String apiKey, + @TemplateProperty( + group = "provider", + label = "Organization ID", + description = + "For members of multiple organizations. Details in the documentation.", + type = TemplateProperty.PropertyType.String, + feel = FeelMode.optional, + optional = true) + String organizationId, + @TemplateProperty( + group = "provider", + label = "Project ID", + description = + "For accounts with multiple projects. Details in the documentation.", + type = TemplateProperty.PropertyType.String, + feel = FeelMode.optional, + optional = true) + String projectId) + implements OpenAiAuthentication { - @Override - public String toString() { - return "OpenAiAuthentication{apiKey=[REDACTED], organizationId=%s, projectId=%s}" - .formatted(organizationId, projectId); + @Override + public String toString() { + return "OpenAiApiKeyAuthentication{apiKey=[REDACTED], organizationId=%s, projectId=%s}" + .formatted(organizationId, projectId); + } + } + + @TemplateSubType(id = "clientCredentials", label = "Client credentials") + record OpenAiClientCredentialsAuthentication( + @NotBlank + @TemplateProperty( + group = "provider", + label = "Client ID", + description = "ID of a Microsoft Entra application", + type = TemplateProperty.PropertyType.String, + feel = FeelMode.optional, + constraints = @TemplateProperty.PropertyConstraints(notEmpty = true)) + String clientId, + @NotBlank + @TemplateProperty( + group = "provider", + label = "Client secret", + description = "Secret of a Microsoft Entra application", + type = TemplateProperty.PropertyType.String, + feel = FeelMode.optional, + constraints = @TemplateProperty.PropertyConstraints(notEmpty = true)) + String clientSecret, + @NotBlank + @TemplateProperty( + group = "provider", + label = "Tenant ID", + description = + "ID of a Microsoft Entra tenant. Details in the documentation.", + type = TemplateProperty.PropertyType.String, + feel = FeelMode.optional) + String tenantId, + @TemplateProperty( + group = "provider", + label = "Authority host", + description = + "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", + type = TemplateProperty.PropertyType.String, + feel = FeelMode.optional, + optional = true) + String authorityHost) + implements OpenAiAuthentication { + + @Override + public String toString() { + return "OpenAiClientCredentialsAuthentication{clientId=%s, clientSecret=[REDACTED], tenantId=%s, authorityHost=%s}" + .formatted(clientId, tenantId, authorityHost); + } } } @@ -162,6 +280,14 @@ public record OpenAiModelParameters( type = TemplateProperty.PropertyType.Number, feel = FeelMode.required, optional = true) - Double topP) {} + Double topP, + @FEEL + @TemplateProperty( + group = "model", + label = "Custom parameters", + description = "Map of additional request parameters to include.", + feel = FeelMode.required, + optional = true) + Map customParameters) {} } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java index 90024ab693d..80d783875b0 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationTest.java @@ -15,9 +15,6 @@ import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicBackend; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicConnection; import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicModel; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration.AzureAuthentication; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration.AzureOpenAiConnection; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration.AzureOpenAiModel; import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration.AwsAuthentication; import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration.BedrockConnection; @@ -26,9 +23,11 @@ import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GoogleGenAiConnection; import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GoogleGenAiModel; import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GoogleGenAiModel.GoogleGenAiModelParameters; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration.OpenAiCompatibleAuthentication; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration.OpenAiCompatibleConnection; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration.OpenAiCompatibleModel; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiAuthentication.OpenAiApiKeyAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiAuthentication.OpenAiClientCredentialsAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiBackend; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiConnection; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiModel; import io.camunda.connector.agenticai.aiagent.model.request.provider.shared.TimeoutConfiguration; import io.camunda.connector.agenticai.util.ConnectorUtils; import jakarta.validation.ConstraintViolation; @@ -42,7 +41,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.NullAndEmptySource; import org.junit.jupiter.params.provider.NullSource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.validation.autoconfigure.ValidationAutoConfiguration; @@ -261,100 +259,205 @@ void validationShouldFail_WhenClientCredentialsAuthUsedWithNonFoundryBackend( } @Nested - class AzureOpenAiConnectionTest { + class OpenAiConnectionTest { - @ParameterizedTest - @MethodSource( - "io.camunda.connector.agenticai.aiagent.model.request.ProviderConfigurationTest#validHttpUrls") - void shouldAcceptValidEndpoint(String endpoint) { + @Test + void validationShouldSucceed_WhenOpenAIBackendWithApiKeyAuth() { var connection = - new AzureOpenAiConnection( - endpoint, - new AzureAuthentication.AzureApiKeyAuthentication("key"), + new OpenAiConnection( + OpenAiBackend.OPENAI, + new OpenAiApiKeyAuthentication("my-api-key", null, null), TIMEOUT, - new AzureOpenAiModel("deployment", null)); + new OpenAiModel("gpt-4o", null), + null, + null, + null, + null); assertThat(validator.validate(connection)).isEmpty(); } - @ParameterizedTest - @MethodSource( - "io.camunda.connector.agenticai.aiagent.model.request.ProviderConfigurationTest#invalidHttpUrls") - void shouldRejectInvalidUrlEndpoint(String endpoint) { + @Test + void validationShouldSucceed_WhenFoundryBackendWithClientCredentialsAuth() { var connection = - new AzureOpenAiConnection( - endpoint, - new AzureAuthentication.AzureApiKeyAuthentication("key"), + new OpenAiConnection( + OpenAiBackend.FOUNDRY, + new OpenAiClientCredentialsAuthentication( + "client-id", "client-secret", "tenant-id", null), TIMEOUT, - new AzureOpenAiModel("deployment", null)); - assertThat(validator.validate(connection)) - .extracting(ConstraintViolation::getMessage) - .contains(HTTP_URL_VALIDATION_MESSAGE); + new OpenAiModel("gpt-4o", null), + null, + "https://my-foundry-endpoint.azure.com", + null, + null); + assertThat(validator.validate(connection)).isEmpty(); + } + + @Test + void validationShouldSucceed_WhenFoundryBackendWithApiKeyAuth() { + var connection = + new OpenAiConnection( + OpenAiBackend.FOUNDRY, + new OpenAiApiKeyAuthentication("my-api-key", null, null), + TIMEOUT, + new OpenAiModel("gpt-4o", null), + null, + "https://my-foundry-endpoint.azure.com", + null, + null); + assertThat(validator.validate(connection)).isEmpty(); + } + + @Test + void validationShouldSucceed_WhenCustomBackendWithApiKeyAuth() { + var connection = + new OpenAiConnection( + OpenAiBackend.CUSTOM, + new OpenAiApiKeyAuthentication(null, null, null), + TIMEOUT, + new OpenAiModel("some-model", null), + null, + "https://custom-endpoint.local/v1", + null, + null); + // apiKey may be null for CUSTOM + assertThat(validator.validate(connection)).isEmpty(); } @ParameterizedTest - @NullAndEmptySource - void shouldRejectBlankEndpoint(String endpoint) { + @EnumSource( + value = OpenAiBackend.class, + names = {"OPENAI", "CUSTOM"}) + void validationShouldFail_WhenClientCredentialsAuthUsedWithNonFoundryBackend( + OpenAiBackend backend) { var connection = - new AzureOpenAiConnection( - endpoint, - new AzureAuthentication.AzureApiKeyAuthentication("key"), + new OpenAiConnection( + backend, + new OpenAiClientCredentialsAuthentication( + "client-id", "client-secret", "tenant-id", null), TIMEOUT, - new AzureOpenAiModel("deployment", null)); + new OpenAiModel("gpt-4o", null), + null, + "https://some-endpoint.local", + null, + null); assertThat(validator.validate(connection)) + .hasSize(1) .extracting(ConstraintViolation::getMessage) - .contains("must not be blank"); + .containsExactly( + "Client credentials authentication is only supported for the FOUNDRY backend"); } - } - - @Nested - class OpenAiCompatibleConnectionTest { @ParameterizedTest @MethodSource( "io.camunda.connector.agenticai.aiagent.model.request.ProviderConfigurationTest#validHttpUrls") + @NullSource void shouldAcceptValidEndpoint(String endpoint) { var connection = - new OpenAiCompatibleConnection( - endpoint, - new OpenAiCompatibleAuthentication("key"), + new OpenAiConnection( + OpenAiBackend.OPENAI, + new OpenAiApiKeyAuthentication("key", null, null), + TIMEOUT, + new OpenAiModel("gpt-4o", null), null, + endpoint, null, - TIMEOUT, - new OpenAiCompatibleModel("model", null)); + null); assertThat(validator.validate(connection)).isEmpty(); } @ParameterizedTest @MethodSource( "io.camunda.connector.agenticai.aiagent.model.request.ProviderConfigurationTest#invalidHttpUrls") - void shouldRejectInvalidUrlEndpoint(String endpoint) { + void shouldRejectInvalidEndpoint(String endpoint) { var connection = - new OpenAiCompatibleConnection( - endpoint, - new OpenAiCompatibleAuthentication("key"), + new OpenAiConnection( + OpenAiBackend.OPENAI, + new OpenAiApiKeyAuthentication("key", null, null), + TIMEOUT, + new OpenAiModel("gpt-4o", null), null, + endpoint, null, - TIMEOUT, - new OpenAiCompatibleModel("model", null)); + null); assertThat(validator.validate(connection)) .extracting(ConstraintViolation::getMessage) .contains(HTTP_URL_VALIDATION_MESSAGE); } + @Test + void validationShouldSucceed_WhenNullBackendDefaultsToOpenAI() { + // Compact constructor sets null backend → OPENAI + var connection = + new OpenAiConnection( + new OpenAiApiKeyAuthentication("my-api-key", null, null), + TIMEOUT, + new OpenAiModel("gpt-4o", null)); + assertThat(connection.backend()).isEqualTo(OpenAiBackend.OPENAI); + assertThat(validator.validate(connection)).isEmpty(); + } + @ParameterizedTest - @NullAndEmptySource - void shouldRejectBlankEndpoint(String endpoint) { + @EnumSource( + value = OpenAiBackend.class, + names = {"FOUNDRY", "CUSTOM"}) + void validationShouldFail_WhenEndpointMissingForBackendThatRequiresIt(OpenAiBackend backend) { var connection = - new OpenAiCompatibleConnection( - endpoint, - new OpenAiCompatibleAuthentication("key"), + new OpenAiConnection( + backend, + new OpenAiApiKeyAuthentication("my-api-key", null, null), + TIMEOUT, + new OpenAiModel("gpt-4o", null), + null, null, null, + null); + assertThat(validator.validate(connection)) + .hasSize(1) + .extracting(ConstraintViolation::getMessage) + .containsExactly("Endpoint is required for FOUNDRY and CUSTOM backends"); + } + + @ParameterizedTest + @EnumSource( + value = OpenAiBackend.class, + names = {"FOUNDRY", "CUSTOM"}) + void validationShouldFail_WhenEndpointBlankForBackendThatRequiresIt(OpenAiBackend backend) { + var connection = + new OpenAiConnection( + backend, + new OpenAiApiKeyAuthentication("my-api-key", null, null), TIMEOUT, - new OpenAiCompatibleModel("model", null)); + new OpenAiModel("gpt-4o", null), + null, + " ", + null, + null); + // Both @HttpUrl and @AssertFalse validations trigger for blank endpoint assertThat(validator.validate(connection)) + .hasSize(2) .extracting(ConstraintViolation::getMessage) - .contains("must not be blank"); + .containsExactlyInAnyOrder( + "Must be an HTTP or HTTPS URL", + "Endpoint is required for FOUNDRY and CUSTOM backends"); + } + + @ParameterizedTest + @EnumSource( + value = OpenAiBackend.class, + names = {"FOUNDRY", "CUSTOM"}) + void validationShouldSucceed_WhenEndpointProvidedForBackendThatRequiresIt( + OpenAiBackend backend) { + var connection = + new OpenAiConnection( + backend, + new OpenAiApiKeyAuthentication("my-api-key", null, null), + TIMEOUT, + new OpenAiModel("gpt-4o", null), + null, + "https://my-endpoint.local/v1", + null, + null); + assertThat(validator.validate(connection)).isEmpty(); } } From f4b755bd223a2dd391c3cc15a78562cdaef558d6 Mon Sep 17 00:00:00 2001 From: Dmitri Nikonov Date: Fri, 8 May 2026 11:33:47 +0200 Subject: [PATCH 77/81] feat(agentic-ai): add ProviderConfigurationDeserializer for backward-compatible migration (Phase F) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces ProviderConfigurationDeserializer, a permanent StdDeserializer that rewrites legacy ProviderConfiguration JSON shapes to the current canonical form before Jackson's polymorphic dispatch resolves the concrete subtype. Covers all 8 migration rules from ADR 005: bedrock+Anthropic model → anthropic/bedrock, googleVertexAi → googleGenAi/vertex, azureOpenAi → openai/foundry, openaiCompatible → openai/custom, and missing-field defaults for anthropic (backend, auth type discriminator) and openai (backend, apiFamily). Wired via @JsonDeserialize(using = ProviderConfigurationDeserializer.class) on the interface. Test coverage includes every ADR migration row, idempotence round-trips, and ConnectorsObjectMapper no-collision checks. --- .../provider/ProviderConfiguration.java | 16 +- .../ProviderConfigurationDeserializer.java | 452 +++++++++++++++ ...ProviderConfigurationDeserializerTest.java | 533 ++++++++++++++++++ 3 files changed, 992 insertions(+), 9 deletions(-) create mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/ProviderConfigurationDeserializer.java create mode 100644 connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationDeserializerTest.java diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/ProviderConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/ProviderConfiguration.java index c3b5c5d279f..1f7ce47840a 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/ProviderConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/ProviderConfiguration.java @@ -7,26 +7,26 @@ package io.camunda.connector.agenticai.aiagent.model.request.provider; import static io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.ANTHROPIC_ID; -import static io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration.AZURE_OPENAI_ID; import static io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration.BEDROCK_ID; import static io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GOOGLE_GENAI_ID; -import static io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration.OPENAI_COMPATIBLE_ID; import static io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OPENAI_ID; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import io.camunda.connector.generator.java.annotation.TemplateDiscriminatorProperty; +@JsonDeserialize(using = ProviderConfigurationDeserializer.class) @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes({ @JsonSubTypes.Type(value = AnthropicProviderConfiguration.class, name = ANTHROPIC_ID), @JsonSubTypes.Type(value = BedrockProviderConfiguration.class, name = BEDROCK_ID), - @JsonSubTypes.Type(value = AzureOpenAiProviderConfiguration.class, name = AZURE_OPENAI_ID), @JsonSubTypes.Type(value = GoogleGenAiProviderConfiguration.class, name = GOOGLE_GENAI_ID), @JsonSubTypes.Type(value = OpenAiProviderConfiguration.class, name = OPENAI_ID), - @JsonSubTypes.Type( - value = OpenAiCompatibleProviderConfiguration.class, - name = OPENAI_COMPATIBLE_ID) + // Legacy type IDs — routed through ProviderConfigurationDeserializer for migration + @JsonSubTypes.Type(value = GoogleGenAiProviderConfiguration.class, name = "googleVertexAi"), + @JsonSubTypes.Type(value = OpenAiProviderConfiguration.class, name = "azureOpenAi"), + @JsonSubTypes.Type(value = OpenAiProviderConfiguration.class, name = "openaiCompatible"), }) @TemplateDiscriminatorProperty( label = "Provider", @@ -37,10 +37,8 @@ public sealed interface ProviderConfiguration permits AnthropicProviderConfiguration, BedrockProviderConfiguration, - AzureOpenAiProviderConfiguration, GoogleGenAiProviderConfiguration, - OpenAiProviderConfiguration, - OpenAiCompatibleProviderConfiguration { + OpenAiProviderConfiguration { /** Type of the provider implementation used to resolve the backing chat model. */ String providerType(); diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/ProviderConfigurationDeserializer.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/ProviderConfigurationDeserializer.java new file mode 100644 index 00000000000..207030e16d6 --- /dev/null +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/ProviderConfigurationDeserializer.java @@ -0,0 +1,452 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.model.request.provider; + +import static io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.ANTHROPIC_ID; +import static io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration.BEDROCK_ID; +import static io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GOOGLE_GENAI_ID; +import static io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OPENAI_ID; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.IOException; + +/** + * Migration deserializer for {@link ProviderConfiguration}. + * + *

        Rewrites legacy serialized provider configuration shapes to the current canonical form before + * standard Jackson polymorphic dispatch takes over. This handles backward-compatibility for process + * instances that were saved with older configuration shapes, including: + * + *

          + *
        • Bedrock configurations with Anthropic model IDs → AnthropicProviderConfiguration with + * backend=bedrock + *
        • googleVertexAi discriminator → googleGenAi with backend=vertex + *
        • anthropic without backend field → inject backend=direct + *
        • anthropic without auth type discriminator → inject type=apiKey + *
        • openai without backend/apiFamily → inject backend=openai, apiFamily=completions + *
        • azureOpenAi → openai/foundry with field mapping + *
        • openaiCompatible → openai/custom with field mapping + *
        + * + *

        Dispatch to concrete subtypes via {@code mapper.treeToValue(migrated, SubType.class)} does NOT + * re-enter this deserializer because Jackson's BeanDeserializer for the resolved concrete subtype + * takes over directly. + * + *

        This deserializer is permanent infrastructure — kept indefinitely so that stale process + * variables remain readable. + */ +public class ProviderConfigurationDeserializer extends StdDeserializer { + + private static final String GOOGLE_VERTEX_AI_LEGACY_ID = "googleVertexAi"; + private static final String AZURE_OPENAI_LEGACY_ID = "azureOpenAi"; + private static final String OPENAI_COMPATIBLE_LEGACY_ID = "openaiCompatible"; + + private static final String FIELD_TYPE = "type"; + private static final String FIELD_BACKEND = "backend"; + private static final String FIELD_API_FAMILY = "apiFamily"; + private static final String FIELD_ENDPOINT = "endpoint"; + private static final String FIELD_AUTHENTICATION = "authentication"; + private static final String FIELD_MODEL = "model"; + private static final String FIELD_TIMEOUTS = "timeouts"; + private static final String FIELD_HEADERS = "headers"; + private static final String FIELD_QUERY_PARAMETERS = "queryParameters"; + private static final String FIELD_API_KEY = "apiKey"; + private static final String FIELD_DEPLOYMENT_NAME = "deploymentName"; + + public ProviderConfigurationDeserializer() { + super(ProviderConfiguration.class); + } + + @Override + public ProviderConfiguration deserialize(JsonParser jp, DeserializationContext ctxt) + throws IOException { + ObjectMapper mapper = (ObjectMapper) jp.getCodec(); + ObjectNode node = (ObjectNode) mapper.readTree(jp); + + // The `type` property may be absent from the node when Jackson's AsPropertyTypeDeserializer + // has already consumed it before calling this deserializer. In that case, infer the original + // type discriminator from the node's top-level keys (e.g. "azureOpenAi", "anthropic"). + String type = node.has(FIELD_TYPE) ? node.path(FIELD_TYPE).asText(null) : inferType(node); + + ObjectNode migrated = migrate(node, type, mapper); + String newType = + migrated.has(FIELD_TYPE) ? migrated.path(FIELD_TYPE).asText(null) : inferType(migrated); + + // Dispatch by extracting the inner connection sub-object and deserializing it directly as the + // concrete connection type (e.g. AnthropicConnection, not AnthropicProviderConfiguration). + // This avoids routing through ObjectMapper.treeToValue(ProviderConfiguration subtype), which + // would re-trigger the interface-level @JsonTypeInfo + @JsonDeserialize and cause recursion. + ObjectNode innerNode = + newType != null ? (ObjectNode) migrated.path(newType) : mapper.createObjectNode(); + + if (ANTHROPIC_ID.equals(newType)) { + return new AnthropicProviderConfiguration( + mapper.treeToValue(innerNode, AnthropicProviderConfiguration.AnthropicConnection.class)); + } else if (BEDROCK_ID.equals(newType)) { + return new BedrockProviderConfiguration( + mapper.treeToValue(innerNode, BedrockProviderConfiguration.BedrockConnection.class)); + } else if (GOOGLE_GENAI_ID.equals(newType)) { + return new GoogleGenAiProviderConfiguration( + mapper.treeToValue( + innerNode, GoogleGenAiProviderConfiguration.GoogleGenAiConnection.class)); + } else if (OPENAI_ID.equals(newType)) { + return new OpenAiProviderConfiguration( + mapper.treeToValue(innerNode, OpenAiProviderConfiguration.OpenAiConnection.class)); + } else { + throw new JsonMappingException(jp, "Unknown provider type: " + newType); + } + } + + /** + * Infers the provider type discriminator from the node's top-level keys when the {@code type} + * property has been consumed by Jackson's polymorphic type resolver before reaching this + * deserializer. + */ + private String inferType(ObjectNode node) { + if (node.has(ANTHROPIC_ID)) return ANTHROPIC_ID; + if (node.has(BEDROCK_ID)) return BEDROCK_ID; + if (node.has(GOOGLE_GENAI_ID)) return GOOGLE_GENAI_ID; + if (node.has(OPENAI_ID)) return OPENAI_ID; + if (node.has(GOOGLE_VERTEX_AI_LEGACY_ID)) return GOOGLE_VERTEX_AI_LEGACY_ID; + if (node.has(AZURE_OPENAI_LEGACY_ID)) return AZURE_OPENAI_LEGACY_ID; + if (node.has(OPENAI_COMPATIBLE_LEGACY_ID)) return OPENAI_COMPATIBLE_LEGACY_ID; + return null; + } + + /** + * Applies all migration rules in order and returns the (possibly rewritten) node. + * + *

        Rules are applied in order; some rules may change the type discriminator, which causes + * subsequent rules to operate on the updated type. + */ + private ObjectNode migrate(ObjectNode node, String type, ObjectMapper mapper) { + // Rule 1 & 2: bedrock with Anthropic model → rewrite to anthropic/bedrock; non-Anthropic + // model → passthrough + if (BEDROCK_ID.equals(type)) { + return migrateBedrockNode(node, mapper); + } + + // Rule 3: googleVertexAi → googleGenAi/vertex + if (GOOGLE_VERTEX_AI_LEGACY_ID.equals(type)) { + return migrateGoogleVertexAiNode(node, mapper); + } + + // Rule 7: azureOpenAi → openai/foundry + if (AZURE_OPENAI_LEGACY_ID.equals(type)) { + return migrateAzureOpenAiNode(node, mapper); + } + + // Rule 8: openaiCompatible → openai/custom + if (OPENAI_COMPATIBLE_LEGACY_ID.equals(type)) { + return migrateOpenAiCompatibleNode(node, mapper); + } + + // Rules 4 & 5: anthropic without backend or without auth type discriminator + if (ANTHROPIC_ID.equals(type)) { + return migrateAnthropicNode(node, mapper); + } + + // Rule 6: openai without backend/apiFamily + if (OPENAI_ID.equals(type)) { + return migrateOpenAiNode(node, mapper); + } + + return node; + } + + /** + * Rule 1: bedrock with an {@code anthropic.*} model ID → rewrite to anthropic/bedrock. Rule 2: + * bedrock with non-Anthropic model → passthrough. + */ + private ObjectNode migrateBedrockNode(ObjectNode node, ObjectMapper mapper) { + ObjectNode bedrockObj = (ObjectNode) node.path(BEDROCK_ID); + if (bedrockObj == null || bedrockObj.isMissingNode()) { + return node; + } + + ObjectNode modelObj = (ObjectNode) bedrockObj.path(FIELD_MODEL); + String modelId = + (modelObj != null && !modelObj.isMissingNode()) + ? modelObj.path(FIELD_MODEL).asText(null) + : null; + + if (modelId == null || !modelId.startsWith("anthropic.")) { + // Rule 2: non-Anthropic model — passthrough + return node; + } + + // Rule 1: rewrite to anthropic/bedrock + ObjectNode anthropicObj = mapper.createObjectNode(); + anthropicObj.put(FIELD_BACKEND, "bedrock"); + + // Authentication from Bedrock (AWS credentials types) is incompatible with + // AnthropicAuthentication (apiKey / clientCredentials). Do NOT copy it; the + // migrated configuration will need to supply Anthropic auth separately. + + // Copy timeouts if present + if (bedrockObj.has(FIELD_TIMEOUTS)) { + anthropicObj.set(FIELD_TIMEOUTS, bedrockObj.get(FIELD_TIMEOUTS)); + } + + // Copy endpoint if present + if (bedrockObj.has(FIELD_ENDPOINT)) { + anthropicObj.set(FIELD_ENDPOINT, bedrockObj.get(FIELD_ENDPOINT)); + } + + // Copy model (BedrockModel has same field name as AnthropicModel) + if (bedrockObj.has(FIELD_MODEL)) { + anthropicObj.set(FIELD_MODEL, bedrockObj.get(FIELD_MODEL)); + } + + ObjectNode result = mapper.createObjectNode(); + result.put(FIELD_TYPE, ANTHROPIC_ID); + result.set(ANTHROPIC_ID, anthropicObj); + return result; + } + + /** + * Rule 3: googleVertexAi → googleGenAi with backend=vertex. + * + *

        The old shape uses {@code googleVertexAi} as both the discriminator and the nested object + * key. The new shape uses {@code googleGenAi} as both. + */ + private ObjectNode migrateGoogleVertexAiNode(ObjectNode node, ObjectMapper mapper) { + ObjectNode googleObj = mapper.createObjectNode(); + googleObj.put(FIELD_BACKEND, "vertex"); + + ObjectNode legacyObj = (ObjectNode) node.path(GOOGLE_VERTEX_AI_LEGACY_ID); + if (legacyObj != null && !legacyObj.isMissingNode()) { + legacyObj.fields().forEachRemaining(entry -> googleObj.set(entry.getKey(), entry.getValue())); + } + + ObjectNode result = mapper.createObjectNode(); + result.put(FIELD_TYPE, GOOGLE_GENAI_ID); + result.set(GOOGLE_GENAI_ID, googleObj); + return result; + } + + /** + * Rules 4 & 5: inject missing backend and/or auth type discriminator for anthropic. + * + *

        Rule 4: {@code anthropic} without backend → inject {@code backend: direct}. Rule 5: {@code + * anthropic.authentication} without {@code type} → inject {@code type: apiKey}. + */ + private ObjectNode migrateAnthropicNode(ObjectNode node, ObjectMapper mapper) { + ObjectNode anthropicObj = (ObjectNode) node.path(ANTHROPIC_ID); + if (anthropicObj == null || anthropicObj.isMissingNode()) { + return node; + } + + // Rule 4: inject backend=direct if missing + if (!anthropicObj.has(FIELD_BACKEND)) { + anthropicObj.put(FIELD_BACKEND, "direct"); + } + + // Rule 5: inject authentication.type=apiKey if auth present but missing type discriminator + if (anthropicObj.has(FIELD_AUTHENTICATION)) { + var authNode = anthropicObj.get(FIELD_AUTHENTICATION); + if (authNode != null && authNode.isObject()) { + ObjectNode authObj = (ObjectNode) authNode; + if (!authObj.has(FIELD_TYPE)) { + authObj.put(FIELD_TYPE, "apiKey"); + } + } + } + + return node; + } + + /** Rule 6: openai without backend → inject backend=openai, apiFamily=completions. */ + private ObjectNode migrateOpenAiNode(ObjectNode node, ObjectMapper mapper) { + ObjectNode openaiObj = (ObjectNode) node.path(OPENAI_ID); + if (openaiObj == null || openaiObj.isMissingNode()) { + return node; + } + + if (!openaiObj.has(FIELD_BACKEND)) { + openaiObj.put(FIELD_BACKEND, "openai"); + } + + if (!openaiObj.has(FIELD_API_FAMILY)) { + openaiObj.put(FIELD_API_FAMILY, "completions"); + } + + return node; + } + + /** + * Rule 7: azureOpenAi → openai/foundry with field mapping. + * + *

        Old structure: + * + * 

        +   * {
+   *   "type": "azureOpenAi",
+   *   "azureOpenAi": {
+   *     "endpoint": "...",
+   *     "authentication": { "type": "apiKey", "apiKey": "..." },
+   *     "model": { "deploymentName": "...", "parameters": {...} },
+   *     "timeouts": {...}
+   *   }
+   * }
+   *
        + * + *

        New structure: + * + * 

        +   * {
+   *   "type": "openai",
+   *   "openai": {
+   *     "backend": "foundry",
+   *     "apiFamily": "completions",
+   *     "endpoint": "...",
+   *     "authentication": { "type": "apiKey", "apiKey": "..." },
+   *     "model": { "model": "...", "parameters": {...} },
+   *     "timeouts": {...}
+   *   }
+   * }
+   *
        + * + *

        Note: {@code deploymentName} in the old model maps to {@code model} in the new model. + */ + private ObjectNode migrateAzureOpenAiNode(ObjectNode node, ObjectMapper mapper) { + ObjectNode azureObj = (ObjectNode) node.path(AZURE_OPENAI_LEGACY_ID); + + ObjectNode openaiObj = mapper.createObjectNode(); + openaiObj.put(FIELD_BACKEND, "foundry"); + openaiObj.put(FIELD_API_FAMILY, "completions"); + + if (azureObj != null && !azureObj.isMissingNode()) { + // Copy endpoint + if (azureObj.has(FIELD_ENDPOINT)) { + openaiObj.set(FIELD_ENDPOINT, azureObj.get(FIELD_ENDPOINT)); + } + + // Copy authentication sub-tree directly (same structure) + if (azureObj.has(FIELD_AUTHENTICATION)) { + openaiObj.set(FIELD_AUTHENTICATION, azureObj.get(FIELD_AUTHENTICATION)); + } + + // Map model: deploymentName → model + if (azureObj.has(FIELD_MODEL)) { + ObjectNode oldModel = (ObjectNode) azureObj.get(FIELD_MODEL); + ObjectNode newModel = mapper.createObjectNode(); + if (oldModel.has(FIELD_DEPLOYMENT_NAME)) { + newModel.set(FIELD_MODEL, oldModel.get(FIELD_DEPLOYMENT_NAME)); + } + if (oldModel.has("parameters")) { + newModel.set("parameters", oldModel.get("parameters")); + } + openaiObj.set(FIELD_MODEL, newModel); + } + + // Copy timeouts + if (azureObj.has(FIELD_TIMEOUTS)) { + openaiObj.set(FIELD_TIMEOUTS, azureObj.get(FIELD_TIMEOUTS)); + } + } + + ObjectNode result = mapper.createObjectNode(); + result.put(FIELD_TYPE, OPENAI_ID); + result.set(OPENAI_ID, openaiObj); + return result; + } + + /** + * Rule 8: openaiCompatible → openai/custom with field mapping and auth discriminator injection. + * + *

        Old structure: + * + * 

        +   * {
+   *   "type": "openaiCompatible",
+   *   "openaiCompatible": {
+   *     "endpoint": "...",
+   *     "authentication": { "apiKey": "..." },
+   *     "headers": {...},
+   *     "queryParameters": {...},
+   *     "timeouts": {...},
+   *     "model": { "model": "...", "parameters": {...} }
+   *   }
+   * }
+   *
        + * + *

        New structure: + * + * 

        +   * {
+   *   "type": "openai",
+   *   "openai": {
+   *     "backend": "custom",
+   *     "apiFamily": "completions",
+   *     "endpoint": "...",
+   *     "authentication": { "type": "apiKey", "apiKey": "..." },
+   *     "headers": {...},
+   *     "queryParameters": {...},
+   *     "timeouts": {...},
+   *     "model": { "model": "...", "parameters": {...} }
+   *   }
+   * }
+   *
        + * + *

        Note: old authentication was a flat record with just {@code apiKey}; the new shape requires + * a {@code type: apiKey} discriminator. + */ + private ObjectNode migrateOpenAiCompatibleNode(ObjectNode node, ObjectMapper mapper) { + ObjectNode compatObj = (ObjectNode) node.path(OPENAI_COMPATIBLE_LEGACY_ID); + + ObjectNode openaiObj = mapper.createObjectNode(); + openaiObj.put(FIELD_BACKEND, "custom"); + openaiObj.put(FIELD_API_FAMILY, "completions"); + + if (compatObj != null && !compatObj.isMissingNode()) { + // Copy endpoint + if (compatObj.has(FIELD_ENDPOINT)) { + openaiObj.set(FIELD_ENDPOINT, compatObj.get(FIELD_ENDPOINT)); + } + + // Map authentication: inject type=apiKey discriminator if missing + if (compatObj.has(FIELD_AUTHENTICATION)) { + ObjectNode oldAuth = (ObjectNode) compatObj.get(FIELD_AUTHENTICATION); + if (!oldAuth.has(FIELD_TYPE)) { + oldAuth.put(FIELD_TYPE, "apiKey"); + } + openaiObj.set(FIELD_AUTHENTICATION, oldAuth); + } + + // Copy headers + if (compatObj.has(FIELD_HEADERS)) { + openaiObj.set(FIELD_HEADERS, compatObj.get(FIELD_HEADERS)); + } + + // Copy queryParameters + if (compatObj.has(FIELD_QUERY_PARAMETERS)) { + openaiObj.set(FIELD_QUERY_PARAMETERS, compatObj.get(FIELD_QUERY_PARAMETERS)); + } + + // Copy timeouts + if (compatObj.has(FIELD_TIMEOUTS)) { + openaiObj.set(FIELD_TIMEOUTS, compatObj.get(FIELD_TIMEOUTS)); + } + + // Copy model (same structure) + if (compatObj.has(FIELD_MODEL)) { + openaiObj.set(FIELD_MODEL, compatObj.get(FIELD_MODEL)); + } + } + + ObjectNode result = mapper.createObjectNode(); + result.put(FIELD_TYPE, OPENAI_ID); + result.set(OPENAI_ID, openaiObj); + return result; + } +} diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationDeserializerTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationDeserializerTest.java new file mode 100644 index 00000000000..6af509d7d9f --- /dev/null +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/model/request/ProviderConfigurationDeserializerTest.java @@ -0,0 +1,533 @@ +/* + * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH + * under one or more contributor license agreements. Licensed under a proprietary license. + * See the License.txt file for more information. You may not use this file + * except in compliance with the proprietary license. + */ +package io.camunda.connector.agenticai.aiagent.model.request; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicAuthentication.AnthropicApiKeyAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.AnthropicProviderConfiguration.AnthropicBackend; +import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration.GoogleBackend; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.ApiFamily; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiAuthentication.OpenAiApiKeyAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiBackend; +import io.camunda.connector.agenticai.aiagent.model.request.provider.ProviderConfiguration; +import io.camunda.connector.agenticai.util.TestObjectMapperSupplier; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link + * io.camunda.connector.agenticai.aiagent.model.request.provider.ProviderConfigurationDeserializer}. + * + *

        Covers every row of the ADR 005 migration table: + * + *

          + *
        1. Old bedrock + Anthropic model → AnthropicProviderConfiguration(backend=bedrock) + *
        2. Old bedrock + Amazon model → BedrockProviderConfiguration (passthrough) + *
        3. Old googleVertexAi → GoogleGenAiProviderConfiguration(backend=vertex) + *
        4. anthropic (no backend) → backend=direct injected + *
        5. anthropic (no auth type) → auth.type=apiKey injected + *
        6. openai (no backend) → backend=openai, apiFamily=completions injected + *
        7. Old azureOpenAi → openai/foundry with auth + endpoint mapping + *
        8. Old openaiCompatible → openai/custom with auth discriminator injected + *
        9. Idempotence: deserialize old → serialize → deserialize → same result + *
        10. ConnectorsObjectMapper round-trip: no collision with document/FEEL modules + *
        + */ +class ProviderConfigurationDeserializerTest { + + // Use the ConnectorsObjectMapper configuration (ignores unknown properties, handles + // Java time types) — required so that round-trip serialization of records whose + // @AssertFalse boolean methods become JSON properties does not fail on re-read. + private static final ObjectMapper MAPPER = TestObjectMapperSupplier.getInstance(); + + // ───────────────────────────────────────────────────────────────────────────── + // Rule 1: old bedrock + Anthropic model → anthropic/bedrock + // ───────────────────────────────────────────────────────────────────────────── + + @Test + void rule1_oldBedrockWithAnthropicModel_becomesAnthropicBackendBedrock() throws Exception { + var json = + """ + { + "type": "bedrock", + "bedrock": { + "region": "us-east-1", + "authentication": { "type": "credentials", "accessKey": "ak", "secretKey": "sk" }, + "model": { "model": "anthropic.claude-sonnet-4-5" } + } + } + """; + + var result = MAPPER.readValue(json, ProviderConfiguration.class); + + assertThat(result).isInstanceOf(AnthropicProviderConfiguration.class); + var anthropic = (AnthropicProviderConfiguration) result; + assertThat(anthropic.anthropic().backend()).isEqualTo(AnthropicBackend.BEDROCK); + assertThat(anthropic.anthropic().model().model()).isEqualTo("anthropic.claude-sonnet-4-5"); + } + + @Test + void rule1_oldBedrockWithAnthropicCrossRegionModelPrefix_becomesAnthropicBackendBedrock() + throws Exception { + var json = + """ + { + "type": "bedrock", + "bedrock": { + "region": "us-east-1", + "authentication": { "type": "defaultCredentialsChain" }, + "model": { "model": "anthropic.claude-3-haiku-20240307-v1:0" } + } + } + """; + + var result = MAPPER.readValue(json, ProviderConfiguration.class); + + assertThat(result).isInstanceOf(AnthropicProviderConfiguration.class); + var anthropic = (AnthropicProviderConfiguration) result; + assertThat(anthropic.anthropic().backend()).isEqualTo(AnthropicBackend.BEDROCK); + } + + // ───────────────────────────────────────────────────────────────────────────── + // Rule 2: old bedrock + non-Anthropic model → passthrough + // ───────────────────────────────────────────────────────────────────────────── + + @Test + void rule2_oldBedrockWithAmazonModel_passthrough() throws Exception { + var json = + """ + { + "type": "bedrock", + "bedrock": { + "region": "us-east-1", + "authentication": { "type": "credentials", "accessKey": "ak", "secretKey": "sk" }, + "model": { "model": "amazon.nova-pro-v1:0" } + } + } + """; + + var result = MAPPER.readValue(json, ProviderConfiguration.class); + + assertThat(result).isInstanceOf(BedrockProviderConfiguration.class); + var bedrock = (BedrockProviderConfiguration) result; + assertThat(bedrock.bedrock().model().model()).isEqualTo("amazon.nova-pro-v1:0"); + } + + @Test + void rule2_oldBedrockWithMetaModel_passthrough() throws Exception { + var json = + """ + { + "type": "bedrock", + "bedrock": { + "region": "eu-west-1", + "authentication": { "type": "credentials", "accessKey": "ak", "secretKey": "sk" }, + "model": { "model": "meta.llama3-8b-instruct-v1:0" } + } + } + """; + + var result = MAPPER.readValue(json, ProviderConfiguration.class); + + assertThat(result).isInstanceOf(BedrockProviderConfiguration.class); + } + + // ───────────────────────────────────────────────────────────────────────────── + // Rule 3: googleVertexAi → googleGenAi/vertex + // ───────────────────────────────────────────────────────────────────────────── + + @Test + void rule3_oldGoogleVertexAi_becomesGoogleGenAiBackendVertex() throws Exception { + var json = + """ + { + "type": "googleVertexAi", + "googleVertexAi": { + "projectId": "my-project", + "region": "us-central1", + "authentication": { "type": "serviceAccountCredentials", "jsonKey": "{}" }, + "model": { "model": "gemini-1.5-flash" } + } + } + """; + + var result = MAPPER.readValue(json, ProviderConfiguration.class); + + assertThat(result).isInstanceOf(GoogleGenAiProviderConfiguration.class); + var google = (GoogleGenAiProviderConfiguration) result; + assertThat(google.googleGenAi().backend()).isEqualTo(GoogleBackend.VERTEX); + assertThat(google.googleGenAi().projectId()).isEqualTo("my-project"); + assertThat(google.googleGenAi().region()).isEqualTo("us-central1"); + assertThat(google.googleGenAi().model().model()).isEqualTo("gemini-1.5-flash"); + } + + // ───────────────────────────────────────────────────────────────────────────── + // Rule 4: anthropic without backend → inject backend=direct + // ───────────────────────────────────────────────────────────────────────────── + + @Test + void rule4_anthropicWithoutBackend_injectsBackendDirect() throws Exception { + var json = + """ + { + "type": "anthropic", + "anthropic": { + "authentication": { "type": "apiKey", "apiKey": "my-key" }, + "model": { "model": "claude-sonnet-4-6" } + } + } + """; + + var result = MAPPER.readValue(json, ProviderConfiguration.class); + + assertThat(result).isInstanceOf(AnthropicProviderConfiguration.class); + var anthropic = (AnthropicProviderConfiguration) result; + assertThat(anthropic.anthropic().backend()).isEqualTo(AnthropicBackend.DIRECT); + } + + // ───────────────────────────────────────────────────────────────────────────── + // Rule 5: anthropic with authentication but missing type discriminator + // ───────────────────────────────────────────────────────────────────────────── + + @Test + void rule5_anthropicWithoutAuthType_injectsApiKeyDiscriminator() throws Exception { + var json = + """ + { + "type": "anthropic", + "anthropic": { + "backend": "direct", + "authentication": { "apiKey": "my-secret-key" }, + "model": { "model": "claude-sonnet-4-6" } + } + } + """; + + var result = MAPPER.readValue(json, ProviderConfiguration.class); + + assertThat(result).isInstanceOf(AnthropicProviderConfiguration.class); + var anthropic = (AnthropicProviderConfiguration) result; + assertThat(anthropic.anthropic().authentication()) + .isInstanceOf(AnthropicApiKeyAuthentication.class); + assertThat(((AnthropicApiKeyAuthentication) anthropic.anthropic().authentication()).apiKey()) + .isEqualTo("my-secret-key"); + } + + @Test + void rule4and5_anthropicWithoutBackendOrAuthType_injectsBoth() throws Exception { + var json = + """ + { + "type": "anthropic", + "anthropic": { + "authentication": { "apiKey": "combined-key" }, + "model": { "model": "claude-haiku-4-5" } + } + } + """; + + var result = MAPPER.readValue(json, ProviderConfiguration.class); + + assertThat(result).isInstanceOf(AnthropicProviderConfiguration.class); + var anthropic = (AnthropicProviderConfiguration) result; + assertThat(anthropic.anthropic().backend()).isEqualTo(AnthropicBackend.DIRECT); + assertThat(anthropic.anthropic().authentication()) + .isInstanceOf(AnthropicApiKeyAuthentication.class); + } + + // ───────────────────────────────────────────────────────────────────────────── + // Rule 6: openai without backend/apiFamily → inject defaults + // ───────────────────────────────────────────────────────────────────────────── + + @Test + void rule6_openaiWithoutBackend_injectsBackendOpenaiAndApiFamilyCompletions() throws Exception { + var json = + """ + { + "type": "openai", + "openai": { + "authentication": { "type": "apiKey", "apiKey": "sk-test" }, + "model": { "model": "gpt-4o" } + } + } + """; + + var result = MAPPER.readValue(json, ProviderConfiguration.class); + + assertThat(result).isInstanceOf(OpenAiProviderConfiguration.class); + var openai = (OpenAiProviderConfiguration) result; + assertThat(openai.openai().backend()).isEqualTo(OpenAiBackend.OPENAI); + assertThat(openai.openai().apiFamily()).isEqualTo(ApiFamily.COMPLETIONS); + } + + // ───────────────────────────────────────────────────────────────────────────── + // Rule 7: azureOpenAi → openai/foundry + // ───────────────────────────────────────────────────────────────────────────── + + @Test + void rule7_azureOpenAiWithApiKey_becomesOpenAiFoundryWithFieldMapping() throws Exception { + var json = + """ + { + "type": "azureOpenAi", + "azureOpenAi": { + "endpoint": "https://my-azure.openai.azure.com", + "authentication": { "type": "apiKey", "apiKey": "azure-api-key" }, + "model": { "deploymentName": "gpt-4o-deployment", "parameters": { "maxCompletionTokens": 1000 } } + } + } + """; + + var result = MAPPER.readValue(json, ProviderConfiguration.class); + + assertThat(result).isInstanceOf(OpenAiProviderConfiguration.class); + var openai = (OpenAiProviderConfiguration) result; + assertThat(openai.openai().backend()).isEqualTo(OpenAiBackend.FOUNDRY); + assertThat(openai.openai().apiFamily()).isEqualTo(ApiFamily.COMPLETIONS); + assertThat(openai.openai().endpoint()).isEqualTo("https://my-azure.openai.azure.com"); + assertThat(openai.openai().authentication()).isInstanceOf(OpenAiApiKeyAuthentication.class); + assertThat(((OpenAiApiKeyAuthentication) openai.openai().authentication()).apiKey()) + .isEqualTo("azure-api-key"); + // deploymentName maps to model.model + assertThat(openai.openai().model().model()).isEqualTo("gpt-4o-deployment"); + assertThat(openai.openai().model().parameters().maxCompletionTokens()).isEqualTo(1000); + } + + @Test + void rule7_azureOpenAiWithClientCredentials_becomesOpenAiFoundry() throws Exception { + var json = + """ + { + "type": "azureOpenAi", + "azureOpenAi": { + "endpoint": "https://my-azure.openai.azure.com", + "authentication": { + "type": "clientCredentials", + "clientId": "cid", + "clientSecret": "csecret", + "tenantId": "tid" + }, + "model": { "deploymentName": "gpt-4o-turbo" } + } + } + """; + + var result = MAPPER.readValue(json, ProviderConfiguration.class); + + assertThat(result).isInstanceOf(OpenAiProviderConfiguration.class); + var openai = (OpenAiProviderConfiguration) result; + assertThat(openai.openai().backend()).isEqualTo(OpenAiBackend.FOUNDRY); + assertThat(openai.openai().authentication()) + .isInstanceOf( + OpenAiProviderConfiguration.OpenAiAuthentication.OpenAiClientCredentialsAuthentication + .class); + assertThat(openai.openai().model().model()).isEqualTo("gpt-4o-turbo"); + } + + // ───────────────────────────────────────────────────────────────────────────── + // Rule 8: openaiCompatible → openai/custom + // ───────────────────────────────────────────────────────────────────────────── + + @Test + void rule8_openaiCompatible_becomesOpenAiCustomWithAuthDiscriminator() throws Exception { + var json = + """ + { + "type": "openaiCompatible", + "openaiCompatible": { + "endpoint": "https://my-ollama.local/v1", + "authentication": { "apiKey": "ollama-key" }, + "headers": { "X-Custom": "value" }, + "queryParameters": { "version": "v2" }, + "model": { "model": "llama3" } + } + } + """; + + var result = MAPPER.readValue(json, ProviderConfiguration.class); + + assertThat(result).isInstanceOf(OpenAiProviderConfiguration.class); + var openai = (OpenAiProviderConfiguration) result; + assertThat(openai.openai().backend()).isEqualTo(OpenAiBackend.CUSTOM); + assertThat(openai.openai().apiFamily()).isEqualTo(ApiFamily.COMPLETIONS); + assertThat(openai.openai().endpoint()).isEqualTo("https://my-ollama.local/v1"); + assertThat(openai.openai().authentication()).isInstanceOf(OpenAiApiKeyAuthentication.class); + assertThat(((OpenAiApiKeyAuthentication) openai.openai().authentication()).apiKey()) + .isEqualTo("ollama-key"); + assertThat(openai.openai().headers()).containsEntry("X-Custom", "value"); + assertThat(openai.openai().queryParameters()).containsEntry("version", "v2"); + assertThat(openai.openai().model().model()).isEqualTo("llama3"); + } + + @Test + void rule8_openaiCompatibleWithoutAuth_becomesOpenAiCustomWithNullableApiKey() throws Exception { + var json = + """ + { + "type": "openaiCompatible", + "openaiCompatible": { + "endpoint": "https://my-vllm.local/v1", + "model": { "model": "mistral-7b" } + } + } + """; + + var result = MAPPER.readValue(json, ProviderConfiguration.class); + + assertThat(result).isInstanceOf(OpenAiProviderConfiguration.class); + var openai = (OpenAiProviderConfiguration) result; + assertThat(openai.openai().backend()).isEqualTo(OpenAiBackend.CUSTOM); + assertThat(openai.openai().model().model()).isEqualTo("mistral-7b"); + } + + // ───────────────────────────────────────────────────────────────────────────── + // Rule 9: Idempotence — deserialize old → serialize → deserialize → same result + // ───────────────────────────────────────────────────────────────────────────── + + @Test + void idempotence_oldBedrockAnthropicModel_roundTripProducesSameResult() throws Exception { + var oldJson = + """ + { + "type": "bedrock", + "bedrock": { + "region": "us-east-1", + "authentication": { "type": "credentials", "accessKey": "ak", "secretKey": "sk" }, + "model": { "model": "anthropic.claude-sonnet-4-5" } + } + } + """; + + var first = MAPPER.readValue(oldJson, ProviderConfiguration.class); + var serialized = MAPPER.writeValueAsString(first); + var second = MAPPER.readValue(serialized, ProviderConfiguration.class); + + assertThat(second).isInstanceOf(AnthropicProviderConfiguration.class); + assertThat(second).usingRecursiveComparison().isEqualTo(first); + } + + @Test + void idempotence_azureOpenAi_roundTripProducesSameResult() throws Exception { + var oldJson = + """ + { + "type": "azureOpenAi", + "azureOpenAi": { + "endpoint": "https://my-azure.openai.azure.com", + "authentication": { "type": "apiKey", "apiKey": "azure-api-key" }, + "model": { "deploymentName": "gpt-4o-deployment" } + } + } + """; + + var first = MAPPER.readValue(oldJson, ProviderConfiguration.class); + var serialized = MAPPER.writeValueAsString(first); + var second = MAPPER.readValue(serialized, ProviderConfiguration.class); + + assertThat(second).isInstanceOf(OpenAiProviderConfiguration.class); + assertThat(second).usingRecursiveComparison().isEqualTo(first); + } + + @Test + void idempotence_openaiCompatible_roundTripProducesSameResult() throws Exception { + var oldJson = + """ + { + "type": "openaiCompatible", + "openaiCompatible": { + "endpoint": "https://my-ollama.local/v1", + "authentication": { "apiKey": "ollama-key" }, + "model": { "model": "llama3" } + } + } + """; + + var first = MAPPER.readValue(oldJson, ProviderConfiguration.class); + var serialized = MAPPER.writeValueAsString(first); + var second = MAPPER.readValue(serialized, ProviderConfiguration.class); + + assertThat(second).isInstanceOf(OpenAiProviderConfiguration.class); + assertThat(second).usingRecursiveComparison().isEqualTo(first); + } + + @Test + void idempotence_googleVertexAi_roundTripProducesSameResult() throws Exception { + var oldJson = + """ + { + "type": "googleVertexAi", + "googleVertexAi": { + "projectId": "my-project", + "region": "us-central1", + "authentication": { "type": "serviceAccountCredentials", "jsonKey": "{}" }, + "model": { "model": "gemini-1.5-flash" } + } + } + """; + + var first = MAPPER.readValue(oldJson, ProviderConfiguration.class); + var serialized = MAPPER.writeValueAsString(first); + var second = MAPPER.readValue(serialized, ProviderConfiguration.class); + + assertThat(second).isInstanceOf(GoogleGenAiProviderConfiguration.class); + assertThat(second).usingRecursiveComparison().isEqualTo(first); + } + + // ───────────────────────────────────────────────────────────────────────────── + // Rule 10: ConnectorsObjectMapper round-trip — no collision with document/FEEL modules + // ───────────────────────────────────────────────────────────────────────────── + + @Test + void connectorsObjectMapper_roundTripAnthropicConfiguration_noCollisionWithDocumentOrFeelModules() + throws Exception { + var connectorsMapper = TestObjectMapperSupplier.getInstance(); + + var original = + new AnthropicProviderConfiguration( + new AnthropicProviderConfiguration.AnthropicConnection( + null, + AnthropicBackend.DIRECT, + new AnthropicApiKeyAuthentication("my-key"), + null, + new AnthropicProviderConfiguration.AnthropicModel("claude-sonnet-4-6", null))); + + var json = connectorsMapper.writeValueAsString(original); + var deserialized = connectorsMapper.readValue(json, ProviderConfiguration.class); + + assertThat(deserialized).isInstanceOf(AnthropicProviderConfiguration.class); + assertThat(deserialized).usingRecursiveComparison().isEqualTo(original); + } + + @Test + void connectorsObjectMapper_canDeserializeLegacyShapes_withFullModuleStack() throws Exception { + var connectorsMapper = TestObjectMapperSupplier.getInstance(); + + var json = + """ + { + "type": "azureOpenAi", + "azureOpenAi": { + "endpoint": "https://my-foundry.azure.com", + "authentication": { "type": "apiKey", "apiKey": "foundry-key" }, + "model": { "deploymentName": "gpt-4o" } + } + } + """; + + var result = connectorsMapper.readValue(json, ProviderConfiguration.class); + + assertThat(result).isInstanceOf(OpenAiProviderConfiguration.class); + var openai = (OpenAiProviderConfiguration) result; + assertThat(openai.openai().backend()).isEqualTo(OpenAiBackend.FOUNDRY); + assertThat(openai.openai().model().model()).isEqualTo("gpt-4o"); + } +} From 8f818e996c6fb8d5c84f15b29fc1c7a5c436f4d1 Mon Sep 17 00:00:00 2001 From: Dmitri Nikonov Date: Fri, 8 May 2026 11:37:25 +0200 Subject: [PATCH 78/81] refactor(agentic-ai): guard against ClassCastException in deserializer dispatch (Phase F) --- .../provider/ProviderConfigurationDeserializer.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/ProviderConfigurationDeserializer.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/ProviderConfigurationDeserializer.java index 207030e16d6..6ae34627238 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/ProviderConfigurationDeserializer.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/ProviderConfigurationDeserializer.java @@ -14,6 +14,7 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.deser.std.StdDeserializer; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -85,8 +86,12 @@ public ProviderConfiguration deserialize(JsonParser jp, DeserializationContext c // concrete connection type (e.g. AnthropicConnection, not AnthropicProviderConfiguration). // This avoids routing through ObjectMapper.treeToValue(ProviderConfiguration subtype), which // would re-trigger the interface-level @JsonTypeInfo + @JsonDeserialize and cause recursion. - ObjectNode innerNode = - newType != null ? (ObjectNode) migrated.path(newType) : mapper.createObjectNode(); + JsonNode rawInner = newType != null ? migrated.path(newType) : mapper.createObjectNode(); + if (!rawInner.isObject()) { + throw JsonMappingException.from( + jp, "Provider configuration is missing connection object for type: " + newType); + } + ObjectNode innerNode = (ObjectNode) rawInner; if (ANTHROPIC_ID.equals(newType)) { return new AnthropicProviderConfiguration( From 291e4f8efed7d291807edca4ba6d085ccc0d29eb Mon Sep 17 00:00:00 2001 From: Dmitri Nikonov Date: Fri, 8 May 2026 11:51:09 +0200 Subject: [PATCH 79/81] =?UTF-8?q?feat(agentic-ai):=20bump=20element=20temp?= =?UTF-8?q?late=20v11=E2=86=92v12=20with=20backend=20dropdowns?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add backend selection dropdowns to the AI Agent element template for all three configurable providers: - Anthropic: Direct / AWS Bedrock / Google Vertex AI / Azure AI Foundry - Google GenAI: Vertex AI / Developer API (Google AI Studio) - OpenAI: OpenAI / Azure AI Foundry / Custom Version bumped from 11 to 12 in @ElementTemplate; v11 templates backed up to element-templates/versioned/. README updated to reflect v12 as current. --- connectors/agentic-ai/AI_AGENT.md | 2 +- .../agentic-ai/element-templates/README.md | 6 +- .../agenticai-aiagent-job-worker.json | 779 ++++---- .../agenticai-aiagent-outbound-connector.json | 779 ++++---- .../agenticai-aiagent-job-worker-hybrid.json | 779 ++++---- ...cai-aiagent-outbound-connector-hybrid.json | 779 ++++---- .../agenticai-aiagent-job-worker-11.json | 1700 +++++++++++++++++ ...enticai-aiagent-outbound-connector-11.json | 1679 ++++++++++++++++ .../agenticai/aiagent/AiAgentFunction.java | 2 +- .../AnthropicProviderConfiguration.java | 21 +- .../GoogleGenAiProviderConfiguration.java | 15 +- .../provider/OpenAiProviderConfiguration.java | 16 +- 12 files changed, 4913 insertions(+), 1644 deletions(-) create mode 100644 connectors/agentic-ai/element-templates/versioned/agenticai-aiagent-job-worker-11.json create mode 100644 connectors/agentic-ai/element-templates/versioned/agenticai-aiagent-outbound-connector-11.json diff --git a/connectors/agentic-ai/AI_AGENT.md b/connectors/agentic-ai/AI_AGENT.md index f143e824789..e2a85230d1d 100644 --- a/connectors/agentic-ai/AI_AGENT.md +++ b/connectors/agentic-ai/AI_AGENT.md @@ -229,6 +229,6 @@ leading to the following result | Connector Info | | | --- | --- | | Type | io.camunda.agenticai:aiagent:1 | -| Version | 11 | +| Version | 12 | | Supported element types | | diff --git a/connectors/agentic-ai/element-templates/README.md b/connectors/agentic-ai/element-templates/README.md index 39b1ad31766..da06e1359f5 100644 --- a/connectors/agentic-ai/element-templates/README.md +++ b/connectors/agentic-ai/element-templates/README.md @@ -15,7 +15,7 @@ field in each JSON file captures the same information (e.g. `^8.9` means "requires Camunda 8.9 or later"). For example, if you are on Camunda 8.9, use the AI Agent template version `7`; -if you are on Camunda 8.10, use version `11`. +if you are on Camunda 8.10, use version `12`. ## AI Agent connectors @@ -26,7 +26,7 @@ The AI Agent ships in two flavors that share the same versioning scheme. | Connector | Minimum Camunda version | Template version | File | | --- | --- | --- | --- | -| AI Agent Task | 8.10 | 11 | [`agenticai-aiagent-outbound-connector.json`](./agenticai-aiagent-outbound-connector.json) | +| AI Agent Task | 8.10 | 12 | [`agenticai-aiagent-outbound-connector.json`](./agenticai-aiagent-outbound-connector.json) | | AI Agent Task | 8.9 | 7 | [`versioned/agenticai-aiagent-outbound-connector-7.json`](./versioned/agenticai-aiagent-outbound-connector-7.json) | | AI Agent Task | 8.8 | 5 | [`versioned/agenticai-aiagent-outbound-connector-5.json`](./versioned/agenticai-aiagent-outbound-connector-5.json) | @@ -34,7 +34,7 @@ The AI Agent ships in two flavors that share the same versioning scheme. | Connector | Minimum Camunda version | Template version | File | | --- | --- | --- | --- | -| AI Agent Sub-process | 8.10 | 11 | [`agenticai-aiagent-job-worker.json`](./agenticai-aiagent-job-worker.json) | +| AI Agent Sub-process | 8.10 | 12 | [`agenticai-aiagent-job-worker.json`](./agenticai-aiagent-job-worker.json) | | AI Agent Sub-process | 8.9 | 7 | [`versioned/agenticai-aiagent-job-worker-7.json`](./versioned/agenticai-aiagent-job-worker-7.json) | | AI Agent Sub-process | 8.8 | 5 | [`versioned/agenticai-aiagent-job-worker-5.json`](./versioned/agenticai-aiagent-job-worker-5.json) | diff --git a/connectors/agentic-ai/element-templates/agenticai-aiagent-job-worker.json b/connectors/agentic-ai/element-templates/agenticai-aiagent-job-worker.json index c100c2cffb1..ff8e1f99ed1 100644 --- a/connectors/agentic-ai/element-templates/agenticai-aiagent-job-worker.json +++ b/connectors/agentic-ai/element-templates/agenticai-aiagent-job-worker.json @@ -5,7 +5,7 @@ "description" : "Run a multi-step AI reasoning loop with dynamic tool selection", "keywords" : [ "AI", "AI Agent", "agentic orchestration" ], "documentationRef" : "https://docs.camunda.io/docs/8.10/components/connectors/out-of-the-box-connectors/agentic-ai-aiagent-subprocess/", - "version" : 11, + "version" : 12, "category" : { "id" : "connectors", "name" : "Connectors" @@ -113,17 +113,11 @@ "name" : "AWS Bedrock", "value" : "bedrock" }, { - "name" : "Azure OpenAI", - "value" : "azureOpenAi" - }, { - "name" : "Google Vertex AI", - "value" : "google-vertex-ai" + "name" : "Google GenAI", + "value" : "googleGenAi" }, { "name" : "OpenAI", "value" : "openai" - }, { - "name" : "OpenAI Compatible", - "value" : "openaiCompatible" } ] }, { "id" : "provider.anthropic.endpoint", @@ -142,6 +136,59 @@ "type" : "simple" }, "type" : "String" + }, { + "id" : "provider.anthropic.backend", + "label" : "Backend", + "description" : "Specify the Anthropic backend to use.", + "optional" : false, + "value" : "direct", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.backend", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Direct (Anthropic API)", + "value" : "direct" + }, { + "name" : "AWS Bedrock", + "value" : "bedrock" + }, { + "name" : "Google Vertex AI", + "value" : "vertex" + }, { + "name" : "Azure AI Foundry", + "value" : "foundry" + } ] + }, { + "id" : "provider.anthropic.authentication.type", + "label" : "Authentication", + "description" : "Specify the Anthropic authentication strategy.", + "value" : "apiKey", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "API key", + "value" : "apiKey" + }, { + "name" : "Client credentials", + "value" : "clientCredentials" + } ] }, { "id" : "provider.anthropic.authentication.apiKey", "label" : "Anthropic API key", @@ -156,9 +203,116 @@ "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "anthropic", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.clientId", + "label" : "Client ID", + "description" : "ID of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.clientId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.clientSecret", + "label" : "Client secret", + "description" : "Secret of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.clientSecret", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.tenantId", + "label" : "Tenant ID", + "description" : "ID of a Microsoft Entra tenant. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.tenantId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.authorityHost", + "label" : "Authority host", + "description" : "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.authorityHost", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] }, "type" : "String" }, { @@ -337,9 +491,9 @@ }, "type" : "String" }, { - "id" : "provider.azureOpenAi.endpoint", - "label" : "Endpoint", - "description" : "Specify Azure OpenAI endpoint. Details in the documentation.", + "id" : "provider.googleGenAi.projectId", + "label" : "Project ID", + "description" : "Specify Google Cloud project ID", "optional" : false, "constraints" : { "notEmpty" : true @@ -347,41 +501,19 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.endpoint", + "name" : "provider.googleGenAi.projectId", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "azureOpenAi", + "equals" : "googleGenAi", "type" : "simple" }, "type" : "String" }, { - "id" : "provider.azureOpenAi.authentication.type", - "label" : "Authentication", - "description" : "Specify the Azure OpenAI authentication strategy.", - "value" : "apiKey", - "group" : "provider", - "binding" : { - "name" : "provider.azureOpenAi.authentication.type", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "type" : "Dropdown", - "choices" : [ { - "name" : "API key", - "value" : "apiKey" - }, { - "name" : "Client credentials", - "value" : "clientCredentials" - } ] - }, { - "id" : "provider.azureOpenAi.authentication.apiKey", - "label" : "API key", + "id" : "provider.googleGenAi.region", + "label" : "Region", + "description" : "Specify the region where AI inference should take place", "optional" : false, "constraints" : { "notEmpty" : true @@ -389,51 +521,42 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.apiKey", + "name" : "provider.googleGenAi.region", "type" : "zeebe:input" }, "condition" : { - "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "apiKey", - "type" : "simple" - }, { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - } ] + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" }, "type" : "String" }, { - "id" : "provider.azureOpenAi.authentication.clientId", - "label" : "Client ID", - "description" : "ID of a Microsoft Entra application", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", + "id" : "provider.googleGenAi.authentication.type", + "label" : "Authentication", + "description" : "Specify the Google Vertex AI authentication strategy.", + "value" : "serviceAccountCredentials", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.clientId", + "name" : "provider.googleGenAi.authentication.type", "type" : "zeebe:input" }, "condition" : { - "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "clientCredentials", - "type" : "simple" - }, { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - } ] + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" }, - "type" : "String" + "type" : "Dropdown", + "choices" : [ { + "name" : "Service account credentials", + "value" : "serviceAccountCredentials" + }, { + "name" : "Application default credentials (Hybrid/Self-Managed only)", + "value" : "applicationDefaultCredentials" + } ] }, { - "id" : "provider.azureOpenAi.authentication.clientSecret", - "label" : "Client secret", - "description" : "Secret of a Microsoft Entra application", + "id" : "provider.googleGenAi.authentication.jsonKey", + "label" : "JSON key of the service account", + "description" : "This is the key of the service account in JSON format.", "optional" : false, "constraints" : { "notEmpty" : true @@ -441,154 +564,167 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.clientSecret", + "name" : "provider.googleGenAi.authentication.jsonKey", "type" : "zeebe:input" }, "condition" : { "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "clientCredentials", + "property" : "provider.googleGenAi.authentication.type", + "equals" : "serviceAccountCredentials", "type" : "simple" }, { "property" : "provider.type", - "equals" : "azureOpenAi", + "equals" : "googleGenAi", "type" : "simple" } ] }, "type" : "String" }, { - "id" : "provider.azureOpenAi.authentication.tenantId", - "label" : "Tenant ID", - "description" : "ID of a Microsoft Entra tenant. Details in the documentation.", + "id" : "provider.googleGenAi.backend", + "label" : "Backend", + "description" : "Specify the Google GenAI backend to use.", "optional" : false, - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", + "value" : "vertex", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.tenantId", + "name" : "provider.googleGenAi.backend", "type" : "zeebe:input" }, "condition" : { - "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "clientCredentials", - "type" : "simple" - }, { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - } ] + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" }, - "type" : "String" + "type" : "Dropdown", + "choices" : [ { + "name" : "Vertex AI", + "value" : "vertex" + }, { + "name" : "Developer API (Google AI Studio)", + "value" : "developer-api" + } ] }, { - "id" : "provider.azureOpenAi.authentication.authorityHost", - "label" : "Authority host", - "description" : "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", - "optional" : true, - "feel" : "optional", + "id" : "provider.openai.backend", + "label" : "Backend", + "description" : "Specify the OpenAI backend to use.", + "optional" : false, + "value" : "openai", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.authorityHost", + "name" : "provider.openai.backend", "type" : "zeebe:input" }, "condition" : { - "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "clientCredentials", - "type" : "simple" - }, { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - } ] + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" }, - "type" : "String" + "type" : "Dropdown", + "choices" : [ { + "name" : "OpenAI", + "value" : "openai" + }, { + "name" : "Azure AI Foundry", + "value" : "foundry" + }, { + "name" : "Custom", + "value" : "custom" + } ] }, { - "id" : "provider.azureOpenAi.timeouts.timeout", - "label" : "Timeout", - "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", - "optional" : true, - "feel" : "optional", + "id" : "provider.openai.authentication.type", + "label" : "Authentication", + "description" : "Specify the OpenAI authentication strategy.", + "value" : "apiKey", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.timeouts.timeout", + "name" : "provider.openai.authentication.type", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "azureOpenAi", + "equals" : "openai", "type" : "simple" }, - "type" : "String" + "type" : "Dropdown", + "choices" : [ { + "name" : "API key", + "value" : "apiKey" + }, { + "name" : "Client credentials", + "value" : "clientCredentials" + } ] }, { - "id" : "provider.googleVertexAi.projectId", - "label" : "Project ID", - "description" : "Specify Google Cloud project ID", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, + "id" : "provider.openai.authentication.apiKey", + "label" : "API key", + "optional" : true, "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.googleVertexAi.projectId", + "name" : "provider.openai.authentication.apiKey", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "google-vertex-ai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { - "id" : "provider.googleVertexAi.region", - "label" : "Region", - "description" : "Specify the region where AI inference should take place", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, + "id" : "provider.openai.authentication.organizationId", + "label" : "Organization ID", + "description" : "For members of multiple organizations. Details in the documentation.", + "optional" : true, "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.googleVertexAi.region", + "name" : "provider.openai.authentication.organizationId", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "google-vertex-ai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { - "id" : "provider.googleVertexAi.authentication.type", - "label" : "Authentication", - "description" : "Specify the Google Vertex AI authentication strategy.", - "value" : "serviceAccountCredentials", + "id" : "provider.openai.authentication.projectId", + "label" : "Project ID", + "description" : "For accounts with multiple projects. Details in the documentation.", + "optional" : true, + "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.googleVertexAi.authentication.type", + "name" : "provider.openai.authentication.projectId", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "google-vertex-ai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, - "type" : "Dropdown", - "choices" : [ { - "name" : "Service account credentials", - "value" : "serviceAccountCredentials" - }, { - "name" : "Application default credentials (Hybrid/Self-Managed only)", - "value" : "applicationDefaultCredentials" - } ] + "type" : "String" }, { - "id" : "provider.googleVertexAi.authentication.jsonKey", - "label" : "JSON key of the service account", - "description" : "This is the key of the service account in JSON format.", + "id" : "provider.openai.authentication.clientId", + "label" : "Client ID", + "description" : "ID of a Microsoft Entra application", "optional" : false, "constraints" : { "notEmpty" : true @@ -596,24 +732,25 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.googleVertexAi.authentication.jsonKey", + "name" : "provider.openai.authentication.clientId", "type" : "zeebe:input" }, "condition" : { "allMatch" : [ { - "property" : "provider.googleVertexAi.authentication.type", - "equals" : "serviceAccountCredentials", + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", "type" : "simple" }, { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "openai", "type" : "simple" } ] }, "type" : "String" }, { - "id" : "provider.openai.authentication.apiKey", - "label" : "OpenAI API key", + "id" : "provider.openai.authentication.clientSecret", + "label" : "Client secret", + "description" : "Secret of a Microsoft Entra application", "optional" : false, "constraints" : { "notEmpty" : true @@ -621,47 +758,68 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.openai.authentication.apiKey", + "name" : "provider.openai.authentication.clientSecret", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "openai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { - "id" : "provider.openai.authentication.organizationId", - "label" : "Organization ID", - "description" : "For members of multiple organizations. Details in the documentation.", - "optional" : true, + "id" : "provider.openai.authentication.tenantId", + "label" : "Tenant ID", + "description" : "ID of a Microsoft Entra tenant. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.openai.authentication.organizationId", + "name" : "provider.openai.authentication.tenantId", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "openai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { - "id" : "provider.openai.authentication.projectId", - "label" : "Project ID", - "description" : "For accounts with multiple projects. Details in the documentation.", + "id" : "provider.openai.authentication.authorityHost", + "label" : "Authority host", + "description" : "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", "optional" : true, "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.openai.authentication.projectId", + "name" : "provider.openai.authentication.authorityHost", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "openai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { @@ -707,7 +865,7 @@ } ] }, { "id" : "provider.openai.endpoint", - "label" : "Custom API endpoint", + "label" : "Endpoint", "description" : "Optional. Override the default OpenAI base URL (e.g. for an OpenAI proxy or gateway). Leave blank to use the SDK default.", "optional" : true, "feel" : "optional", @@ -723,90 +881,36 @@ }, "type" : "String" }, { - "id" : "provider.openaiCompatible.endpoint", - "label" : "API endpoint", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", - "group" : "provider", - "binding" : { - "name" : "provider.openaiCompatible.endpoint", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "Specify an endpoint to use the connector with an OpenAI compatible API. ", - "type" : "String" - }, { - "id" : "provider.openaiCompatible.authentication.apiKey", - "label" : "API key", - "optional" : true, - "feel" : "optional", - "group" : "provider", - "binding" : { - "name" : "provider.openaiCompatible.authentication.apiKey", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "Leave blank if using HTTP headers for authentication.
        If an Authorization header is specified in the headers, then the API key is ignored.", - "type" : "String" - }, { - "id" : "provider.openaiCompatible.headers", + "id" : "provider.openai.headers", "label" : "Headers", "description" : "Map of HTTP headers to add to the request.", "optional" : true, "feel" : "required", "group" : "provider", "binding" : { - "name" : "provider.openaiCompatible.headers", + "name" : "provider.openai.headers", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "openaiCompatible", + "equals" : "openai", "type" : "simple" }, "type" : "String" }, { - "id" : "provider.openaiCompatible.queryParameters", + "id" : "provider.openai.queryParameters", "label" : "Query Parameters", "description" : "Map of query parameters to add to the request URL.", "optional" : true, "feel" : "required", "group" : "provider", "binding" : { - "name" : "provider.openaiCompatible.queryParameters", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "type" : "String" - }, { - "id" : "provider.openaiCompatible.timeouts.timeout", - "label" : "Timeout", - "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", - "optional" : true, - "feel" : "optional", - "group" : "provider", - "binding" : { - "name" : "provider.openaiCompatible.timeouts.timeout", + "name" : "provider.openai.queryParameters", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "openaiCompatible", + "equals" : "openai", "type" : "simple" }, "type" : "String" @@ -972,78 +1076,7 @@ "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.azureOpenAi.model.deploymentName", - "label" : "Model deployment name", - "description" : "Specify the model deployment name. Details in the documentation.", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", - "group" : "model", - "binding" : { - "name" : "provider.azureOpenAi.model.deploymentName", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "type" : "String" - }, { - "id" : "provider.azureOpenAi.model.parameters.maxTokens", - "label" : "Maximum tokens", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.azureOpenAi.model.parameters.maxTokens", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "tooltip" : "The maximum number of tokens per request to generate before stopping.

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.azureOpenAi.model.parameters.temperature", - "label" : "Temperature", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.azureOpenAi.model.parameters.temperature", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "tooltip" : "Floating point number between 0 and 2. The higher the number, the more randomness will be injected into the response.

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.azureOpenAi.model.parameters.topP", - "label" : "top P", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.azureOpenAi.model.parameters.topP", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.googleVertexAi.model.model", + "id" : "provider.googleGenAi.model.model", "label" : "Model", "description" : "Specify the model ID. Details in the documentation.", "optional" : false, @@ -1053,79 +1086,79 @@ "feel" : "optional", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.model", + "name" : "provider.googleGenAi.model.model", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "type" : "String" }, { - "id" : "provider.googleVertexAi.model.parameters.maxOutputTokens", + "id" : "provider.googleGenAi.model.parameters.maxOutputTokens", "label" : "Maximum output tokens", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.parameters.maxOutputTokens", + "name" : "provider.googleGenAi.model.parameters.maxOutputTokens", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "tooltip" : "Maximum number of tokens that can be generated in the response.

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.googleVertexAi.model.parameters.temperature", + "id" : "provider.googleGenAi.model.parameters.temperature", "label" : "Temperature", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.parameters.temperature", + "name" : "provider.googleGenAi.model.parameters.temperature", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "tooltip" : "Controls the degree of randomness in token selection.

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.googleVertexAi.model.parameters.topP", + "id" : "provider.googleGenAi.model.parameters.topP", "label" : "top P", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.parameters.topP", + "name" : "provider.googleGenAi.model.parameters.topP", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.googleVertexAi.model.parameters.topK", + "id" : "provider.googleGenAi.model.parameters.topK", "label" : "top K", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.parameters.topK", + "name" : "provider.googleGenAi.model.parameters.topK", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "tooltip" : "Integer greater than 0. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", @@ -1203,91 +1236,19 @@ "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.openaiCompatible.model.model", - "label" : "Model", - "description" : "Specify the model ID. Details in the documentation.", - "optional" : false, - "value" : "gpt-4o", - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", - "group" : "model", - "binding" : { - "name" : "provider.openaiCompatible.model.model", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "type" : "String" - }, { - "id" : "provider.openaiCompatible.model.parameters.maxCompletionTokens", - "label" : "Maximum completion tokens", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.openaiCompatible.model.parameters.maxCompletionTokens", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "The maximum number of tokens per request to generate before stopping.

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.openaiCompatible.model.parameters.temperature", - "label" : "Temperature", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.openaiCompatible.model.parameters.temperature", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "Floating point number between 0 and 2. The higher the number, the more randomness will be injected into the response.

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.openaiCompatible.model.parameters.topP", - "label" : "top P", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.openaiCompatible.model.parameters.topP", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.openaiCompatible.model.parameters.customParameters", + "id" : "provider.openai.model.parameters.customParameters", "label" : "Custom parameters", "description" : "Map of additional request parameters to include.", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.openaiCompatible.model.parameters.customParameters", + "name" : "provider.openai.model.parameters.customParameters", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "openaiCompatible", + "equals" : "openai", "type" : "simple" }, "type" : "String" @@ -1744,7 +1705,7 @@ "id" : "version", "label" : "Version", "description" : "Version of the element template", - "value" : "11", + "value" : "12", "group" : "connector", "binding" : { "key" : "elementTemplateVersion", diff --git a/connectors/agentic-ai/element-templates/agenticai-aiagent-outbound-connector.json b/connectors/agentic-ai/element-templates/agenticai-aiagent-outbound-connector.json index 512e85a5036..3982bf9e981 100644 --- a/connectors/agentic-ai/element-templates/agenticai-aiagent-outbound-connector.json +++ b/connectors/agentic-ai/element-templates/agenticai-aiagent-outbound-connector.json @@ -5,7 +5,7 @@ "description" : "Execute a single AI-powered action with tool calling capabilities", "keywords" : [ "AI", "AI Agent", "agentic orchestration" ], "documentationRef" : "https://docs.camunda.io/docs/8.10/components/connectors/out-of-the-box-connectors/agentic-ai-aiagent-task/", - "version" : 11, + "version" : 12, "category" : { "id" : "connectors", "name" : "Connectors" @@ -92,17 +92,11 @@ "name" : "AWS Bedrock", "value" : "bedrock" }, { - "name" : "Azure OpenAI", - "value" : "azureOpenAi" - }, { - "name" : "Google Vertex AI", - "value" : "google-vertex-ai" + "name" : "Google GenAI", + "value" : "googleGenAi" }, { "name" : "OpenAI", "value" : "openai" - }, { - "name" : "OpenAI Compatible", - "value" : "openaiCompatible" } ] }, { "id" : "provider.anthropic.endpoint", @@ -121,6 +115,59 @@ "type" : "simple" }, "type" : "String" + }, { + "id" : "provider.anthropic.backend", + "label" : "Backend", + "description" : "Specify the Anthropic backend to use.", + "optional" : false, + "value" : "direct", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.backend", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Direct (Anthropic API)", + "value" : "direct" + }, { + "name" : "AWS Bedrock", + "value" : "bedrock" + }, { + "name" : "Google Vertex AI", + "value" : "vertex" + }, { + "name" : "Azure AI Foundry", + "value" : "foundry" + } ] + }, { + "id" : "provider.anthropic.authentication.type", + "label" : "Authentication", + "description" : "Specify the Anthropic authentication strategy.", + "value" : "apiKey", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "API key", + "value" : "apiKey" + }, { + "name" : "Client credentials", + "value" : "clientCredentials" + } ] }, { "id" : "provider.anthropic.authentication.apiKey", "label" : "Anthropic API key", @@ -135,9 +182,116 @@ "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "anthropic", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.clientId", + "label" : "Client ID", + "description" : "ID of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.clientId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.clientSecret", + "label" : "Client secret", + "description" : "Secret of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.clientSecret", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.tenantId", + "label" : "Tenant ID", + "description" : "ID of a Microsoft Entra tenant. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.tenantId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.authorityHost", + "label" : "Authority host", + "description" : "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.authorityHost", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] }, "type" : "String" }, { @@ -316,9 +470,9 @@ }, "type" : "String" }, { - "id" : "provider.azureOpenAi.endpoint", - "label" : "Endpoint", - "description" : "Specify Azure OpenAI endpoint. Details in the documentation.", + "id" : "provider.googleGenAi.projectId", + "label" : "Project ID", + "description" : "Specify Google Cloud project ID", "optional" : false, "constraints" : { "notEmpty" : true @@ -326,41 +480,19 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.endpoint", + "name" : "provider.googleGenAi.projectId", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "azureOpenAi", + "equals" : "googleGenAi", "type" : "simple" }, "type" : "String" }, { - "id" : "provider.azureOpenAi.authentication.type", - "label" : "Authentication", - "description" : "Specify the Azure OpenAI authentication strategy.", - "value" : "apiKey", - "group" : "provider", - "binding" : { - "name" : "provider.azureOpenAi.authentication.type", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "type" : "Dropdown", - "choices" : [ { - "name" : "API key", - "value" : "apiKey" - }, { - "name" : "Client credentials", - "value" : "clientCredentials" - } ] - }, { - "id" : "provider.azureOpenAi.authentication.apiKey", - "label" : "API key", + "id" : "provider.googleGenAi.region", + "label" : "Region", + "description" : "Specify the region where AI inference should take place", "optional" : false, "constraints" : { "notEmpty" : true @@ -368,51 +500,42 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.apiKey", + "name" : "provider.googleGenAi.region", "type" : "zeebe:input" }, "condition" : { - "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "apiKey", - "type" : "simple" - }, { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - } ] + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" }, "type" : "String" }, { - "id" : "provider.azureOpenAi.authentication.clientId", - "label" : "Client ID", - "description" : "ID of a Microsoft Entra application", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", + "id" : "provider.googleGenAi.authentication.type", + "label" : "Authentication", + "description" : "Specify the Google Vertex AI authentication strategy.", + "value" : "serviceAccountCredentials", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.clientId", + "name" : "provider.googleGenAi.authentication.type", "type" : "zeebe:input" }, "condition" : { - "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "clientCredentials", - "type" : "simple" - }, { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - } ] + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" }, - "type" : "String" + "type" : "Dropdown", + "choices" : [ { + "name" : "Service account credentials", + "value" : "serviceAccountCredentials" + }, { + "name" : "Application default credentials (Hybrid/Self-Managed only)", + "value" : "applicationDefaultCredentials" + } ] }, { - "id" : "provider.azureOpenAi.authentication.clientSecret", - "label" : "Client secret", - "description" : "Secret of a Microsoft Entra application", + "id" : "provider.googleGenAi.authentication.jsonKey", + "label" : "JSON key of the service account", + "description" : "This is the key of the service account in JSON format.", "optional" : false, "constraints" : { "notEmpty" : true @@ -420,154 +543,167 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.clientSecret", + "name" : "provider.googleGenAi.authentication.jsonKey", "type" : "zeebe:input" }, "condition" : { "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "clientCredentials", + "property" : "provider.googleGenAi.authentication.type", + "equals" : "serviceAccountCredentials", "type" : "simple" }, { "property" : "provider.type", - "equals" : "azureOpenAi", + "equals" : "googleGenAi", "type" : "simple" } ] }, "type" : "String" }, { - "id" : "provider.azureOpenAi.authentication.tenantId", - "label" : "Tenant ID", - "description" : "ID of a Microsoft Entra tenant. Details in the documentation.", + "id" : "provider.googleGenAi.backend", + "label" : "Backend", + "description" : "Specify the Google GenAI backend to use.", "optional" : false, - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", + "value" : "vertex", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.tenantId", + "name" : "provider.googleGenAi.backend", "type" : "zeebe:input" }, "condition" : { - "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "clientCredentials", - "type" : "simple" - }, { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - } ] + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" }, - "type" : "String" + "type" : "Dropdown", + "choices" : [ { + "name" : "Vertex AI", + "value" : "vertex" + }, { + "name" : "Developer API (Google AI Studio)", + "value" : "developer-api" + } ] }, { - "id" : "provider.azureOpenAi.authentication.authorityHost", - "label" : "Authority host", - "description" : "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", - "optional" : true, - "feel" : "optional", + "id" : "provider.openai.backend", + "label" : "Backend", + "description" : "Specify the OpenAI backend to use.", + "optional" : false, + "value" : "openai", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.authorityHost", + "name" : "provider.openai.backend", "type" : "zeebe:input" }, "condition" : { - "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "clientCredentials", - "type" : "simple" - }, { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - } ] + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" }, - "type" : "String" + "type" : "Dropdown", + "choices" : [ { + "name" : "OpenAI", + "value" : "openai" + }, { + "name" : "Azure AI Foundry", + "value" : "foundry" + }, { + "name" : "Custom", + "value" : "custom" + } ] }, { - "id" : "provider.azureOpenAi.timeouts.timeout", - "label" : "Timeout", - "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", - "optional" : true, - "feel" : "optional", + "id" : "provider.openai.authentication.type", + "label" : "Authentication", + "description" : "Specify the OpenAI authentication strategy.", + "value" : "apiKey", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.timeouts.timeout", + "name" : "provider.openai.authentication.type", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "azureOpenAi", + "equals" : "openai", "type" : "simple" }, - "type" : "String" + "type" : "Dropdown", + "choices" : [ { + "name" : "API key", + "value" : "apiKey" + }, { + "name" : "Client credentials", + "value" : "clientCredentials" + } ] }, { - "id" : "provider.googleVertexAi.projectId", - "label" : "Project ID", - "description" : "Specify Google Cloud project ID", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, + "id" : "provider.openai.authentication.apiKey", + "label" : "API key", + "optional" : true, "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.googleVertexAi.projectId", + "name" : "provider.openai.authentication.apiKey", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "google-vertex-ai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { - "id" : "provider.googleVertexAi.region", - "label" : "Region", - "description" : "Specify the region where AI inference should take place", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, + "id" : "provider.openai.authentication.organizationId", + "label" : "Organization ID", + "description" : "For members of multiple organizations. Details in the documentation.", + "optional" : true, "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.googleVertexAi.region", + "name" : "provider.openai.authentication.organizationId", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "google-vertex-ai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { - "id" : "provider.googleVertexAi.authentication.type", - "label" : "Authentication", - "description" : "Specify the Google Vertex AI authentication strategy.", - "value" : "serviceAccountCredentials", + "id" : "provider.openai.authentication.projectId", + "label" : "Project ID", + "description" : "For accounts with multiple projects. Details in the documentation.", + "optional" : true, + "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.googleVertexAi.authentication.type", + "name" : "provider.openai.authentication.projectId", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "google-vertex-ai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, - "type" : "Dropdown", - "choices" : [ { - "name" : "Service account credentials", - "value" : "serviceAccountCredentials" - }, { - "name" : "Application default credentials (Hybrid/Self-Managed only)", - "value" : "applicationDefaultCredentials" - } ] + "type" : "String" }, { - "id" : "provider.googleVertexAi.authentication.jsonKey", - "label" : "JSON key of the service account", - "description" : "This is the key of the service account in JSON format.", + "id" : "provider.openai.authentication.clientId", + "label" : "Client ID", + "description" : "ID of a Microsoft Entra application", "optional" : false, "constraints" : { "notEmpty" : true @@ -575,24 +711,25 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.googleVertexAi.authentication.jsonKey", + "name" : "provider.openai.authentication.clientId", "type" : "zeebe:input" }, "condition" : { "allMatch" : [ { - "property" : "provider.googleVertexAi.authentication.type", - "equals" : "serviceAccountCredentials", + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", "type" : "simple" }, { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "openai", "type" : "simple" } ] }, "type" : "String" }, { - "id" : "provider.openai.authentication.apiKey", - "label" : "OpenAI API key", + "id" : "provider.openai.authentication.clientSecret", + "label" : "Client secret", + "description" : "Secret of a Microsoft Entra application", "optional" : false, "constraints" : { "notEmpty" : true @@ -600,47 +737,68 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.openai.authentication.apiKey", + "name" : "provider.openai.authentication.clientSecret", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "openai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { - "id" : "provider.openai.authentication.organizationId", - "label" : "Organization ID", - "description" : "For members of multiple organizations. Details in the documentation.", - "optional" : true, + "id" : "provider.openai.authentication.tenantId", + "label" : "Tenant ID", + "description" : "ID of a Microsoft Entra tenant. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.openai.authentication.organizationId", + "name" : "provider.openai.authentication.tenantId", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "openai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { - "id" : "provider.openai.authentication.projectId", - "label" : "Project ID", - "description" : "For accounts with multiple projects. Details in the documentation.", + "id" : "provider.openai.authentication.authorityHost", + "label" : "Authority host", + "description" : "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", "optional" : true, "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.openai.authentication.projectId", + "name" : "provider.openai.authentication.authorityHost", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "openai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { @@ -686,7 +844,7 @@ } ] }, { "id" : "provider.openai.endpoint", - "label" : "Custom API endpoint", + "label" : "Endpoint", "description" : "Optional. Override the default OpenAI base URL (e.g. for an OpenAI proxy or gateway). Leave blank to use the SDK default.", "optional" : true, "feel" : "optional", @@ -702,90 +860,36 @@ }, "type" : "String" }, { - "id" : "provider.openaiCompatible.endpoint", - "label" : "API endpoint", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", - "group" : "provider", - "binding" : { - "name" : "provider.openaiCompatible.endpoint", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "Specify an endpoint to use the connector with an OpenAI compatible API. ", - "type" : "String" - }, { - "id" : "provider.openaiCompatible.authentication.apiKey", - "label" : "API key", - "optional" : true, - "feel" : "optional", - "group" : "provider", - "binding" : { - "name" : "provider.openaiCompatible.authentication.apiKey", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "Leave blank if using HTTP headers for authentication.
        If an Authorization header is specified in the headers, then the API key is ignored.", - "type" : "String" - }, { - "id" : "provider.openaiCompatible.headers", + "id" : "provider.openai.headers", "label" : "Headers", "description" : "Map of HTTP headers to add to the request.", "optional" : true, "feel" : "required", "group" : "provider", "binding" : { - "name" : "provider.openaiCompatible.headers", + "name" : "provider.openai.headers", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "openaiCompatible", + "equals" : "openai", "type" : "simple" }, "type" : "String" }, { - "id" : "provider.openaiCompatible.queryParameters", + "id" : "provider.openai.queryParameters", "label" : "Query Parameters", "description" : "Map of query parameters to add to the request URL.", "optional" : true, "feel" : "required", "group" : "provider", "binding" : { - "name" : "provider.openaiCompatible.queryParameters", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "type" : "String" - }, { - "id" : "provider.openaiCompatible.timeouts.timeout", - "label" : "Timeout", - "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", - "optional" : true, - "feel" : "optional", - "group" : "provider", - "binding" : { - "name" : "provider.openaiCompatible.timeouts.timeout", + "name" : "provider.openai.queryParameters", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "openaiCompatible", + "equals" : "openai", "type" : "simple" }, "type" : "String" @@ -951,78 +1055,7 @@ "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.azureOpenAi.model.deploymentName", - "label" : "Model deployment name", - "description" : "Specify the model deployment name. Details in the documentation.", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", - "group" : "model", - "binding" : { - "name" : "provider.azureOpenAi.model.deploymentName", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "type" : "String" - }, { - "id" : "provider.azureOpenAi.model.parameters.maxTokens", - "label" : "Maximum tokens", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.azureOpenAi.model.parameters.maxTokens", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "tooltip" : "The maximum number of tokens per request to generate before stopping.

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.azureOpenAi.model.parameters.temperature", - "label" : "Temperature", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.azureOpenAi.model.parameters.temperature", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "tooltip" : "Floating point number between 0 and 2. The higher the number, the more randomness will be injected into the response.

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.azureOpenAi.model.parameters.topP", - "label" : "top P", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.azureOpenAi.model.parameters.topP", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.googleVertexAi.model.model", + "id" : "provider.googleGenAi.model.model", "label" : "Model", "description" : "Specify the model ID. Details in the documentation.", "optional" : false, @@ -1032,79 +1065,79 @@ "feel" : "optional", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.model", + "name" : "provider.googleGenAi.model.model", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "type" : "String" }, { - "id" : "provider.googleVertexAi.model.parameters.maxOutputTokens", + "id" : "provider.googleGenAi.model.parameters.maxOutputTokens", "label" : "Maximum output tokens", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.parameters.maxOutputTokens", + "name" : "provider.googleGenAi.model.parameters.maxOutputTokens", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "tooltip" : "Maximum number of tokens that can be generated in the response.

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.googleVertexAi.model.parameters.temperature", + "id" : "provider.googleGenAi.model.parameters.temperature", "label" : "Temperature", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.parameters.temperature", + "name" : "provider.googleGenAi.model.parameters.temperature", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "tooltip" : "Controls the degree of randomness in token selection.

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.googleVertexAi.model.parameters.topP", + "id" : "provider.googleGenAi.model.parameters.topP", "label" : "top P", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.parameters.topP", + "name" : "provider.googleGenAi.model.parameters.topP", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.googleVertexAi.model.parameters.topK", + "id" : "provider.googleGenAi.model.parameters.topK", "label" : "top K", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.parameters.topK", + "name" : "provider.googleGenAi.model.parameters.topK", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "tooltip" : "Integer greater than 0. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", @@ -1182,91 +1215,19 @@ "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.openaiCompatible.model.model", - "label" : "Model", - "description" : "Specify the model ID. Details in the documentation.", - "optional" : false, - "value" : "gpt-4o", - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", - "group" : "model", - "binding" : { - "name" : "provider.openaiCompatible.model.model", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "type" : "String" - }, { - "id" : "provider.openaiCompatible.model.parameters.maxCompletionTokens", - "label" : "Maximum completion tokens", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.openaiCompatible.model.parameters.maxCompletionTokens", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "The maximum number of tokens per request to generate before stopping.

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.openaiCompatible.model.parameters.temperature", - "label" : "Temperature", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.openaiCompatible.model.parameters.temperature", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "Floating point number between 0 and 2. The higher the number, the more randomness will be injected into the response.

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.openaiCompatible.model.parameters.topP", - "label" : "top P", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.openaiCompatible.model.parameters.topP", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.openaiCompatible.model.parameters.customParameters", + "id" : "provider.openai.model.parameters.customParameters", "label" : "Custom parameters", "description" : "Map of additional request parameters to include.", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.openaiCompatible.model.parameters.customParameters", + "name" : "provider.openai.model.parameters.customParameters", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "openaiCompatible", + "equals" : "openai", "type" : "simple" }, "type" : "String" @@ -1718,7 +1679,7 @@ "id" : "version", "label" : "Version", "description" : "Version of the element template", - "value" : "11", + "value" : "12", "group" : "connector", "binding" : { "key" : "elementTemplateVersion", diff --git a/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-job-worker-hybrid.json b/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-job-worker-hybrid.json index 3c9cb6904d2..3ca2a8afb8d 100644 --- a/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-job-worker-hybrid.json +++ b/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-job-worker-hybrid.json @@ -5,7 +5,7 @@ "description" : "Run a multi-step AI reasoning loop with dynamic tool selection", "keywords" : [ "AI", "AI Agent", "agentic orchestration" ], "documentationRef" : "https://docs.camunda.io/docs/8.10/components/connectors/out-of-the-box-connectors/agentic-ai-aiagent-subprocess/", - "version" : 11, + "version" : 12, "category" : { "id" : "connectors", "name" : "Connectors" @@ -118,17 +118,11 @@ "name" : "AWS Bedrock", "value" : "bedrock" }, { - "name" : "Azure OpenAI", - "value" : "azureOpenAi" - }, { - "name" : "Google Vertex AI", - "value" : "google-vertex-ai" + "name" : "Google GenAI", + "value" : "googleGenAi" }, { "name" : "OpenAI", "value" : "openai" - }, { - "name" : "OpenAI Compatible", - "value" : "openaiCompatible" } ] }, { "id" : "provider.anthropic.endpoint", @@ -147,6 +141,59 @@ "type" : "simple" }, "type" : "String" + }, { + "id" : "provider.anthropic.backend", + "label" : "Backend", + "description" : "Specify the Anthropic backend to use.", + "optional" : false, + "value" : "direct", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.backend", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Direct (Anthropic API)", + "value" : "direct" + }, { + "name" : "AWS Bedrock", + "value" : "bedrock" + }, { + "name" : "Google Vertex AI", + "value" : "vertex" + }, { + "name" : "Azure AI Foundry", + "value" : "foundry" + } ] + }, { + "id" : "provider.anthropic.authentication.type", + "label" : "Authentication", + "description" : "Specify the Anthropic authentication strategy.", + "value" : "apiKey", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "API key", + "value" : "apiKey" + }, { + "name" : "Client credentials", + "value" : "clientCredentials" + } ] }, { "id" : "provider.anthropic.authentication.apiKey", "label" : "Anthropic API key", @@ -161,9 +208,116 @@ "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "anthropic", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.clientId", + "label" : "Client ID", + "description" : "ID of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.clientId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.clientSecret", + "label" : "Client secret", + "description" : "Secret of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.clientSecret", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.tenantId", + "label" : "Tenant ID", + "description" : "ID of a Microsoft Entra tenant. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.tenantId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.authorityHost", + "label" : "Authority host", + "description" : "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.authorityHost", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] }, "type" : "String" }, { @@ -342,9 +496,9 @@ }, "type" : "String" }, { - "id" : "provider.azureOpenAi.endpoint", - "label" : "Endpoint", - "description" : "Specify Azure OpenAI endpoint. Details in the documentation.", + "id" : "provider.googleGenAi.projectId", + "label" : "Project ID", + "description" : "Specify Google Cloud project ID", "optional" : false, "constraints" : { "notEmpty" : true @@ -352,41 +506,19 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.endpoint", + "name" : "provider.googleGenAi.projectId", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "azureOpenAi", + "equals" : "googleGenAi", "type" : "simple" }, "type" : "String" }, { - "id" : "provider.azureOpenAi.authentication.type", - "label" : "Authentication", - "description" : "Specify the Azure OpenAI authentication strategy.", - "value" : "apiKey", - "group" : "provider", - "binding" : { - "name" : "provider.azureOpenAi.authentication.type", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "type" : "Dropdown", - "choices" : [ { - "name" : "API key", - "value" : "apiKey" - }, { - "name" : "Client credentials", - "value" : "clientCredentials" - } ] - }, { - "id" : "provider.azureOpenAi.authentication.apiKey", - "label" : "API key", + "id" : "provider.googleGenAi.region", + "label" : "Region", + "description" : "Specify the region where AI inference should take place", "optional" : false, "constraints" : { "notEmpty" : true @@ -394,51 +526,42 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.apiKey", + "name" : "provider.googleGenAi.region", "type" : "zeebe:input" }, "condition" : { - "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "apiKey", - "type" : "simple" - }, { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - } ] + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" }, "type" : "String" }, { - "id" : "provider.azureOpenAi.authentication.clientId", - "label" : "Client ID", - "description" : "ID of a Microsoft Entra application", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", + "id" : "provider.googleGenAi.authentication.type", + "label" : "Authentication", + "description" : "Specify the Google Vertex AI authentication strategy.", + "value" : "serviceAccountCredentials", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.clientId", + "name" : "provider.googleGenAi.authentication.type", "type" : "zeebe:input" }, "condition" : { - "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "clientCredentials", - "type" : "simple" - }, { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - } ] + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" }, - "type" : "String" + "type" : "Dropdown", + "choices" : [ { + "name" : "Service account credentials", + "value" : "serviceAccountCredentials" + }, { + "name" : "Application default credentials (Hybrid/Self-Managed only)", + "value" : "applicationDefaultCredentials" + } ] }, { - "id" : "provider.azureOpenAi.authentication.clientSecret", - "label" : "Client secret", - "description" : "Secret of a Microsoft Entra application", + "id" : "provider.googleGenAi.authentication.jsonKey", + "label" : "JSON key of the service account", + "description" : "This is the key of the service account in JSON format.", "optional" : false, "constraints" : { "notEmpty" : true @@ -446,154 +569,167 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.clientSecret", + "name" : "provider.googleGenAi.authentication.jsonKey", "type" : "zeebe:input" }, "condition" : { "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "clientCredentials", + "property" : "provider.googleGenAi.authentication.type", + "equals" : "serviceAccountCredentials", "type" : "simple" }, { "property" : "provider.type", - "equals" : "azureOpenAi", + "equals" : "googleGenAi", "type" : "simple" } ] }, "type" : "String" }, { - "id" : "provider.azureOpenAi.authentication.tenantId", - "label" : "Tenant ID", - "description" : "ID of a Microsoft Entra tenant. Details in the documentation.", + "id" : "provider.googleGenAi.backend", + "label" : "Backend", + "description" : "Specify the Google GenAI backend to use.", "optional" : false, - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", + "value" : "vertex", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.tenantId", + "name" : "provider.googleGenAi.backend", "type" : "zeebe:input" }, "condition" : { - "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "clientCredentials", - "type" : "simple" - }, { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - } ] + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" }, - "type" : "String" + "type" : "Dropdown", + "choices" : [ { + "name" : "Vertex AI", + "value" : "vertex" + }, { + "name" : "Developer API (Google AI Studio)", + "value" : "developer-api" + } ] }, { - "id" : "provider.azureOpenAi.authentication.authorityHost", - "label" : "Authority host", - "description" : "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", - "optional" : true, - "feel" : "optional", + "id" : "provider.openai.backend", + "label" : "Backend", + "description" : "Specify the OpenAI backend to use.", + "optional" : false, + "value" : "openai", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.authorityHost", + "name" : "provider.openai.backend", "type" : "zeebe:input" }, "condition" : { - "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "clientCredentials", - "type" : "simple" - }, { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - } ] + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" }, - "type" : "String" + "type" : "Dropdown", + "choices" : [ { + "name" : "OpenAI", + "value" : "openai" + }, { + "name" : "Azure AI Foundry", + "value" : "foundry" + }, { + "name" : "Custom", + "value" : "custom" + } ] }, { - "id" : "provider.azureOpenAi.timeouts.timeout", - "label" : "Timeout", - "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", - "optional" : true, - "feel" : "optional", + "id" : "provider.openai.authentication.type", + "label" : "Authentication", + "description" : "Specify the OpenAI authentication strategy.", + "value" : "apiKey", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.timeouts.timeout", + "name" : "provider.openai.authentication.type", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "azureOpenAi", + "equals" : "openai", "type" : "simple" }, - "type" : "String" + "type" : "Dropdown", + "choices" : [ { + "name" : "API key", + "value" : "apiKey" + }, { + "name" : "Client credentials", + "value" : "clientCredentials" + } ] }, { - "id" : "provider.googleVertexAi.projectId", - "label" : "Project ID", - "description" : "Specify Google Cloud project ID", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, + "id" : "provider.openai.authentication.apiKey", + "label" : "API key", + "optional" : true, "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.googleVertexAi.projectId", + "name" : "provider.openai.authentication.apiKey", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "google-vertex-ai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { - "id" : "provider.googleVertexAi.region", - "label" : "Region", - "description" : "Specify the region where AI inference should take place", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, + "id" : "provider.openai.authentication.organizationId", + "label" : "Organization ID", + "description" : "For members of multiple organizations. Details in the documentation.", + "optional" : true, "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.googleVertexAi.region", + "name" : "provider.openai.authentication.organizationId", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "google-vertex-ai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { - "id" : "provider.googleVertexAi.authentication.type", - "label" : "Authentication", - "description" : "Specify the Google Vertex AI authentication strategy.", - "value" : "serviceAccountCredentials", + "id" : "provider.openai.authentication.projectId", + "label" : "Project ID", + "description" : "For accounts with multiple projects. Details in the documentation.", + "optional" : true, + "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.googleVertexAi.authentication.type", + "name" : "provider.openai.authentication.projectId", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "google-vertex-ai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, - "type" : "Dropdown", - "choices" : [ { - "name" : "Service account credentials", - "value" : "serviceAccountCredentials" - }, { - "name" : "Application default credentials (Hybrid/Self-Managed only)", - "value" : "applicationDefaultCredentials" - } ] + "type" : "String" }, { - "id" : "provider.googleVertexAi.authentication.jsonKey", - "label" : "JSON key of the service account", - "description" : "This is the key of the service account in JSON format.", + "id" : "provider.openai.authentication.clientId", + "label" : "Client ID", + "description" : "ID of a Microsoft Entra application", "optional" : false, "constraints" : { "notEmpty" : true @@ -601,24 +737,25 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.googleVertexAi.authentication.jsonKey", + "name" : "provider.openai.authentication.clientId", "type" : "zeebe:input" }, "condition" : { "allMatch" : [ { - "property" : "provider.googleVertexAi.authentication.type", - "equals" : "serviceAccountCredentials", + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", "type" : "simple" }, { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "openai", "type" : "simple" } ] }, "type" : "String" }, { - "id" : "provider.openai.authentication.apiKey", - "label" : "OpenAI API key", + "id" : "provider.openai.authentication.clientSecret", + "label" : "Client secret", + "description" : "Secret of a Microsoft Entra application", "optional" : false, "constraints" : { "notEmpty" : true @@ -626,47 +763,68 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.openai.authentication.apiKey", + "name" : "provider.openai.authentication.clientSecret", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "openai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { - "id" : "provider.openai.authentication.organizationId", - "label" : "Organization ID", - "description" : "For members of multiple organizations. Details in the documentation.", - "optional" : true, + "id" : "provider.openai.authentication.tenantId", + "label" : "Tenant ID", + "description" : "ID of a Microsoft Entra tenant. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.openai.authentication.organizationId", + "name" : "provider.openai.authentication.tenantId", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "openai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { - "id" : "provider.openai.authentication.projectId", - "label" : "Project ID", - "description" : "For accounts with multiple projects. Details in the documentation.", + "id" : "provider.openai.authentication.authorityHost", + "label" : "Authority host", + "description" : "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", "optional" : true, "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.openai.authentication.projectId", + "name" : "provider.openai.authentication.authorityHost", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "openai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { @@ -712,7 +870,7 @@ } ] }, { "id" : "provider.openai.endpoint", - "label" : "Custom API endpoint", + "label" : "Endpoint", "description" : "Optional. Override the default OpenAI base URL (e.g. for an OpenAI proxy or gateway). Leave blank to use the SDK default.", "optional" : true, "feel" : "optional", @@ -728,90 +886,36 @@ }, "type" : "String" }, { - "id" : "provider.openaiCompatible.endpoint", - "label" : "API endpoint", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", - "group" : "provider", - "binding" : { - "name" : "provider.openaiCompatible.endpoint", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "Specify an endpoint to use the connector with an OpenAI compatible API. ", - "type" : "String" - }, { - "id" : "provider.openaiCompatible.authentication.apiKey", - "label" : "API key", - "optional" : true, - "feel" : "optional", - "group" : "provider", - "binding" : { - "name" : "provider.openaiCompatible.authentication.apiKey", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "Leave blank if using HTTP headers for authentication.
        If an Authorization header is specified in the headers, then the API key is ignored.", - "type" : "String" - }, { - "id" : "provider.openaiCompatible.headers", + "id" : "provider.openai.headers", "label" : "Headers", "description" : "Map of HTTP headers to add to the request.", "optional" : true, "feel" : "required", "group" : "provider", "binding" : { - "name" : "provider.openaiCompatible.headers", + "name" : "provider.openai.headers", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "openaiCompatible", + "equals" : "openai", "type" : "simple" }, "type" : "String" }, { - "id" : "provider.openaiCompatible.queryParameters", + "id" : "provider.openai.queryParameters", "label" : "Query Parameters", "description" : "Map of query parameters to add to the request URL.", "optional" : true, "feel" : "required", "group" : "provider", "binding" : { - "name" : "provider.openaiCompatible.queryParameters", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "type" : "String" - }, { - "id" : "provider.openaiCompatible.timeouts.timeout", - "label" : "Timeout", - "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", - "optional" : true, - "feel" : "optional", - "group" : "provider", - "binding" : { - "name" : "provider.openaiCompatible.timeouts.timeout", + "name" : "provider.openai.queryParameters", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "openaiCompatible", + "equals" : "openai", "type" : "simple" }, "type" : "String" @@ -977,78 +1081,7 @@ "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.azureOpenAi.model.deploymentName", - "label" : "Model deployment name", - "description" : "Specify the model deployment name. Details in the documentation.", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", - "group" : "model", - "binding" : { - "name" : "provider.azureOpenAi.model.deploymentName", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "type" : "String" - }, { - "id" : "provider.azureOpenAi.model.parameters.maxTokens", - "label" : "Maximum tokens", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.azureOpenAi.model.parameters.maxTokens", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "tooltip" : "The maximum number of tokens per request to generate before stopping.

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.azureOpenAi.model.parameters.temperature", - "label" : "Temperature", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.azureOpenAi.model.parameters.temperature", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "tooltip" : "Floating point number between 0 and 2. The higher the number, the more randomness will be injected into the response.

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.azureOpenAi.model.parameters.topP", - "label" : "top P", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.azureOpenAi.model.parameters.topP", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.googleVertexAi.model.model", + "id" : "provider.googleGenAi.model.model", "label" : "Model", "description" : "Specify the model ID. Details in the documentation.", "optional" : false, @@ -1058,79 +1091,79 @@ "feel" : "optional", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.model", + "name" : "provider.googleGenAi.model.model", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "type" : "String" }, { - "id" : "provider.googleVertexAi.model.parameters.maxOutputTokens", + "id" : "provider.googleGenAi.model.parameters.maxOutputTokens", "label" : "Maximum output tokens", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.parameters.maxOutputTokens", + "name" : "provider.googleGenAi.model.parameters.maxOutputTokens", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "tooltip" : "Maximum number of tokens that can be generated in the response.

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.googleVertexAi.model.parameters.temperature", + "id" : "provider.googleGenAi.model.parameters.temperature", "label" : "Temperature", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.parameters.temperature", + "name" : "provider.googleGenAi.model.parameters.temperature", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "tooltip" : "Controls the degree of randomness in token selection.

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.googleVertexAi.model.parameters.topP", + "id" : "provider.googleGenAi.model.parameters.topP", "label" : "top P", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.parameters.topP", + "name" : "provider.googleGenAi.model.parameters.topP", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.googleVertexAi.model.parameters.topK", + "id" : "provider.googleGenAi.model.parameters.topK", "label" : "top K", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.parameters.topK", + "name" : "provider.googleGenAi.model.parameters.topK", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "tooltip" : "Integer greater than 0. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", @@ -1208,91 +1241,19 @@ "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.openaiCompatible.model.model", - "label" : "Model", - "description" : "Specify the model ID. Details in the documentation.", - "optional" : false, - "value" : "gpt-4o", - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", - "group" : "model", - "binding" : { - "name" : "provider.openaiCompatible.model.model", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "type" : "String" - }, { - "id" : "provider.openaiCompatible.model.parameters.maxCompletionTokens", - "label" : "Maximum completion tokens", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.openaiCompatible.model.parameters.maxCompletionTokens", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "The maximum number of tokens per request to generate before stopping.

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.openaiCompatible.model.parameters.temperature", - "label" : "Temperature", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.openaiCompatible.model.parameters.temperature", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "Floating point number between 0 and 2. The higher the number, the more randomness will be injected into the response.

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.openaiCompatible.model.parameters.topP", - "label" : "top P", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.openaiCompatible.model.parameters.topP", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.openaiCompatible.model.parameters.customParameters", + "id" : "provider.openai.model.parameters.customParameters", "label" : "Custom parameters", "description" : "Map of additional request parameters to include.", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.openaiCompatible.model.parameters.customParameters", + "name" : "provider.openai.model.parameters.customParameters", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "openaiCompatible", + "equals" : "openai", "type" : "simple" }, "type" : "String" @@ -1749,7 +1710,7 @@ "id" : "version", "label" : "Version", "description" : "Version of the element template", - "value" : "11", + "value" : "12", "group" : "connector", "binding" : { "key" : "elementTemplateVersion", diff --git a/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-outbound-connector-hybrid.json b/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-outbound-connector-hybrid.json index b85ec062563..030e626b39a 100644 --- a/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-outbound-connector-hybrid.json +++ b/connectors/agentic-ai/element-templates/hybrid/agenticai-aiagent-outbound-connector-hybrid.json @@ -5,7 +5,7 @@ "description" : "Execute a single AI-powered action with tool calling capabilities", "keywords" : [ "AI", "AI Agent", "agentic orchestration" ], "documentationRef" : "https://docs.camunda.io/docs/8.10/components/connectors/out-of-the-box-connectors/agentic-ai-aiagent-task/", - "version" : 11, + "version" : 12, "category" : { "id" : "connectors", "name" : "Connectors" @@ -97,17 +97,11 @@ "name" : "AWS Bedrock", "value" : "bedrock" }, { - "name" : "Azure OpenAI", - "value" : "azureOpenAi" - }, { - "name" : "Google Vertex AI", - "value" : "google-vertex-ai" + "name" : "Google GenAI", + "value" : "googleGenAi" }, { "name" : "OpenAI", "value" : "openai" - }, { - "name" : "OpenAI Compatible", - "value" : "openaiCompatible" } ] }, { "id" : "provider.anthropic.endpoint", @@ -126,6 +120,59 @@ "type" : "simple" }, "type" : "String" + }, { + "id" : "provider.anthropic.backend", + "label" : "Backend", + "description" : "Specify the Anthropic backend to use.", + "optional" : false, + "value" : "direct", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.backend", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Direct (Anthropic API)", + "value" : "direct" + }, { + "name" : "AWS Bedrock", + "value" : "bedrock" + }, { + "name" : "Google Vertex AI", + "value" : "vertex" + }, { + "name" : "Azure AI Foundry", + "value" : "foundry" + } ] + }, { + "id" : "provider.anthropic.authentication.type", + "label" : "Authentication", + "description" : "Specify the Anthropic authentication strategy.", + "value" : "apiKey", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "API key", + "value" : "apiKey" + }, { + "name" : "Client credentials", + "value" : "clientCredentials" + } ] }, { "id" : "provider.anthropic.authentication.apiKey", "label" : "Anthropic API key", @@ -140,9 +187,116 @@ "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "anthropic", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.clientId", + "label" : "Client ID", + "description" : "ID of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.clientId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.clientSecret", + "label" : "Client secret", + "description" : "Secret of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.clientSecret", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.tenantId", + "label" : "Tenant ID", + "description" : "ID of a Microsoft Entra tenant. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.tenantId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.authorityHost", + "label" : "Authority host", + "description" : "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.authorityHost", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] }, "type" : "String" }, { @@ -321,9 +475,9 @@ }, "type" : "String" }, { - "id" : "provider.azureOpenAi.endpoint", - "label" : "Endpoint", - "description" : "Specify Azure OpenAI endpoint. Details in the documentation.", + "id" : "provider.googleGenAi.projectId", + "label" : "Project ID", + "description" : "Specify Google Cloud project ID", "optional" : false, "constraints" : { "notEmpty" : true @@ -331,41 +485,19 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.endpoint", + "name" : "provider.googleGenAi.projectId", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "azureOpenAi", + "equals" : "googleGenAi", "type" : "simple" }, "type" : "String" }, { - "id" : "provider.azureOpenAi.authentication.type", - "label" : "Authentication", - "description" : "Specify the Azure OpenAI authentication strategy.", - "value" : "apiKey", - "group" : "provider", - "binding" : { - "name" : "provider.azureOpenAi.authentication.type", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "type" : "Dropdown", - "choices" : [ { - "name" : "API key", - "value" : "apiKey" - }, { - "name" : "Client credentials", - "value" : "clientCredentials" - } ] - }, { - "id" : "provider.azureOpenAi.authentication.apiKey", - "label" : "API key", + "id" : "provider.googleGenAi.region", + "label" : "Region", + "description" : "Specify the region where AI inference should take place", "optional" : false, "constraints" : { "notEmpty" : true @@ -373,51 +505,42 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.apiKey", + "name" : "provider.googleGenAi.region", "type" : "zeebe:input" }, "condition" : { - "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "apiKey", - "type" : "simple" - }, { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - } ] + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" }, "type" : "String" }, { - "id" : "provider.azureOpenAi.authentication.clientId", - "label" : "Client ID", - "description" : "ID of a Microsoft Entra application", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", + "id" : "provider.googleGenAi.authentication.type", + "label" : "Authentication", + "description" : "Specify the Google Vertex AI authentication strategy.", + "value" : "serviceAccountCredentials", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.clientId", + "name" : "provider.googleGenAi.authentication.type", "type" : "zeebe:input" }, "condition" : { - "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "clientCredentials", - "type" : "simple" - }, { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - } ] + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" }, - "type" : "String" + "type" : "Dropdown", + "choices" : [ { + "name" : "Service account credentials", + "value" : "serviceAccountCredentials" + }, { + "name" : "Application default credentials (Hybrid/Self-Managed only)", + "value" : "applicationDefaultCredentials" + } ] }, { - "id" : "provider.azureOpenAi.authentication.clientSecret", - "label" : "Client secret", - "description" : "Secret of a Microsoft Entra application", + "id" : "provider.googleGenAi.authentication.jsonKey", + "label" : "JSON key of the service account", + "description" : "This is the key of the service account in JSON format.", "optional" : false, "constraints" : { "notEmpty" : true @@ -425,154 +548,167 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.clientSecret", + "name" : "provider.googleGenAi.authentication.jsonKey", "type" : "zeebe:input" }, "condition" : { "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "clientCredentials", + "property" : "provider.googleGenAi.authentication.type", + "equals" : "serviceAccountCredentials", "type" : "simple" }, { "property" : "provider.type", - "equals" : "azureOpenAi", + "equals" : "googleGenAi", "type" : "simple" } ] }, "type" : "String" }, { - "id" : "provider.azureOpenAi.authentication.tenantId", - "label" : "Tenant ID", - "description" : "ID of a Microsoft Entra tenant. Details in the documentation.", + "id" : "provider.googleGenAi.backend", + "label" : "Backend", + "description" : "Specify the Google GenAI backend to use.", "optional" : false, - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", + "value" : "vertex", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.tenantId", + "name" : "provider.googleGenAi.backend", "type" : "zeebe:input" }, "condition" : { - "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "clientCredentials", - "type" : "simple" - }, { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - } ] + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" }, - "type" : "String" + "type" : "Dropdown", + "choices" : [ { + "name" : "Vertex AI", + "value" : "vertex" + }, { + "name" : "Developer API (Google AI Studio)", + "value" : "developer-api" + } ] }, { - "id" : "provider.azureOpenAi.authentication.authorityHost", - "label" : "Authority host", - "description" : "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", - "optional" : true, - "feel" : "optional", + "id" : "provider.openai.backend", + "label" : "Backend", + "description" : "Specify the OpenAI backend to use.", + "optional" : false, + "value" : "openai", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.authentication.authorityHost", + "name" : "provider.openai.backend", "type" : "zeebe:input" }, "condition" : { - "allMatch" : [ { - "property" : "provider.azureOpenAi.authentication.type", - "equals" : "clientCredentials", - "type" : "simple" - }, { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - } ] + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" }, - "type" : "String" + "type" : "Dropdown", + "choices" : [ { + "name" : "OpenAI", + "value" : "openai" + }, { + "name" : "Azure AI Foundry", + "value" : "foundry" + }, { + "name" : "Custom", + "value" : "custom" + } ] }, { - "id" : "provider.azureOpenAi.timeouts.timeout", - "label" : "Timeout", - "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", - "optional" : true, - "feel" : "optional", + "id" : "provider.openai.authentication.type", + "label" : "Authentication", + "description" : "Specify the OpenAI authentication strategy.", + "value" : "apiKey", "group" : "provider", "binding" : { - "name" : "provider.azureOpenAi.timeouts.timeout", + "name" : "provider.openai.authentication.type", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "azureOpenAi", + "equals" : "openai", "type" : "simple" }, - "type" : "String" + "type" : "Dropdown", + "choices" : [ { + "name" : "API key", + "value" : "apiKey" + }, { + "name" : "Client credentials", + "value" : "clientCredentials" + } ] }, { - "id" : "provider.googleVertexAi.projectId", - "label" : "Project ID", - "description" : "Specify Google Cloud project ID", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, + "id" : "provider.openai.authentication.apiKey", + "label" : "API key", + "optional" : true, "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.googleVertexAi.projectId", + "name" : "provider.openai.authentication.apiKey", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "google-vertex-ai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { - "id" : "provider.googleVertexAi.region", - "label" : "Region", - "description" : "Specify the region where AI inference should take place", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, + "id" : "provider.openai.authentication.organizationId", + "label" : "Organization ID", + "description" : "For members of multiple organizations. Details in the documentation.", + "optional" : true, "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.googleVertexAi.region", + "name" : "provider.openai.authentication.organizationId", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "google-vertex-ai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { - "id" : "provider.googleVertexAi.authentication.type", - "label" : "Authentication", - "description" : "Specify the Google Vertex AI authentication strategy.", - "value" : "serviceAccountCredentials", + "id" : "provider.openai.authentication.projectId", + "label" : "Project ID", + "description" : "For accounts with multiple projects. Details in the documentation.", + "optional" : true, + "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.googleVertexAi.authentication.type", + "name" : "provider.openai.authentication.projectId", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "google-vertex-ai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, - "type" : "Dropdown", - "choices" : [ { - "name" : "Service account credentials", - "value" : "serviceAccountCredentials" - }, { - "name" : "Application default credentials (Hybrid/Self-Managed only)", - "value" : "applicationDefaultCredentials" - } ] + "type" : "String" }, { - "id" : "provider.googleVertexAi.authentication.jsonKey", - "label" : "JSON key of the service account", - "description" : "This is the key of the service account in JSON format.", + "id" : "provider.openai.authentication.clientId", + "label" : "Client ID", + "description" : "ID of a Microsoft Entra application", "optional" : false, "constraints" : { "notEmpty" : true @@ -580,24 +716,25 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.googleVertexAi.authentication.jsonKey", + "name" : "provider.openai.authentication.clientId", "type" : "zeebe:input" }, "condition" : { "allMatch" : [ { - "property" : "provider.googleVertexAi.authentication.type", - "equals" : "serviceAccountCredentials", + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", "type" : "simple" }, { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "openai", "type" : "simple" } ] }, "type" : "String" }, { - "id" : "provider.openai.authentication.apiKey", - "label" : "OpenAI API key", + "id" : "provider.openai.authentication.clientSecret", + "label" : "Client secret", + "description" : "Secret of a Microsoft Entra application", "optional" : false, "constraints" : { "notEmpty" : true @@ -605,47 +742,68 @@ "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.openai.authentication.apiKey", + "name" : "provider.openai.authentication.clientSecret", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "openai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { - "id" : "provider.openai.authentication.organizationId", - "label" : "Organization ID", - "description" : "For members of multiple organizations. Details in the documentation.", - "optional" : true, + "id" : "provider.openai.authentication.tenantId", + "label" : "Tenant ID", + "description" : "ID of a Microsoft Entra tenant. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.openai.authentication.organizationId", + "name" : "provider.openai.authentication.tenantId", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "openai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { - "id" : "provider.openai.authentication.projectId", - "label" : "Project ID", - "description" : "For accounts with multiple projects. Details in the documentation.", + "id" : "provider.openai.authentication.authorityHost", + "label" : "Authority host", + "description" : "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", "optional" : true, "feel" : "optional", "group" : "provider", "binding" : { - "name" : "provider.openai.authentication.projectId", + "name" : "provider.openai.authentication.authorityHost", "type" : "zeebe:input" }, "condition" : { - "property" : "provider.type", - "equals" : "openai", - "type" : "simple" + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] }, "type" : "String" }, { @@ -691,7 +849,7 @@ } ] }, { "id" : "provider.openai.endpoint", - "label" : "Custom API endpoint", + "label" : "Endpoint", "description" : "Optional. Override the default OpenAI base URL (e.g. for an OpenAI proxy or gateway). Leave blank to use the SDK default.", "optional" : true, "feel" : "optional", @@ -707,90 +865,36 @@ }, "type" : "String" }, { - "id" : "provider.openaiCompatible.endpoint", - "label" : "API endpoint", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", - "group" : "provider", - "binding" : { - "name" : "provider.openaiCompatible.endpoint", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "Specify an endpoint to use the connector with an OpenAI compatible API. ", - "type" : "String" - }, { - "id" : "provider.openaiCompatible.authentication.apiKey", - "label" : "API key", - "optional" : true, - "feel" : "optional", - "group" : "provider", - "binding" : { - "name" : "provider.openaiCompatible.authentication.apiKey", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "Leave blank if using HTTP headers for authentication.
        If an Authorization header is specified in the headers, then the API key is ignored.", - "type" : "String" - }, { - "id" : "provider.openaiCompatible.headers", + "id" : "provider.openai.headers", "label" : "Headers", "description" : "Map of HTTP headers to add to the request.", "optional" : true, "feel" : "required", "group" : "provider", "binding" : { - "name" : "provider.openaiCompatible.headers", + "name" : "provider.openai.headers", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "openaiCompatible", + "equals" : "openai", "type" : "simple" }, "type" : "String" }, { - "id" : "provider.openaiCompatible.queryParameters", + "id" : "provider.openai.queryParameters", "label" : "Query Parameters", "description" : "Map of query parameters to add to the request URL.", "optional" : true, "feel" : "required", "group" : "provider", "binding" : { - "name" : "provider.openaiCompatible.queryParameters", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "type" : "String" - }, { - "id" : "provider.openaiCompatible.timeouts.timeout", - "label" : "Timeout", - "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", - "optional" : true, - "feel" : "optional", - "group" : "provider", - "binding" : { - "name" : "provider.openaiCompatible.timeouts.timeout", + "name" : "provider.openai.queryParameters", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "openaiCompatible", + "equals" : "openai", "type" : "simple" }, "type" : "String" @@ -956,78 +1060,7 @@ "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.azureOpenAi.model.deploymentName", - "label" : "Model deployment name", - "description" : "Specify the model deployment name. Details in the documentation.", - "optional" : false, - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", - "group" : "model", - "binding" : { - "name" : "provider.azureOpenAi.model.deploymentName", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "type" : "String" - }, { - "id" : "provider.azureOpenAi.model.parameters.maxTokens", - "label" : "Maximum tokens", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.azureOpenAi.model.parameters.maxTokens", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "tooltip" : "The maximum number of tokens per request to generate before stopping.

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.azureOpenAi.model.parameters.temperature", - "label" : "Temperature", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.azureOpenAi.model.parameters.temperature", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "tooltip" : "Floating point number between 0 and 2. The higher the number, the more randomness will be injected into the response.

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.azureOpenAi.model.parameters.topP", - "label" : "top P", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.azureOpenAi.model.parameters.topP", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "azureOpenAi", - "type" : "simple" - }, - "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.googleVertexAi.model.model", + "id" : "provider.googleGenAi.model.model", "label" : "Model", "description" : "Specify the model ID. Details in the documentation.", "optional" : false, @@ -1037,79 +1070,79 @@ "feel" : "optional", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.model", + "name" : "provider.googleGenAi.model.model", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "type" : "String" }, { - "id" : "provider.googleVertexAi.model.parameters.maxOutputTokens", + "id" : "provider.googleGenAi.model.parameters.maxOutputTokens", "label" : "Maximum output tokens", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.parameters.maxOutputTokens", + "name" : "provider.googleGenAi.model.parameters.maxOutputTokens", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "tooltip" : "Maximum number of tokens that can be generated in the response.

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.googleVertexAi.model.parameters.temperature", + "id" : "provider.googleGenAi.model.parameters.temperature", "label" : "Temperature", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.parameters.temperature", + "name" : "provider.googleGenAi.model.parameters.temperature", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "tooltip" : "Controls the degree of randomness in token selection.

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.googleVertexAi.model.parameters.topP", + "id" : "provider.googleGenAi.model.parameters.topP", "label" : "top P", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.parameters.topP", + "name" : "provider.googleGenAi.model.parameters.topP", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.googleVertexAi.model.parameters.topK", + "id" : "provider.googleGenAi.model.parameters.topK", "label" : "top K", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.googleVertexAi.model.parameters.topK", + "name" : "provider.googleGenAi.model.parameters.topK", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "google-vertex-ai", + "equals" : "googleGenAi", "type" : "simple" }, "tooltip" : "Integer greater than 0. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", @@ -1187,91 +1220,19 @@ "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", "type" : "Number" }, { - "id" : "provider.openaiCompatible.model.model", - "label" : "Model", - "description" : "Specify the model ID. Details in the documentation.", - "optional" : false, - "value" : "gpt-4o", - "constraints" : { - "notEmpty" : true - }, - "feel" : "optional", - "group" : "model", - "binding" : { - "name" : "provider.openaiCompatible.model.model", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "type" : "String" - }, { - "id" : "provider.openaiCompatible.model.parameters.maxCompletionTokens", - "label" : "Maximum completion tokens", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.openaiCompatible.model.parameters.maxCompletionTokens", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "The maximum number of tokens per request to generate before stopping.

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.openaiCompatible.model.parameters.temperature", - "label" : "Temperature", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.openaiCompatible.model.parameters.temperature", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "Floating point number between 0 and 2. The higher the number, the more randomness will be injected into the response.

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.openaiCompatible.model.parameters.topP", - "label" : "top P", - "optional" : true, - "feel" : "required", - "group" : "model", - "binding" : { - "name" : "provider.openaiCompatible.model.parameters.topP", - "type" : "zeebe:input" - }, - "condition" : { - "property" : "provider.type", - "equals" : "openaiCompatible", - "type" : "simple" - }, - "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", - "type" : "Number" - }, { - "id" : "provider.openaiCompatible.model.parameters.customParameters", + "id" : "provider.openai.model.parameters.customParameters", "label" : "Custom parameters", "description" : "Map of additional request parameters to include.", "optional" : true, "feel" : "required", "group" : "model", "binding" : { - "name" : "provider.openaiCompatible.model.parameters.customParameters", + "name" : "provider.openai.model.parameters.customParameters", "type" : "zeebe:input" }, "condition" : { "property" : "provider.type", - "equals" : "openaiCompatible", + "equals" : "openai", "type" : "simple" }, "type" : "String" @@ -1723,7 +1684,7 @@ "id" : "version", "label" : "Version", "description" : "Version of the element template", - "value" : "11", + "value" : "12", "group" : "connector", "binding" : { "key" : "elementTemplateVersion", diff --git a/connectors/agentic-ai/element-templates/versioned/agenticai-aiagent-job-worker-11.json b/connectors/agentic-ai/element-templates/versioned/agenticai-aiagent-job-worker-11.json new file mode 100644 index 00000000000..b7d505c44ad --- /dev/null +++ b/connectors/agentic-ai/element-templates/versioned/agenticai-aiagent-job-worker-11.json @@ -0,0 +1,1700 @@ +{ + "$schema" : "https://unpkg.com/@camunda/zeebe-element-templates-json-schema/resources/schema.json", + "name" : "AI Agent Sub-process", + "id" : "io.camunda.connectors.agenticai.aiagent.jobworker.v1", + "description" : "Run a multi-step AI reasoning loop with dynamic tool selection", + "keywords" : [ "AI", "AI Agent", "agentic orchestration" ], + "documentationRef" : "https://docs.camunda.io/docs/8.10/components/connectors/out-of-the-box-connectors/agentic-ai-aiagent-subprocess/", + "version" : 11, + "category" : { + "id" : "connectors", + "name" : "Connectors" + }, + "appliesTo" : [ "bpmn:SubProcess" ], + "elementType" : { + "value" : "bpmn:AdHocSubProcess" + }, + "engines" : { + "camunda" : "^8.10" + }, + "groups" : [ { + "id" : "provider", + "label" : "Model provider", + "openByDefault" : false + }, { + "id" : "model", + "label" : "Model", + "openByDefault" : false + }, { + "id" : "systemPrompt", + "label" : "System prompt", + "tooltip" : "A system prompt is a set of foundational instructions given to a model before any user interaction begins. It defines the AI agent’s role, behavior, tone, and communication style, ensuring that responses remain consistent and aligned with the AI agent’s intended purpose. These instructions help shape how the model interprets and responds to user input throughout the conversation.", + "openByDefault" : false + }, { + "id" : "userPrompt", + "label" : "User prompt", + "tooltip" : "A user prompt is the message or question you give to the AI to start or continue a conversation. It tells the AI what you need, whether it's information, help with a task, or just a chat. The AI uses your prompt to understand how to respond.", + "openByDefault" : false + }, { + "id" : "tools", + "label" : "Tools", + "tooltip" : "Tools are optional features the AI Agent can use to perform specific tasks. Configure this if the agent should participate in a tools feedback loop.", + "openByDefault" : false + }, { + "id" : "memory", + "label" : "Memory", + "tooltip" : "Configuration of the Agent's short-term/conversational memory.", + "openByDefault" : false + }, { + "id" : "limits", + "label" : "Limits", + "openByDefault" : false + }, { + "id" : "events", + "label" : "Event handling", + "tooltip" : "Configure how event sub-process results are handled. Results are added as user messages to the running agent.", + "openByDefault" : false + }, { + "id" : "response", + "label" : "Response", + "tooltip" : "Configuration of the model response format and how to map the model response to the connector result.

        Depending on the selection, the model response will be available as response.responseText or response.responseJson.

        See documentation for details.", + "openByDefault" : false + }, { + "id" : "connector", + "label" : "Connector" + }, { + "id" : "output", + "label" : "Output mapping" + }, { + "id" : "error", + "label" : "Error handling" + }, { + "id" : "retries", + "label" : "Retries" + } ], + "properties" : [ { + "value" : "io.camunda.agenticai:aiagent-job-worker:1", + "binding" : { + "property" : "type", + "type" : "zeebe:taskDefinition" + }, + "type" : "Hidden" + }, { + "id" : "outputCollection", + "binding" : { + "property" : "outputCollection", + "type" : "zeebe:adHoc" + }, + "value" : "toolCallResults", + "type" : "Hidden" + }, { + "id" : "outputElement", + "binding" : { + "property" : "outputElement", + "type" : "zeebe:adHoc" + }, + "value" : "={\n id: toolCall._meta.id,\n name: toolCall._meta.name,\n content: toolCallResult\n}", + "type" : "Hidden" + }, { + "id" : "provider.type", + "label" : "Provider", + "description" : "Specify the LLM provider to use.", + "value" : "anthropic", + "group" : "provider", + "binding" : { + "name" : "provider.type", + "type" : "zeebe:input" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Anthropic", + "value" : "anthropic" + }, { + "name" : "AWS Bedrock", + "value" : "bedrock" + }, { + "name" : "Google GenAI", + "value" : "googleGenAi" + }, { + "name" : "OpenAI", + "value" : "openai" + } ] + }, { + "id" : "provider.anthropic.endpoint", + "label" : "Endpoint", + "description" : "Optional custom API endpoint", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.type", + "label" : "Authentication", + "description" : "Specify the Anthropic authentication strategy.", + "value" : "apiKey", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "API key", + "value" : "apiKey" + }, { + "name" : "Client credentials", + "value" : "clientCredentials" + } ] + }, { + "id" : "provider.anthropic.authentication.apiKey", + "label" : "Anthropic API key", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.apiKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.clientId", + "label" : "Client ID", + "description" : "ID of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.clientId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.clientSecret", + "label" : "Client secret", + "description" : "Secret of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.clientSecret", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.tenantId", + "label" : "Tenant ID", + "description" : "ID of a Microsoft Entra tenant. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.tenantId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.authorityHost", + "label" : "Authority host", + "description" : "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.authorityHost", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.timeouts.timeout", + "label" : "Timeout", + "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.timeouts.timeout", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.bedrock.region", + "label" : "Region", + "description" : "Specify the AWS region (example: eu-west-1)", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.region", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.bedrock.endpoint", + "label" : "Endpoint", + "description" : "Custom API endpoint for VPC/PrivateLink configurations, AWS GovCloud, or other non-standard deployments.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.bedrock.authentication.type", + "label" : "Authentication", + "description" : "Specify the AWS authentication strategy. Learn more at the documentation page", + "value" : "credentials", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Credentials", + "value" : "credentials" + }, { + "name" : "API Key", + "value" : "apiKey" + }, { + "name" : "Default Credentials Chain (Hybrid/Self-Managed only)", + "value" : "defaultCredentialsChain" + } ] + }, { + "id" : "provider.bedrock.authentication.accessKey", + "label" : "Access key", + "description" : "Provide an IAM access key tailored to a user, equipped with the necessary permissions", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.authentication.accessKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.bedrock.authentication.type", + "equals" : "credentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.bedrock.authentication.secretKey", + "label" : "Secret key", + "description" : "Provide the secret key for the IAM access key", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.authentication.secretKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.bedrock.authentication.type", + "equals" : "credentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.bedrock.authentication.apiKey", + "label" : "API Key", + "description" : "Provide an API Key with permissions to interact with your AWS Bedrock Instance", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.authentication.apiKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.bedrock.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.bedrock.timeouts.timeout", + "label" : "Timeout", + "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.timeouts.timeout", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.googleGenAi.projectId", + "label" : "Project ID", + "description" : "Specify Google Cloud project ID", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.googleGenAi.projectId", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.googleGenAi.region", + "label" : "Region", + "description" : "Specify the region where AI inference should take place", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.googleGenAi.region", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.googleGenAi.authentication.type", + "label" : "Authentication", + "description" : "Specify the Google Vertex AI authentication strategy.", + "value" : "serviceAccountCredentials", + "group" : "provider", + "binding" : { + "name" : "provider.googleGenAi.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Service account credentials", + "value" : "serviceAccountCredentials" + }, { + "name" : "Application default credentials (Hybrid/Self-Managed only)", + "value" : "applicationDefaultCredentials" + } ] + }, { + "id" : "provider.googleGenAi.authentication.jsonKey", + "label" : "JSON key of the service account", + "description" : "This is the key of the service account in JSON format.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.googleGenAi.authentication.jsonKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.googleGenAi.authentication.type", + "equals" : "serviceAccountCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.type", + "label" : "Authentication", + "description" : "Specify the OpenAI authentication strategy.", + "value" : "apiKey", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "API key", + "value" : "apiKey" + }, { + "name" : "Client credentials", + "value" : "clientCredentials" + } ] + }, { + "id" : "provider.openai.authentication.apiKey", + "label" : "API key", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.apiKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.organizationId", + "label" : "Organization ID", + "description" : "For members of multiple organizations. Details in the documentation.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.organizationId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.projectId", + "label" : "Project ID", + "description" : "For accounts with multiple projects. Details in the documentation.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.projectId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.clientId", + "label" : "Client ID", + "description" : "ID of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.clientId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.clientSecret", + "label" : "Client secret", + "description" : "Secret of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.clientSecret", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.tenantId", + "label" : "Tenant ID", + "description" : "ID of a Microsoft Entra tenant. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.tenantId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.authorityHost", + "label" : "Authority host", + "description" : "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.authorityHost", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.openai.timeouts.timeout", + "label" : "Timeout", + "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.timeouts.timeout", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openai.apiFamily", + "label" : "API", + "description" : "Which OpenAI API family to use. Chat Completions is the default for backward compatibility; Responses is the newer endpoint with structured input/output items.", + "optional" : true, + "value" : "completions", + "group" : "provider", + "binding" : { + "name" : "provider.openai.apiFamily", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Chat Completions (default)", + "value" : "completions" + }, { + "name" : "Responses", + "value" : "responses" + } ] + }, { + "id" : "provider.openai.endpoint", + "label" : "Endpoint", + "description" : "Optional. Override the default OpenAI base URL (e.g. for an OpenAI proxy or gateway). Leave blank to use the SDK default.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openai.headers", + "label" : "Headers", + "description" : "Map of HTTP headers to add to the request.", + "optional" : true, + "feel" : "required", + "group" : "provider", + "binding" : { + "name" : "provider.openai.headers", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openai.queryParameters", + "label" : "Query Parameters", + "description" : "Map of query parameters to add to the request URL.", + "optional" : true, + "feel" : "required", + "group" : "provider", + "binding" : { + "name" : "provider.openai.queryParameters", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.anthropic.model.model", + "label" : "Model", + "description" : "Specify the model ID. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.model", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "placeholder" : "claude-sonnet-4-6", + "type" : "String" + }, { + "id" : "provider.anthropic.model.parameters.maxTokens", + "label" : "Maximum tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.parameters.maxTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "tooltip" : "The maximum number of tokens per request to generate before stopping.

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.anthropic.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. The higher the number, the more randomness will be injected into the response.

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.anthropic.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.anthropic.model.parameters.topK", + "label" : "top K", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.parameters.topK", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "tooltip" : "Integer greater than 0. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.bedrock.model.model", + "label" : "Model", + "description" : "Specify an inference profile ID. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.bedrock.model.model", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "placeholder" : "global.anthropic.claude-sonnet-4-6", + "type" : "String" + }, { + "id" : "provider.bedrock.model.parameters.maxTokens", + "label" : "Maximum tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.bedrock.model.parameters.maxTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "tooltip" : "The maximum number of tokens per request to allow in the generated response.

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.bedrock.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.bedrock.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. The higher the number, the more randomness will be injected into the response.

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.bedrock.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.bedrock.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.googleGenAi.model.model", + "label" : "Model", + "description" : "Specify the model ID. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.googleGenAi.model.model", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.googleGenAi.model.parameters.maxOutputTokens", + "label" : "Maximum output tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.googleGenAi.model.parameters.maxOutputTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" + }, + "tooltip" : "Maximum number of tokens that can be generated in the response.

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.googleGenAi.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.googleGenAi.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" + }, + "tooltip" : "Controls the degree of randomness in token selection.

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.googleGenAi.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.googleGenAi.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.googleGenAi.model.parameters.topK", + "label" : "top K", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.googleGenAi.model.parameters.topK", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" + }, + "tooltip" : "Integer greater than 0. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openai.model.model", + "label" : "Model", + "description" : "Specify the model ID. Details in the documentation.", + "optional" : false, + "value" : "gpt-4o", + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.openai.model.model", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openai.model.parameters.maxCompletionTokens", + "label" : "Maximum completion tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openai.model.parameters.maxCompletionTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "tooltip" : "The maximum number of tokens per request to generate before stopping.

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openai.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openai.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 2. The higher the number, the more randomness will be injected into the response.

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openai.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openai.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openai.model.parameters.customParameters", + "label" : "Custom parameters", + "description" : "Map of additional request parameters to include.", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openai.model.parameters.customParameters", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.systemPrompt.prompt", + "label" : "System prompt", + "optional" : false, + "value" : "=\"You are **TaskAgent**, a helpful, generic chat agent that can handle a wide variety of customer requests using your own domain knowledge **and** any tools explicitly provided to you at runtime.\n\nIf tools are provided, you should prefer them instead of guessing an answer. You can call the same tool multiple times by providing different input values. Don't guess any tools which were not explicitly configured. If no tool matches the request, try to generate an answer. If you're not able to find a good answer, return with a message stating why you're not able to.\n\nWrap minimal, inspectable reasoning in *exactly* this XML template:\n\n\n…briefly state the customer’s need and current state…\n…list candidate tools, justify which you will call next and why…\n\n\nReveal **no** additional private reasoning outside these tags.\"", + "constraints" : { + "notEmpty" : true + }, + "feel" : "required", + "group" : "systemPrompt", + "binding" : { + "name" : "data.systemPrompt.prompt", + "type" : "zeebe:input" + }, + "type" : "Text" + }, { + "id" : "data.userPrompt.prompt", + "label" : "User prompt", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "required", + "group" : "userPrompt", + "binding" : { + "name" : "data.userPrompt.prompt", + "type" : "zeebe:input" + }, + "type" : "Text" + }, { + "id" : "data.userPrompt.documents", + "label" : "Documents", + "description" : "Documents to be included in the user prompt.", + "optional" : true, + "feel" : "required", + "group" : "userPrompt", + "binding" : { + "name" : "data.userPrompt.documents", + "type" : "zeebe:input" + }, + "tooltip" : "Referenced documents will be automatically added to the user prompt. See documentation for details and supported file types.", + "type" : "String" + }, { + "id" : "agentContext", + "label" : "Agent context", + "description" : "Initial agent context from previous interactions. Avoid reusing context variables across agents to prevent issues with stale data or tool access.", + "optional" : false, + "feel" : "required", + "group" : "memory", + "binding" : { + "name" : "agentContext", + "type" : "zeebe:input" + }, + "tooltip" : "The agent context variable containing all relevant data for the agent to support the feedback loop between user requests, tool calls and LLM responses. Make sure this variable points to the context variable which is returned from the agent response. See documentation for details.", + "type" : "Text" + }, { + "id" : "data.memory.storage.type", + "label" : "Memory storage type", + "description" : "Specify how to store the conversation memory.", + "value" : "in-process", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.type", + "type" : "zeebe:input" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "In Process (part of agent context)", + "value" : "in-process" + }, { + "name" : "Camunda Document Storage", + "value" : "camunda-document" + }, { + "name" : "AWS AgentCore Memory", + "value" : "aws-agentcore" + }, { + "name" : "Custom Implementation (Hybrid/Self-Managed only)", + "value" : "custom" + } ] + }, { + "id" : "data.memory.storage.timeToLive", + "label" : "Document TTL", + "description" : "How long to retain the conversation document as ISO-8601 duration (example: P14D).", + "optional" : true, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.timeToLive", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "camunda-document", + "type" : "simple" + }, + "tooltip" : "Will use the cluster default TTL (time-to-live) if not specified. Make sure to set this value to a reasonable duration matching your process lifecycle.", + "type" : "String" + }, { + "id" : "data.memory.storage.customProperties", + "label" : "Custom document properties", + "description" : "An optional map of custom properties to be stored with the conversation document.", + "optional" : true, + "feel" : "required", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.customProperties", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "camunda-document", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.region", + "label" : "Region", + "description" : "Specify the AWS region (example: us-east-1)", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.region", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.endpoint", + "label" : "Endpoint", + "description" : "Custom API endpoint for VPC/PrivateLink configurations, AWS GovCloud, or other non-standard deployments.", + "optional" : true, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.authentication.type", + "label" : "Authentication", + "description" : "Specify the AWS authentication strategy for AgentCore Memory access.", + "value" : "credentials", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Credentials", + "value" : "credentials" + }, { + "name" : "Default Credentials Chain (Hybrid/Self-Managed only)", + "value" : "defaultCredentialsChain" + } ] + }, { + "id" : "data.memory.storage.authentication.accessKey", + "label" : "Access key", + "description" : "Provide an IAM access key with permissions for bedrock-agentcore:CreateEvent and bedrock-agentcore:ListEvents", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.authentication.accessKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "data.memory.storage.authentication.type", + "equals" : "credentials", + "type" : "simple" + }, { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "data.memory.storage.authentication.secretKey", + "label" : "Secret key", + "description" : "Provide the secret key for the IAM access key", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.authentication.secretKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "data.memory.storage.authentication.type", + "equals" : "credentials", + "type" : "simple" + }, { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "data.memory.storage.memoryId", + "label" : "Memory ID", + "description" : "The ID of the pre-provisioned AgentCore Memory resource.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.memoryId", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.actorId", + "label" : "Actor ID", + "description" : "Identifier of the actor associated with events (e.g., end-user or agent/user combination).", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.actorId", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.storeType", + "label" : "Implementation type", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.storeType", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "custom", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.parameters", + "label" : "Parameters", + "description" : "Parameters for the custom memory storage implementation.", + "optional" : true, + "feel" : "required", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.parameters", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "custom", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.contextWindowSize", + "label" : "Context window size", + "description" : "Maximum number of recent conversation messages which are passed to the model.", + "optional" : false, + "value" : 20, + "feel" : "static", + "group" : "memory", + "binding" : { + "name" : "data.memory.contextWindowSize", + "type" : "zeebe:input" + }, + "tooltip" : "Use this to limit the number of messages which are sent to the model. The agent will only send the most recent messages up to the configured limit to the LLM. Older messages will be kept in the conversation store, but not sent to the model. See documentation for details.", + "type" : "Number" + }, { + "id" : "data.limits.maxModelCalls", + "label" : "Maximum model calls", + "description" : "Maximum number of calls to the model as a safety limit to prevent infinite loops.", + "optional" : false, + "value" : 10, + "feel" : "static", + "group" : "limits", + "binding" : { + "name" : "data.limits.maxModelCalls", + "type" : "zeebe:input" + }, + "type" : "Number" + }, { + "id" : "data.events.behavior", + "label" : "Event handling behavior", + "description" : "Behavior on completing an event sub-process.", + "optional" : false, + "value" : "WAIT_FOR_TOOL_CALL_RESULTS", + "constraints" : { + "notEmpty" : true + }, + "group" : "events", + "binding" : { + "name" : "data.events.behavior", + "type" : "zeebe:input" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Wait for tool call results", + "value" : "WAIT_FOR_TOOL_CALL_RESULTS" + }, { + "name" : "Cancel tool calls", + "value" : "INTERRUPT_TOOL_CALLS" + } ] + }, { + "id" : "data.response.format.type", + "label" : "Response format", + "description" : "Specify the response format. Support for JSON mode varies by provider.", + "value" : "text", + "group" : "response", + "binding" : { + "name" : "data.response.format.type", + "type" : "zeebe:input" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Text", + "value" : "text" + }, { + "name" : "JSON", + "value" : "json" + } ] + }, { + "id" : "data.response.format.parseJson", + "label" : "Parse text as JSON", + "description" : "Tries to parse the LLM response text as JSON object.", + "optional" : true, + "feel" : "static", + "group" : "response", + "binding" : { + "name" : "data.response.format.parseJson", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.response.format.type", + "equals" : "text", + "type" : "simple" + }, + "tooltip" : "Use this option in combination with models which don't support native JSON mode/structured tool calling (e.g. Anthropic). Make sure to instruct the model to return valid JSON in the system prompt. The parsed JSON will be available as response.responseJson.

        If parsing fails, null will be returned as JSON response, but the text content will still be available as response.responseText.", + "type" : "Boolean" + }, { + "id" : "data.response.format.schema", + "label" : "Response JSON schema", + "description" : "An optional response JSON Schema to instruct the model how to structure the JSON output.", + "optional" : true, + "feel" : "required", + "group" : "response", + "binding" : { + "name" : "data.response.format.schema", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.response.format.type", + "equals" : "json", + "type" : "simple" + }, + "tooltip" : "If supported by the model, the response will be structured according to the provided schema. A parsed version of the response will be available as response.responseJson.", + "type" : "String" + }, { + "id" : "data.response.format.schemaName", + "label" : "Response JSON schema name", + "description" : "An optional name for the response JSON Schema to make the model aware of the expected output.", + "optional" : true, + "value" : "Response", + "feel" : "optional", + "group" : "response", + "binding" : { + "name" : "data.response.format.schemaName", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.response.format.type", + "equals" : "json", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.response.includeAssistantMessage", + "label" : "Include assistant message", + "description" : "Include the full assistant message as part of the result object.", + "optional" : true, + "feel" : "static", + "group" : "response", + "binding" : { + "name" : "data.response.includeAssistantMessage", + "type" : "zeebe:input" + }, + "tooltip" : "In addition to the text content, the assistant message may include multiple additional content blocks and metadata (such as token usage). The message will be available as response.responseMessage.", + "type" : "Boolean" + }, { + "id" : "data.response.includeAgentContext", + "label" : "Include agent context", + "description" : "Include the agent context as part of the result object.", + "optional" : true, + "feel" : "static", + "group" : "response", + "binding" : { + "name" : "data.response.includeAgentContext", + "type" : "zeebe:input" + }, + "tooltip" : "Use this option if you need to re-inject the previous agent context into a future agent execution, for example when modeling a user feedback loop between an agent and a user task.", + "type" : "Boolean" + }, { + "id" : "version", + "label" : "Version", + "description" : "Version of the element template", + "value" : "11", + "group" : "connector", + "binding" : { + "key" : "elementTemplateVersion", + "type" : "zeebe:taskHeader" + }, + "type" : "Hidden" + }, { + "id" : "id", + "label" : "ID", + "description" : "ID of the element template", + "value" : "io.camunda.connectors.agenticai.aiagent.jobworker.v1", + "group" : "connector", + "binding" : { + "key" : "elementTemplateId", + "type" : "zeebe:taskHeader" + }, + "type" : "Hidden" + }, { + "id" : "resultVariable", + "label" : "Result variable", + "description" : "Name of variable to store the response in. Details in the documentation.", + "value" : "agent", + "group" : "output", + "binding" : { + "source" : "=agent", + "type" : "zeebe:output" + }, + "type" : "String" + }, { + "id" : "errorExpression", + "label" : "Error expression", + "description" : "Expression to handle errors. Details in the documentation.", + "feel" : "required", + "group" : "error", + "binding" : { + "key" : "errorExpression", + "type" : "zeebe:taskHeader" + }, + "type" : "Text" + }, { + "id" : "retryCount", + "label" : "Retries", + "description" : "Number of retries", + "value" : "3", + "feel" : "optional", + "group" : "retries", + "binding" : { + "property" : "retries", + "type" : "zeebe:taskDefinition" + }, + "type" : "String" + }, { + "id" : "retryBackoff", + "label" : "Retry backoff", + "description" : "ISO-8601 duration to wait between retries", + "value" : "PT0S", + "group" : "retries", + "binding" : { + "key" : "retryBackoff", + "type" : "zeebe:taskHeader" + }, + "type" : "String" + }, { + "binding" : { + "name" : "agent", + "type" : "zeebe:input" + }, + "type" : "Hidden" + } ], + "icon" : { + "contents" : "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMzIiIGhlaWdodD0iMzIiIHZpZXdCb3g9IjAgMCAzMiAzMiIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPGNpcmNsZSBjeD0iMTYiIGN5PSIxNiIgcj0iMTYiIGZpbGw9IiNBNTZFRkYiLz4KPG1hc2sgaWQ9InBhdGgtMi1vdXRzaWRlLTFfMTg1XzYiIG1hc2tVbml0cz0idXNlclNwYWNlT25Vc2UiIHg9IjQiIHk9IjQiIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgZmlsbD0iYmxhY2siPgo8cmVjdCBmaWxsPSJ3aGl0ZSIgeD0iNCIgeT0iNCIgd2lkdGg9IjI0IiBoZWlnaHQ9IjI0Ii8+CjxwYXRoIGZpbGwtcnVsZT0iZXZlbm9kZCIgY2xpcC1ydWxlPSJldmVub2RkIiBkPSJNMjAuMDEwNSAxMi4wOTg3QzE4LjQ5IDEwLjU4OTQgMTcuMTU5NCA4LjEwODE0IDE2LjE3OTkgNi4wMTEwM0MxNi4xNTIgNi4wMDQ1MSAxNi4xMTc2IDYgMTYuMDc5NCA2QzE2LjA0MTEgNiAxNi4wMDY2IDYuMDA0NTEgMTUuOTc4OCA2LjAxMTA0QzE0Ljk5OTQgOC4xMDgxNCAxMy42Njk3IDEwLjU4ODkgMTIuMTQ4MSAxMi4wOTgxQzEwLjYyNjkgMTMuNjA3MSA4LjEyNTY4IDE0LjkyNjQgNi4wMTE1NyAxNS44OTgxQzYuMDA0NzQgMTUuOTI2MSA2IDE1Ljk2MTEgNiAxNkM2IDE2LjAzODcgNi4wMDQ2OCAxNi4wNzM2IDYuMDExNDQgMTYuMTAxNEM4LjEyNTE5IDE3LjA3MjkgMTAuNjI2MiAxOC4zOTE5IDEyLjE0NzcgMTkuOTAxNkMxMy42Njk3IDIxLjQxMDcgMTQuOTk5NiAyMy44OTIgMTUuOTc5MSAyNS45ODlDMTYuMDA2OCAyNS45OTU2IDE2LjA0MTEgMjYgMTYuMDc5MyAyNkMxNi4xMTc1IDI2IDE2LjE1MTkgMjUuOTk1NCAxNi4xNzk2IDI1Ljk4OUMxNy4xNTkxIDIzLjg5MiAxOC40ODg4IDIxLjQxMSAyMC4wMDk5IDE5LjkwMjFNMjAuMDA5OSAxOS45MDIxQzIxLjUyNTMgMTguMzk4NyAyMy45NDY1IDE3LjA2NjkgMjUuOTkxNSAxNi4wODI0QzI1Ljk5NjUgMTYuMDU5MyAyNiAxNi4wMzEgMjYgMTUuOTk5N0MyNiAxNS45Njg0IDI1Ljk5NjUgMTUuOTQwMyAyNS45OTE1IDE1LjkxNzFDMjMuOTQ3NCAxNC45MzI3IDIxLjUyNTkgMTMuNjAxIDIwLjAxMDUgMTIuMDk4NyIvPgo8L21hc2s+CjxwYXRoIGZpbGwtcnVsZT0iZXZlbm9kZCIgY2xpcC1ydWxlPSJldmVub2RkIiBkPSJNMjAuMDEwNSAxMi4wOTg3QzE4LjQ5IDEwLjU4OTQgMTcuMTU5NCA4LjEwODE0IDE2LjE3OTkgNi4wMTEwM0MxNi4xNTIgNi4wMDQ1MSAxNi4xMTc2IDYgMTYuMDc5NCA2QzE2LjA0MTEgNiAxNi4wMDY2IDYuMDA0NTEgMTUuOTc4OCA2LjAxMTA0QzE0Ljk5OTQgOC4xMDgxNCAxMy42Njk3IDEwLjU4ODkgMTIuMTQ4MSAxMi4wOTgxQzEwLjYyNjkgMTMuNjA3MSA4LjEyNTY4IDE0LjkyNjQgNi4wMTE1NyAxNS44OTgxQzYuMDA0NzQgMTUuOTI2MSA2IDE1Ljk2MTEgNiAxNkM2IDE2LjAzODcgNi4wMDQ2OCAxNi4wNzM2IDYuMDExNDQgMTYuMTAxNEM4LjEyNTE5IDE3LjA3MjkgMTAuNjI2MiAxOC4zOTE5IDEyLjE0NzcgMTkuOTAxNkMxMy42Njk3IDIxLjQxMDcgMTQuOTk5NiAyMy44OTIgMTUuOTc5MSAyNS45ODlDMTYuMDA2OCAyNS45OTU2IDE2LjA0MTEgMjYgMTYuMDc5MyAyNkMxNi4xMTc1IDI2IDE2LjE1MTkgMjUuOTk1NCAxNi4xNzk2IDI1Ljk4OUMxNy4xNTkxIDIzLjg5MiAxOC40ODg4IDIxLjQxMSAyMC4wMDk5IDE5LjkwMjFNMjAuMDA5OSAxOS45MDIxQzIxLjUyNTMgMTguMzk4NyAyMy45NDY1IDE3LjA2NjkgMjUuOTkxNSAxNi4wODI0QzI1Ljk5NjUgMTYuMDU5MyAyNiAxNi4wMzEgMjYgMTUuOTk5N0MyNiAxNS45Njg0IDI1Ljk5NjUgMTUuOTQwMyAyNS45OTE1IDE1LjkxNzFDMjMuOTQ3NCAxNC45MzI3IDIxLjUyNTkgMTMuNjAxIDIwLjAxMDUgMTIuMDk4NyIgZmlsbD0id2hpdGUiLz4KPHBhdGggZmlsbC1ydWxlPSJldmVub2RkIiBjbGlwLXJ1bGU9ImV2ZW5vZGQiIGQ9Ik0yMC4wMTA1IDEyLjA5ODdDMTguNDkgMTAuNTg5NCAxNy4xNTk0IDguMTA4MTQgMTYuMTc5OSA2LjAxMTAzQzE2LjE1MiA2LjAwNDUxIDE2LjExNzYgNiAxNi4wNzk0IDZDMTYuMDQxMSA2IDE2LjAwNjYgNi4wMDQ1MSAxNS45Nzg4IDYuMDExMDRDMTQuOTk5NCA4LjEwODE0IDEzLjY2OTcgMTAuNTg4OSAxMi4xNDgxIDEyLjA5ODFDMTAuNjI2OSAxMy42MDcxIDguMTI1NjggMTQuOTI2NCA2LjAxMTU3IDE1Ljg5ODFDNi4wMDQ3NCAxNS45MjYxIDYgMTUuOTYxMSA2IDE2QzYgMTYuMDM4NyA2LjAwNDY4IDE2LjA3MzYgNi4wMTE0NCAxNi4xMDE0QzguMTI1MTkgMTcuMDcyOSAxMC42MjYyIDE4LjM5MTkgMTIuMTQ3NyAxOS45MDE2QzEzLjY2OTcgMjEuNDEwNyAxNC45OTk2IDIzLjg5MiAxNS45NzkxIDI1Ljk4OUMxNi4wMDY4IDI1Ljk5NTYgMTYuMDQxMSAyNiAxNi4wNzkzIDI2QzE2LjExNzUgMjYgMTYuMTUxOSAyNS45OTU0IDE2LjE3OTYgMjUuOTg5QzE3LjE1OTEgMjMuODkyIDE4LjQ4ODggMjEuNDExIDIwLjAwOTkgMTkuOTAyMU0yMC4wMDk5IDE5LjkwMjFDMjEuNTI1MyAxOC4zOTg3IDIzLjk0NjUgMTcuMDY2OSAyNS45OTE1IDE2LjA4MjRDMjUuOTk2NSAxNi4wNTkzIDI2IDE2LjAzMSAyNiAxNS45OTk3QzI2IDE1Ljk2ODQgMjUuOTk2NSAxNS45NDAzIDI1Ljk5MTUgMTUuOTE3MUMyMy45NDc0IDE0LjkzMjcgMjEuNTI1OSAxMy42MDEgMjAuMDEwNSAxMi4wOTg3IiBzdHJva2U9IiM0OTFEOEIiIHN0cm9rZS13aWR0aD0iNCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIgbWFzaz0idXJsKCNwYXRoLTItb3V0c2lkZS0xXzE4NV82KSIvPgo8L3N2Zz4K" + } +} \ No newline at end of file diff --git a/connectors/agentic-ai/element-templates/versioned/agenticai-aiagent-outbound-connector-11.json b/connectors/agentic-ai/element-templates/versioned/agenticai-aiagent-outbound-connector-11.json new file mode 100644 index 00000000000..522e0b5ff29 --- /dev/null +++ b/connectors/agentic-ai/element-templates/versioned/agenticai-aiagent-outbound-connector-11.json @@ -0,0 +1,1679 @@ +{ + "$schema" : "https://unpkg.com/@camunda/zeebe-element-templates-json-schema/resources/schema.json", + "name" : "AI Agent Task", + "id" : "io.camunda.connectors.agenticai.aiagent.v1", + "description" : "Execute a single AI-powered action with tool calling capabilities", + "keywords" : [ "AI", "AI Agent", "agentic orchestration" ], + "documentationRef" : "https://docs.camunda.io/docs/8.10/components/connectors/out-of-the-box-connectors/agentic-ai-aiagent-task/", + "version" : 11, + "category" : { + "id" : "connectors", + "name" : "Connectors" + }, + "appliesTo" : [ "bpmn:Task" ], + "elementType" : { + "value" : "bpmn:ServiceTask" + }, + "engines" : { + "camunda" : "^8.10" + }, + "groups" : [ { + "id" : "provider", + "label" : "Model provider", + "openByDefault" : false + }, { + "id" : "model", + "label" : "Model", + "openByDefault" : false + }, { + "id" : "systemPrompt", + "label" : "System prompt", + "tooltip" : "A system prompt is a set of foundational instructions given to a model before any user interaction begins. It defines the AI agent’s role, behavior, tone, and communication style, ensuring that responses remain consistent and aligned with the AI agent’s intended purpose. These instructions help shape how the model interprets and responds to user input throughout the conversation.", + "openByDefault" : false + }, { + "id" : "userPrompt", + "label" : "User prompt", + "tooltip" : "A user prompt is the message or question you give to the AI to start or continue a conversation. It tells the AI what you need, whether it's information, help with a task, or just a chat. The AI uses your prompt to understand how to respond.", + "openByDefault" : false + }, { + "id" : "tools", + "label" : "Tools", + "tooltip" : "Tools are optional features the AI Agent can use to perform specific tasks. Configure this if the agent should participate in a tools feedback loop.", + "openByDefault" : false + }, { + "id" : "memory", + "label" : "Memory", + "tooltip" : "Configuration of the Agent's short-term/conversational memory.", + "openByDefault" : false + }, { + "id" : "limits", + "label" : "Limits", + "openByDefault" : false + }, { + "id" : "response", + "label" : "Response", + "tooltip" : "Configuration of the model response format and how to map the model response to the connector result.

        Depending on the selection, the model response will be available as response.responseText or response.responseJson.

        See documentation for details.", + "openByDefault" : false + }, { + "id" : "connector", + "label" : "Connector" + }, { + "id" : "output", + "label" : "Output mapping" + }, { + "id" : "error", + "label" : "Error handling" + }, { + "id" : "retries", + "label" : "Retries" + } ], + "properties" : [ { + "value" : "io.camunda.agenticai:aiagent:1", + "binding" : { + "property" : "type", + "type" : "zeebe:taskDefinition" + }, + "type" : "Hidden" + }, { + "id" : "provider.type", + "label" : "Provider", + "description" : "Specify the LLM provider to use.", + "value" : "anthropic", + "group" : "provider", + "binding" : { + "name" : "provider.type", + "type" : "zeebe:input" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Anthropic", + "value" : "anthropic" + }, { + "name" : "AWS Bedrock", + "value" : "bedrock" + }, { + "name" : "Google GenAI", + "value" : "googleGenAi" + }, { + "name" : "OpenAI", + "value" : "openai" + } ] + }, { + "id" : "provider.anthropic.endpoint", + "label" : "Endpoint", + "description" : "Optional custom API endpoint", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.type", + "label" : "Authentication", + "description" : "Specify the Anthropic authentication strategy.", + "value" : "apiKey", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "API key", + "value" : "apiKey" + }, { + "name" : "Client credentials", + "value" : "clientCredentials" + } ] + }, { + "id" : "provider.anthropic.authentication.apiKey", + "label" : "Anthropic API key", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.apiKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.clientId", + "label" : "Client ID", + "description" : "ID of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.clientId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.clientSecret", + "label" : "Client secret", + "description" : "Secret of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.clientSecret", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.tenantId", + "label" : "Tenant ID", + "description" : "ID of a Microsoft Entra tenant. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.tenantId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.authentication.authorityHost", + "label" : "Authority host", + "description" : "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.authentication.authorityHost", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.anthropic.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.anthropic.timeouts.timeout", + "label" : "Timeout", + "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.anthropic.timeouts.timeout", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.bedrock.region", + "label" : "Region", + "description" : "Specify the AWS region (example: eu-west-1)", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.region", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.bedrock.endpoint", + "label" : "Endpoint", + "description" : "Custom API endpoint for VPC/PrivateLink configurations, AWS GovCloud, or other non-standard deployments.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.bedrock.authentication.type", + "label" : "Authentication", + "description" : "Specify the AWS authentication strategy. Learn more at the documentation page", + "value" : "credentials", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Credentials", + "value" : "credentials" + }, { + "name" : "API Key", + "value" : "apiKey" + }, { + "name" : "Default Credentials Chain (Hybrid/Self-Managed only)", + "value" : "defaultCredentialsChain" + } ] + }, { + "id" : "provider.bedrock.authentication.accessKey", + "label" : "Access key", + "description" : "Provide an IAM access key tailored to a user, equipped with the necessary permissions", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.authentication.accessKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.bedrock.authentication.type", + "equals" : "credentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.bedrock.authentication.secretKey", + "label" : "Secret key", + "description" : "Provide the secret key for the IAM access key", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.authentication.secretKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.bedrock.authentication.type", + "equals" : "credentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.bedrock.authentication.apiKey", + "label" : "API Key", + "description" : "Provide an API Key with permissions to interact with your AWS Bedrock Instance", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.authentication.apiKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.bedrock.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.bedrock.timeouts.timeout", + "label" : "Timeout", + "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.bedrock.timeouts.timeout", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.googleGenAi.projectId", + "label" : "Project ID", + "description" : "Specify Google Cloud project ID", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.googleGenAi.projectId", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.googleGenAi.region", + "label" : "Region", + "description" : "Specify the region where AI inference should take place", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.googleGenAi.region", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.googleGenAi.authentication.type", + "label" : "Authentication", + "description" : "Specify the Google Vertex AI authentication strategy.", + "value" : "serviceAccountCredentials", + "group" : "provider", + "binding" : { + "name" : "provider.googleGenAi.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Service account credentials", + "value" : "serviceAccountCredentials" + }, { + "name" : "Application default credentials (Hybrid/Self-Managed only)", + "value" : "applicationDefaultCredentials" + } ] + }, { + "id" : "provider.googleGenAi.authentication.jsonKey", + "label" : "JSON key of the service account", + "description" : "This is the key of the service account in JSON format.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.googleGenAi.authentication.jsonKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.googleGenAi.authentication.type", + "equals" : "serviceAccountCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.type", + "label" : "Authentication", + "description" : "Specify the OpenAI authentication strategy.", + "value" : "apiKey", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "API key", + "value" : "apiKey" + }, { + "name" : "Client credentials", + "value" : "clientCredentials" + } ] + }, { + "id" : "provider.openai.authentication.apiKey", + "label" : "API key", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.apiKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.organizationId", + "label" : "Organization ID", + "description" : "For members of multiple organizations. Details in the documentation.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.organizationId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.projectId", + "label" : "Project ID", + "description" : "For accounts with multiple projects. Details in the documentation.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.projectId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "apiKey", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.clientId", + "label" : "Client ID", + "description" : "ID of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.clientId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.clientSecret", + "label" : "Client secret", + "description" : "Secret of a Microsoft Entra application", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.clientSecret", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.tenantId", + "label" : "Tenant ID", + "description" : "ID of a Microsoft Entra tenant. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.tenantId", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.openai.authentication.authorityHost", + "label" : "Authority host", + "description" : "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.authentication.authorityHost", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "provider.openai.authentication.type", + "equals" : "clientCredentials", + "type" : "simple" + }, { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "provider.openai.timeouts.timeout", + "label" : "Timeout", + "description" : "Timeout specification for API calls to the model provider defined as ISO-8601 duration (example: PT60S).", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.timeouts.timeout", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openai.apiFamily", + "label" : "API", + "description" : "Which OpenAI API family to use. Chat Completions is the default for backward compatibility; Responses is the newer endpoint with structured input/output items.", + "optional" : true, + "value" : "completions", + "group" : "provider", + "binding" : { + "name" : "provider.openai.apiFamily", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Chat Completions (default)", + "value" : "completions" + }, { + "name" : "Responses", + "value" : "responses" + } ] + }, { + "id" : "provider.openai.endpoint", + "label" : "Endpoint", + "description" : "Optional. Override the default OpenAI base URL (e.g. for an OpenAI proxy or gateway). Leave blank to use the SDK default.", + "optional" : true, + "feel" : "optional", + "group" : "provider", + "binding" : { + "name" : "provider.openai.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openai.headers", + "label" : "Headers", + "description" : "Map of HTTP headers to add to the request.", + "optional" : true, + "feel" : "required", + "group" : "provider", + "binding" : { + "name" : "provider.openai.headers", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openai.queryParameters", + "label" : "Query Parameters", + "description" : "Map of query parameters to add to the request URL.", + "optional" : true, + "feel" : "required", + "group" : "provider", + "binding" : { + "name" : "provider.openai.queryParameters", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.anthropic.model.model", + "label" : "Model", + "description" : "Specify the model ID. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.model", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "placeholder" : "claude-sonnet-4-6", + "type" : "String" + }, { + "id" : "provider.anthropic.model.parameters.maxTokens", + "label" : "Maximum tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.parameters.maxTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "tooltip" : "The maximum number of tokens per request to generate before stopping.

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.anthropic.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. The higher the number, the more randomness will be injected into the response.

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.anthropic.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.anthropic.model.parameters.topK", + "label" : "top K", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.anthropic.model.parameters.topK", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "anthropic", + "type" : "simple" + }, + "tooltip" : "Integer greater than 0. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.bedrock.model.model", + "label" : "Model", + "description" : "Specify an inference profile ID. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.bedrock.model.model", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "placeholder" : "global.anthropic.claude-sonnet-4-6", + "type" : "String" + }, { + "id" : "provider.bedrock.model.parameters.maxTokens", + "label" : "Maximum tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.bedrock.model.parameters.maxTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "tooltip" : "The maximum number of tokens per request to allow in the generated response.

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.bedrock.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.bedrock.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. The higher the number, the more randomness will be injected into the response.

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.bedrock.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.bedrock.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "bedrock", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.googleGenAi.model.model", + "label" : "Model", + "description" : "Specify the model ID. Details in the documentation.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.googleGenAi.model.model", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.googleGenAi.model.parameters.maxOutputTokens", + "label" : "Maximum output tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.googleGenAi.model.parameters.maxOutputTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" + }, + "tooltip" : "Maximum number of tokens that can be generated in the response.

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.googleGenAi.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.googleGenAi.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" + }, + "tooltip" : "Controls the degree of randomness in token selection.

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.googleGenAi.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.googleGenAi.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 1. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.googleGenAi.model.parameters.topK", + "label" : "top K", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.googleGenAi.model.parameters.topK", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "googleGenAi", + "type" : "simple" + }, + "tooltip" : "Integer greater than 0. Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openai.model.model", + "label" : "Model", + "description" : "Specify the model ID. Details in the documentation.", + "optional" : false, + "value" : "gpt-4o", + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "model", + "binding" : { + "name" : "provider.openai.model.model", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "provider.openai.model.parameters.maxCompletionTokens", + "label" : "Maximum completion tokens", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openai.model.parameters.maxCompletionTokens", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "tooltip" : "The maximum number of tokens per request to generate before stopping.

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openai.model.parameters.temperature", + "label" : "Temperature", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openai.model.parameters.temperature", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "tooltip" : "Floating point number between 0 and 2. The higher the number, the more randomness will be injected into the response.

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openai.model.parameters.topP", + "label" : "top P", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openai.model.parameters.topP", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "tooltip" : "Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", + "type" : "Number" + }, { + "id" : "provider.openai.model.parameters.customParameters", + "label" : "Custom parameters", + "description" : "Map of additional request parameters to include.", + "optional" : true, + "feel" : "required", + "group" : "model", + "binding" : { + "name" : "provider.openai.model.parameters.customParameters", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "provider.type", + "equals" : "openai", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.systemPrompt.prompt", + "label" : "System prompt", + "optional" : false, + "value" : "=\"You are **TaskAgent**, a helpful, generic chat agent that can handle a wide variety of customer requests using your own domain knowledge **and** any tools explicitly provided to you at runtime.\n\nIf tools are provided, you should prefer them instead of guessing an answer. You can call the same tool multiple times by providing different input values. Don't guess any tools which were not explicitly configured. If no tool matches the request, try to generate an answer. If you're not able to find a good answer, return with a message stating why you're not able to.\n\nWrap minimal, inspectable reasoning in *exactly* this XML template:\n\n\n…briefly state the customer’s need and current state…\n…list candidate tools, justify which you will call next and why…\n\n\nReveal **no** additional private reasoning outside these tags.\"", + "constraints" : { + "notEmpty" : true + }, + "feel" : "required", + "group" : "systemPrompt", + "binding" : { + "name" : "data.systemPrompt.prompt", + "type" : "zeebe:input" + }, + "type" : "Text" + }, { + "id" : "data.userPrompt.prompt", + "label" : "User prompt", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "required", + "group" : "userPrompt", + "binding" : { + "name" : "data.userPrompt.prompt", + "type" : "zeebe:input" + }, + "type" : "Text" + }, { + "id" : "data.userPrompt.documents", + "label" : "Documents", + "description" : "Documents to be included in the user prompt.", + "optional" : true, + "feel" : "required", + "group" : "userPrompt", + "binding" : { + "name" : "data.userPrompt.documents", + "type" : "zeebe:input" + }, + "tooltip" : "Referenced documents will be automatically added to the user prompt. See documentation for details and supported file types.", + "type" : "String" + }, { + "id" : "data.tools.containerElementId", + "label" : "Ad-hoc sub-process ID", + "description" : "ID of the sub-process that contains the tools the AI agent can use.", + "optional" : true, + "feel" : "optional", + "group" : "tools", + "binding" : { + "name" : "data.tools.containerElementId", + "type" : "zeebe:input" + }, + "tooltip" : "Add an ad-hoc sub-process ID to attach the AI agent to the tools. Ensure your process includes a tools feedback loop routing into the ad-hoc sub-process and back to the AI agent connector. See documentation for details.", + "type" : "String" + }, { + "id" : "data.tools.toolCallResults", + "label" : "Tool call results", + "description" : "Tool call results as returned by the sub-process.", + "optional" : true, + "feel" : "required", + "group" : "tools", + "binding" : { + "name" : "data.tools.toolCallResults", + "type" : "zeebe:input" + }, + "tooltip" : "This defines where to handle tool call results returned by the ad-hoc sub-process. Model this as part of your process and route it into the tools feedback loop. See documentation for details.", + "type" : "Text" + }, { + "id" : "data.agentContext", + "label" : "Agent context", + "description" : "Avoid reusing context variables across agents to prevent issues with stale data or tool access.", + "optional" : false, + "value" : "=agent.context", + "constraints" : { + "notEmpty" : true + }, + "feel" : "required", + "group" : "memory", + "binding" : { + "name" : "data.context", + "type" : "zeebe:input" + }, + "tooltip" : "The agent context variable containing all relevant data for the agent to support the feedback loop between user requests, tool calls and LLM responses. Make sure this variable points to the context variable which is returned from the agent response. See documentation for details.", + "type" : "Text" + }, { + "id" : "data.memory.storage.type", + "label" : "Memory storage type", + "description" : "Specify how to store the conversation memory.", + "value" : "in-process", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.type", + "type" : "zeebe:input" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "In Process (part of agent context)", + "value" : "in-process" + }, { + "name" : "Camunda Document Storage", + "value" : "camunda-document" + }, { + "name" : "AWS AgentCore Memory", + "value" : "aws-agentcore" + }, { + "name" : "Custom Implementation (Hybrid/Self-Managed only)", + "value" : "custom" + } ] + }, { + "id" : "data.memory.storage.timeToLive", + "label" : "Document TTL", + "description" : "How long to retain the conversation document as ISO-8601 duration (example: P14D).", + "optional" : true, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.timeToLive", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "camunda-document", + "type" : "simple" + }, + "tooltip" : "Will use the cluster default TTL (time-to-live) if not specified. Make sure to set this value to a reasonable duration matching your process lifecycle.", + "type" : "String" + }, { + "id" : "data.memory.storage.customProperties", + "label" : "Custom document properties", + "description" : "An optional map of custom properties to be stored with the conversation document.", + "optional" : true, + "feel" : "required", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.customProperties", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "camunda-document", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.region", + "label" : "Region", + "description" : "Specify the AWS region (example: us-east-1)", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.region", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.endpoint", + "label" : "Endpoint", + "description" : "Custom API endpoint for VPC/PrivateLink configurations, AWS GovCloud, or other non-standard deployments.", + "optional" : true, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.endpoint", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.authentication.type", + "label" : "Authentication", + "description" : "Specify the AWS authentication strategy for AgentCore Memory access.", + "value" : "credentials", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.authentication.type", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Credentials", + "value" : "credentials" + }, { + "name" : "Default Credentials Chain (Hybrid/Self-Managed only)", + "value" : "defaultCredentialsChain" + } ] + }, { + "id" : "data.memory.storage.authentication.accessKey", + "label" : "Access key", + "description" : "Provide an IAM access key with permissions for bedrock-agentcore:CreateEvent and bedrock-agentcore:ListEvents", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.authentication.accessKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "data.memory.storage.authentication.type", + "equals" : "credentials", + "type" : "simple" + }, { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "data.memory.storage.authentication.secretKey", + "label" : "Secret key", + "description" : "Provide the secret key for the IAM access key", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.authentication.secretKey", + "type" : "zeebe:input" + }, + "condition" : { + "allMatch" : [ { + "property" : "data.memory.storage.authentication.type", + "equals" : "credentials", + "type" : "simple" + }, { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + } ] + }, + "type" : "String" + }, { + "id" : "data.memory.storage.memoryId", + "label" : "Memory ID", + "description" : "The ID of the pre-provisioned AgentCore Memory resource.", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.memoryId", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.actorId", + "label" : "Actor ID", + "description" : "Identifier of the actor associated with events (e.g., end-user or agent/user combination).", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.actorId", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "aws-agentcore", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.storeType", + "label" : "Implementation type", + "optional" : false, + "constraints" : { + "notEmpty" : true + }, + "feel" : "optional", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.storeType", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "custom", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.storage.parameters", + "label" : "Parameters", + "description" : "Parameters for the custom memory storage implementation.", + "optional" : true, + "feel" : "required", + "group" : "memory", + "binding" : { + "name" : "data.memory.storage.parameters", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.memory.storage.type", + "equals" : "custom", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.memory.contextWindowSize", + "label" : "Context window size", + "description" : "Maximum number of recent conversation messages which are passed to the model.", + "optional" : false, + "value" : 20, + "feel" : "static", + "group" : "memory", + "binding" : { + "name" : "data.memory.contextWindowSize", + "type" : "zeebe:input" + }, + "tooltip" : "Use this to limit the number of messages which are sent to the model. The agent will only send the most recent messages up to the configured limit to the LLM. Older messages will be kept in the conversation store, but not sent to the model. See documentation for details.", + "type" : "Number" + }, { + "id" : "data.limits.maxModelCalls", + "label" : "Maximum model calls", + "description" : "Maximum number of calls to the model as a safety limit to prevent infinite loops.", + "optional" : false, + "value" : 10, + "feel" : "static", + "group" : "limits", + "binding" : { + "name" : "data.limits.maxModelCalls", + "type" : "zeebe:input" + }, + "type" : "Number" + }, { + "id" : "data.response.format.type", + "label" : "Response format", + "description" : "Specify the response format. Support for JSON mode varies by provider.", + "value" : "text", + "group" : "response", + "binding" : { + "name" : "data.response.format.type", + "type" : "zeebe:input" + }, + "type" : "Dropdown", + "choices" : [ { + "name" : "Text", + "value" : "text" + }, { + "name" : "JSON", + "value" : "json" + } ] + }, { + "id" : "data.response.format.parseJson", + "label" : "Parse text as JSON", + "description" : "Tries to parse the LLM response text as JSON object.", + "optional" : true, + "feel" : "static", + "group" : "response", + "binding" : { + "name" : "data.response.format.parseJson", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.response.format.type", + "equals" : "text", + "type" : "simple" + }, + "tooltip" : "Use this option in combination with models which don't support native JSON mode/structured tool calling (e.g. Anthropic). Make sure to instruct the model to return valid JSON in the system prompt. The parsed JSON will be available as response.responseJson.

        If parsing fails, null will be returned as JSON response, but the text content will still be available as response.responseText.", + "type" : "Boolean" + }, { + "id" : "data.response.format.schema", + "label" : "Response JSON schema", + "description" : "An optional response JSON Schema to instruct the model how to structure the JSON output.", + "optional" : true, + "feel" : "required", + "group" : "response", + "binding" : { + "name" : "data.response.format.schema", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.response.format.type", + "equals" : "json", + "type" : "simple" + }, + "tooltip" : "If supported by the model, the response will be structured according to the provided schema. A parsed version of the response will be available as response.responseJson.", + "type" : "String" + }, { + "id" : "data.response.format.schemaName", + "label" : "Response JSON schema name", + "description" : "An optional name for the response JSON Schema to make the model aware of the expected output.", + "optional" : true, + "value" : "Response", + "feel" : "optional", + "group" : "response", + "binding" : { + "name" : "data.response.format.schemaName", + "type" : "zeebe:input" + }, + "condition" : { + "property" : "data.response.format.type", + "equals" : "json", + "type" : "simple" + }, + "type" : "String" + }, { + "id" : "data.response.includeAssistantMessage", + "label" : "Include assistant message", + "description" : "Include the full assistant message as part of the result object.", + "optional" : true, + "feel" : "static", + "group" : "response", + "binding" : { + "name" : "data.response.includeAssistantMessage", + "type" : "zeebe:input" + }, + "tooltip" : "In addition to the text content, the assistant message may include multiple additional content blocks and metadata (such as token usage). The message will be available as response.responseMessage.", + "type" : "Boolean" + }, { + "id" : "version", + "label" : "Version", + "description" : "Version of the element template", + "value" : "11", + "group" : "connector", + "binding" : { + "key" : "elementTemplateVersion", + "type" : "zeebe:taskHeader" + }, + "type" : "Hidden" + }, { + "id" : "id", + "label" : "ID", + "description" : "ID of the element template", + "value" : "io.camunda.connectors.agenticai.aiagent.v1", + "group" : "connector", + "binding" : { + "key" : "elementTemplateId", + "type" : "zeebe:taskHeader" + }, + "type" : "Hidden" + }, { + "id" : "resultVariable", + "label" : "Result variable", + "description" : "Name of variable to store the response in. Details in the documentation.", + "value" : "agent", + "group" : "output", + "binding" : { + "key" : "resultVariable", + "type" : "zeebe:taskHeader" + }, + "type" : "String" + }, { + "id" : "resultExpression", + "label" : "Result expression", + "description" : "Expression to map the response into process variables. Details in the documentation.", + "feel" : "required", + "group" : "output", + "binding" : { + "key" : "resultExpression", + "type" : "zeebe:taskHeader" + }, + "type" : "Text" + }, { + "id" : "errorExpression", + "label" : "Error expression", + "description" : "Expression to handle errors. Details in the documentation.", + "feel" : "required", + "group" : "error", + "binding" : { + "key" : "errorExpression", + "type" : "zeebe:taskHeader" + }, + "type" : "Text" + }, { + "id" : "retryCount", + "label" : "Retries", + "description" : "Number of retries", + "value" : "3", + "feel" : "optional", + "group" : "retries", + "binding" : { + "property" : "retries", + "type" : "zeebe:taskDefinition" + }, + "type" : "String" + }, { + "id" : "retryBackoff", + "label" : "Retry backoff", + "description" : "ISO-8601 duration to wait between retries", + "value" : "PT0S", + "group" : "retries", + "binding" : { + "key" : "retryBackoff", + "type" : "zeebe:taskHeader" + }, + "type" : "String" + } ], + "icon" : { + "contents" : "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMzIiIGhlaWdodD0iMzIiIHZpZXdCb3g9IjAgMCAzMiAzMiIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPGNpcmNsZSBjeD0iMTYiIGN5PSIxNiIgcj0iMTYiIGZpbGw9IiNBNTZFRkYiLz4KPG1hc2sgaWQ9InBhdGgtMi1vdXRzaWRlLTFfMTg1XzYiIG1hc2tVbml0cz0idXNlclNwYWNlT25Vc2UiIHg9IjQiIHk9IjQiIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgZmlsbD0iYmxhY2siPgo8cmVjdCBmaWxsPSJ3aGl0ZSIgeD0iNCIgeT0iNCIgd2lkdGg9IjI0IiBoZWlnaHQ9IjI0Ii8+CjxwYXRoIGZpbGwtcnVsZT0iZXZlbm9kZCIgY2xpcC1ydWxlPSJldmVub2RkIiBkPSJNMjAuMDEwNSAxMi4wOTg3QzE4LjQ5IDEwLjU4OTQgMTcuMTU5NCA4LjEwODE0IDE2LjE3OTkgNi4wMTEwM0MxNi4xNTIgNi4wMDQ1MSAxNi4xMTc2IDYgMTYuMDc5NCA2QzE2LjA0MTEgNiAxNi4wMDY2IDYuMDA0NTEgMTUuOTc4OCA2LjAxMTA0QzE0Ljk5OTQgOC4xMDgxNCAxMy42Njk3IDEwLjU4ODkgMTIuMTQ4MSAxMi4wOTgxQzEwLjYyNjkgMTMuNjA3MSA4LjEyNTY4IDE0LjkyNjQgNi4wMTE1NyAxNS44OTgxQzYuMDA0NzQgMTUuOTI2MSA2IDE1Ljk2MTEgNiAxNkM2IDE2LjAzODcgNi4wMDQ2OCAxNi4wNzM2IDYuMDExNDQgMTYuMTAxNEM4LjEyNTE5IDE3LjA3MjkgMTAuNjI2MiAxOC4zOTE5IDEyLjE0NzcgMTkuOTAxNkMxMy42Njk3IDIxLjQxMDcgMTQuOTk5NiAyMy44OTIgMTUuOTc5MSAyNS45ODlDMTYuMDA2OCAyNS45OTU2IDE2LjA0MTEgMjYgMTYuMDc5MyAyNkMxNi4xMTc1IDI2IDE2LjE1MTkgMjUuOTk1NCAxNi4xNzk2IDI1Ljk4OUMxNy4xNTkxIDIzLjg5MiAxOC40ODg4IDIxLjQxMSAyMC4wMDk5IDE5LjkwMjFNMjAuMDA5OSAxOS45MDIxQzIxLjUyNTMgMTguMzk4NyAyMy45NDY1IDE3LjA2NjkgMjUuOTkxNSAxNi4wODI0QzI1Ljk5NjUgMTYuMDU5MyAyNiAxNi4wMzEgMjYgMTUuOTk5N0MyNiAxNS45Njg0IDI1Ljk5NjUgMTUuOTQwMyAyNS45OTE1IDE1LjkxNzFDMjMuOTQ3NCAxNC45MzI3IDIxLjUyNTkgMTMuNjAxIDIwLjAxMDUgMTIuMDk4NyIvPgo8L21hc2s+CjxwYXRoIGZpbGwtcnVsZT0iZXZlbm9kZCIgY2xpcC1ydWxlPSJldmVub2RkIiBkPSJNMjAuMDEwNSAxMi4wOTg3QzE4LjQ5IDEwLjU4OTQgMTcuMTU5NCA4LjEwODE0IDE2LjE3OTkgNi4wMTEwM0MxNi4xNTIgNi4wMDQ1MSAxNi4xMTc2IDYgMTYuMDc5NCA2QzE2LjA0MTEgNiAxNi4wMDY2IDYuMDA0NTEgMTUuOTc4OCA2LjAxMTA0QzE0Ljk5OTQgOC4xMDgxNCAxMy42Njk3IDEwLjU4ODkgMTIuMTQ4MSAxMi4wOTgxQzEwLjYyNjkgMTMuNjA3MSA4LjEyNTY4IDE0LjkyNjQgNi4wMTE1NyAxNS44OTgxQzYuMDA0NzQgMTUuOTI2MSA2IDE1Ljk2MTEgNiAxNkM2IDE2LjAzODcgNi4wMDQ2OCAxNi4wNzM2IDYuMDExNDQgMTYuMTAxNEM4LjEyNTE5IDE3LjA3MjkgMTAuNjI2MiAxOC4zOTE5IDEyLjE0NzcgMTkuOTAxNkMxMy42Njk3IDIxLjQxMDcgMTQuOTk5NiAyMy44OTIgMTUuOTc5MSAyNS45ODlDMTYuMDA2OCAyNS45OTU2IDE2LjA0MTEgMjYgMTYuMDc5MyAyNkMxNi4xMTc1IDI2IDE2LjE1MTkgMjUuOTk1NCAxNi4xNzk2IDI1Ljk4OUMxNy4xNTkxIDIzLjg5MiAxOC40ODg4IDIxLjQxMSAyMC4wMDk5IDE5LjkwMjFNMjAuMDA5OSAxOS45MDIxQzIxLjUyNTMgMTguMzk4NyAyMy45NDY1IDE3LjA2NjkgMjUuOTkxNSAxNi4wODI0QzI1Ljk5NjUgMTYuMDU5MyAyNiAxNi4wMzEgMjYgMTUuOTk5N0MyNiAxNS45Njg0IDI1Ljk5NjUgMTUuOTQwMyAyNS45OTE1IDE1LjkxNzFDMjMuOTQ3NCAxNC45MzI3IDIxLjUyNTkgMTMuNjAxIDIwLjAxMDUgMTIuMDk4NyIgZmlsbD0id2hpdGUiLz4KPHBhdGggZmlsbC1ydWxlPSJldmVub2RkIiBjbGlwLXJ1bGU9ImV2ZW5vZGQiIGQ9Ik0yMC4wMTA1IDEyLjA5ODdDMTguNDkgMTAuNTg5NCAxNy4xNTk0IDguMTA4MTQgMTYuMTc5OSA2LjAxMTAzQzE2LjE1MiA2LjAwNDUxIDE2LjExNzYgNiAxNi4wNzk0IDZDMTYuMDQxMSA2IDE2LjAwNjYgNi4wMDQ1MSAxNS45Nzg4IDYuMDExMDRDMTQuOTk5NCA4LjEwODE0IDEzLjY2OTcgMTAuNTg4OSAxMi4xNDgxIDEyLjA5ODFDMTAuNjI2OSAxMy42MDcxIDguMTI1NjggMTQuOTI2NCA2LjAxMTU3IDE1Ljg5ODFDNi4wMDQ3NCAxNS45MjYxIDYgMTUuOTYxMSA2IDE2QzYgMTYuMDM4NyA2LjAwNDY4IDE2LjA3MzYgNi4wMTE0NCAxNi4xMDE0QzguMTI1MTkgMTcuMDcyOSAxMC42MjYyIDE4LjM5MTkgMTIuMTQ3NyAxOS45MDE2QzEzLjY2OTcgMjEuNDEwNyAxNC45OTk2IDIzLjg5MiAxNS45NzkxIDI1Ljk4OUMxNi4wMDY4IDI1Ljk5NTYgMTYuMDQxMSAyNiAxNi4wNzkzIDI2QzE2LjExNzUgMjYgMTYuMTUxOSAyNS45OTU0IDE2LjE3OTYgMjUuOTg5QzE3LjE1OTEgMjMuODkyIDE4LjQ4ODggMjEuNDExIDIwLjAwOTkgMTkuOTAyMU0yMC4wMDk5IDE5LjkwMjFDMjEuNTI1MyAxOC4zOTg3IDIzLjk0NjUgMTcuMDY2OSAyNS45OTE1IDE2LjA4MjRDMjUuOTk2NSAxNi4wNTkzIDI2IDE2LjAzMSAyNiAxNS45OTk3QzI2IDE1Ljk2ODQgMjUuOTk2NSAxNS45NDAzIDI1Ljk5MTUgMTUuOTE3MUMyMy45NDc0IDE0LjkzMjcgMjEuNTI1OSAxMy42MDEgMjAuMDEwNSAxMi4wOTg3IiBzdHJva2U9IiM0OTFEOEIiIHN0cm9rZS13aWR0aD0iNCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIgbWFzaz0idXJsKCNwYXRoLTItb3V0c2lkZS0xXzE4NV82KSIvPgo8L3N2Zz4K" + } +} \ No newline at end of file diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/AiAgentFunction.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/AiAgentFunction.java index 93f6e3df72f..cfc0d9d20ad 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/AiAgentFunction.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/AiAgentFunction.java @@ -38,7 +38,7 @@ documentationRef = "https://docs.camunda.io/docs/8.10/components/connectors/out-of-the-box-connectors/agentic-ai-aiagent-task/", engineVersion = "^8.10", - version = 11, + version = 12, inputDataClass = OutboundConnectorAgentRequest.class, outputDataClass = AgentResponse.class, defaultResultVariable = "agent", diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/AnthropicProviderConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/AnthropicProviderConfiguration.java index 611b003ae33..59ed0e0a8db 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/AnthropicProviderConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/AnthropicProviderConfiguration.java @@ -58,7 +58,26 @@ public record AnthropicConnection( feel = FeelMode.optional, optional = true) String endpoint, - @TemplateProperty(ignore = true) AnthropicBackend backend, + @TemplateProperty( + group = "provider", + label = "Backend", + description = "Specify the Anthropic backend to use.", + type = TemplateProperty.PropertyType.Dropdown, + defaultValue = "direct", + defaultValueType = TemplateProperty.DefaultValueType.String, + choices = { + @TemplateProperty.DropdownPropertyChoice( + label = "Direct (Anthropic API)", + value = "direct"), + @TemplateProperty.DropdownPropertyChoice(label = "AWS Bedrock", value = "bedrock"), + @TemplateProperty.DropdownPropertyChoice( + label = "Google Vertex AI", + value = "vertex"), + @TemplateProperty.DropdownPropertyChoice( + label = "Azure AI Foundry", + value = "foundry") + }) + AnthropicBackend backend, @Valid @NotNull AnthropicAuthentication authentication, @Valid TimeoutConfiguration timeouts, @Valid @NotNull AnthropicModel model) { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/GoogleGenAiProviderConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/GoogleGenAiProviderConfiguration.java index 1aa51c2ccda..2eb77e2e221 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/GoogleGenAiProviderConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/GoogleGenAiProviderConfiguration.java @@ -63,7 +63,20 @@ public record GoogleGenAiConnection( String region, @Valid @NotNull GoogleGenAiAuthentication authentication, @Valid @NotNull GoogleGenAiModel model, - @TemplateProperty(ignore = true) GoogleBackend backend) { + @TemplateProperty( + group = "provider", + label = "Backend", + description = "Specify the Google GenAI backend to use.", + type = TemplateProperty.PropertyType.Dropdown, + defaultValue = "vertex", + defaultValueType = TemplateProperty.DefaultValueType.String, + choices = { + @TemplateProperty.DropdownPropertyChoice(label = "Vertex AI", value = "vertex"), + @TemplateProperty.DropdownPropertyChoice( + label = "Developer API (Google AI Studio)", + value = "developer-api") + }) + GoogleBackend backend) { public GoogleGenAiConnection { if (backend == null) { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiProviderConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiProviderConfiguration.java index 27d1e7c3308..92c6fb68a76 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiProviderConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiProviderConfiguration.java @@ -49,7 +49,21 @@ public enum OpenAiBackend { } public record OpenAiConnection( - @TemplateProperty(ignore = true) OpenAiBackend backend, + @TemplateProperty( + group = "provider", + label = "Backend", + description = "Specify the OpenAI backend to use.", + type = TemplateProperty.PropertyType.Dropdown, + defaultValue = "openai", + defaultValueType = TemplateProperty.DefaultValueType.String, + choices = { + @TemplateProperty.DropdownPropertyChoice(label = "OpenAI", value = "openai"), + @TemplateProperty.DropdownPropertyChoice( + label = "Azure AI Foundry", + value = "foundry"), + @TemplateProperty.DropdownPropertyChoice(label = "Custom", value = "custom") + }) + OpenAiBackend backend, @Valid @NotNull OpenAiAuthentication authentication, @Valid TimeoutConfiguration timeouts, @Valid @NotNull OpenAiModel model, From 77b5c91632b0cabe74135725136229d61c4eb390 Mon Sep 17 00:00:00 2001 From: Dmitri Nikonov Date: Fri, 8 May 2026 11:52:05 +0200 Subject: [PATCH 80/81] feat(agentic-ai): Implement provider configuration (Phase F) --- .../docs/adr-005-implementation-plan.md | 57 ++++-- .../adr/005-replace-langchain4j-framework.md | 64 ++++--- ...icAiLangchain4JFrameworkConfiguration.java | 20 +- .../AzureOpenAiChatModelProvider.java | 34 ++-- .../provider/OpenAiChatModelProvider.java | 10 +- .../OpenAiCompatibleChatModelProvider.java | 27 ++- .../OpenAiCompatibleChatModelApiFactory.java | 106 ----------- .../AzureOpenAiProviderConfiguration.java | 178 ------------------ ...OpenAiCompatibleProviderConfiguration.java | 139 -------------- .../AzureOpenAiChatModelProviderTest.java | 92 +++++---- .../provider/OpenAiChatModelProviderTest.java | 14 +- ...OpenAiCompatibleChatModelProviderTest.java | 115 +++++------ 12 files changed, 254 insertions(+), 602 deletions(-) delete mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiCompatibleChatModelApiFactory.java delete mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/AzureOpenAiProviderConfiguration.java delete mode 100644 connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiCompatibleProviderConfiguration.java diff --git a/connectors/agentic-ai/docs/adr-005-implementation-plan.md b/connectors/agentic-ai/docs/adr-005-implementation-plan.md index a6d46f8f2e1..4fcbd1b71bd 100644 --- a/connectors/agentic-ai/docs/adr-005-implementation-plan.md +++ b/connectors/agentic-ai/docs/adr-005-implementation-plan.md @@ -500,13 +500,39 @@ inline-native path in E4. **Goal**: introduce the canonical discriminator scheme without breaking saved process state. Element template version bump 11 → 12 (after D's bump). -**Files to modify**: +**Files to modify / create**: - `AnthropicProviderConfiguration`: add `AnthropicBackend { DIRECT, BEDROCK, VERTEX, FOUNDRY }`; - conditional auth fields per backend. -- `OpenAiProviderConfiguration`: `apiFamily` already added in Phase D — extend handling here as - needed. -- `AzureOpenAiProviderConfiguration`: add `apiFamily` matching the OpenAI shape. -- `OpenAiCompatibleProviderConfiguration`: stays Completions-only (no `apiFamily` field). + conditional auth fields per backend. `AnthropicAuthentication` is converted from a flat record + to a `sealed interface` with a `type` discriminator — same pattern as `AzureAuthentication` in + `AzureOpenAiProviderConfiguration`. Two non-platform variants: + - `AnthropicApiKeyAuthentication(@NotBlank String apiKey)` — preserves the current single field. + Valid for **DIRECT** and **FOUNDRY** backends. + - `AnthropicClientCredentialsAuthentication(@NotBlank String clientId, @NotBlank String clientSecret, + @NotBlank String tenantId, String authorityHost)` — exact field-for-field mirror of + `AzureClientCredentialsAuthentication` (`authorityHost` optional). Valid for **FOUNDRY** only. + Backend × auth validity enforced at construction time: reject `(DIRECT, clientCredentials)` and + any `(BEDROCK | VERTEX, apiKey | clientCredentials)` pairing with a clear error. BEDROCK and + VERTEX auth shapes (AWS credentials chain and Google service-account / ADC) follow the patterns + established in `BedrockProviderConfiguration` and `GoogleVertexAiProviderConfiguration`. +- `OpenAiProviderConfiguration`: consolidates the three existing OpenAI-family configs into one. + - Add `OpenAiBackend { OPENAI, FOUNDRY, CUSTOM }` enum + `backend` field. + - `apiFamily` (already added in Phase D) stays and is present for all backends. + - Auth becomes conditional on backend: + - `OPENAI`: existing apiKey (+ optional organizationId, projectId). + - `FOUNDRY`: apiKey **or** clientCredentials (Entra ID / Client Credentials, same shape as the + existing `AzureAuthentication` sealed type). + - `CUSTOM`: apiKey (optional). + - `endpoint`: required for FOUNDRY and CUSTOM backends. + - `headers`, `queryParameters`, `customParameters` (`Map`): CUSTOM backend only, + carried over from `OpenAiCompatibleProviderConfiguration`. + - Discriminator stays `openai`; model type stays `OpenAiModel`. +- **Remove** `AzureOpenAiProviderConfiguration` and `OpenAiCompatibleProviderConfiguration` as + sealed interface members; drop their `@JsonSubTypes` entries. Existing Java source files are + deleted; the migration deserializer handles their saved shapes. +- **Remove** `OpenAiCompatibleChatModelApiFactory`: its logic (custom base URL + optional API key) + folds into `OpenAiChatModelApiFactory` under the `CUSTOM` backend branch. The `OpenAiChatModelApiFactory` + now branches on `backend` (to build the right `OpenAIClient`) and then on `apiFamily` (to pick + the impl class). - `GoogleVertexAiProviderConfiguration` → `GoogleGenAiProviderConfiguration`: add `GoogleBackend { DEVELOPER_API, VERTEX }`; rename discriminator `googleVertexAi` → `googleGenAi`. - `BedrockProviderConfiguration`: validate at construction that model ID is non-Anthropic @@ -523,8 +549,12 @@ Element template version bump 11 → 12 (after D's bump). the new shape — `@JsonDeserialize` only affects the read path. Inside the deserializer, dispatch to concrete subtypes via `mapper.treeToValue(migrated, SubType.class)`; that doesn't re-enter the custom deserializer because Jackson's `BeanDeserializer` for the resolved subtype takes over. -- `element-templates/agenticai-aiagent-outbound-connector.json`: bump 11 → 12; add conditional UI - groups for `backend`. Maven regenerates the versioned snapshot + job-worker template. +- `element-templates/agenticai-aiagent-outbound-connector.json`: bump 11 → 12. The three separate + OpenAI provider entries (`openai`, `azureOpenAi`, `openaiCompatible`) are collapsed into one + `openai` entry with a `backend` dropdown (OpenAI / Azure AI Foundry / Custom). Anthropic gains a + `backend` dropdown (Direct / Bedrock / Vertex / Foundry). Google is renamed and gains a `backend` + dropdown (Developer API / Vertex). Auth fields are conditionally shown per backend. Maven + regenerates the versioned snapshot + job-worker template. - `element-templates/README.md`: replace top row again (or insert new row if Camunda min version changes). @@ -551,11 +581,12 @@ covers a saved `agentContext` from a v10/v11 instance. `backend = bedrock | vertex | foundry` (SDK modules: `anthropic-java-bedrock`, `anthropic-java-vertex`, `anthropic-java-foundry`). Same wire format; only client construction differs. -2. **Native Azure OpenAI** — `AzureOpenAiChatModelApiFactory` builds the Azure-flavored - `OpenAIClient` (API key or Entra/AAD auth) and hands to the existing `OpenAiChatCompletions` - / `OpenAiResponses` impl classes. Branches on `config.apiFamily()` exactly like the OpenAI - factory. (If openai-java's Azure variant lags Responses endpoint coverage, Azure stays - Completions-only for one release.) +2. **OpenAI FOUNDRY backend** — extend `OpenAiChatModelApiFactory` to build an Azure-flavored + `OpenAIClient` when `config.backend() == FOUNDRY` (API key or Entra/AAD clientCredentials auth, + endpoint from config). Hands to the existing `OpenAiChatCompletions` / `OpenAiResponses` impl + classes; branches on `config.apiFamily()` as today. No separate factory class needed. (If + openai-java's Azure variant lags Responses endpoint coverage, FOUNDRY stays Completions-only for + one release.) 3. **Native Google GenAI** — `google-genai-java` SDK; backend toggle (`developer-api` / `vertex`). Reasoning via `thoughtSignature`; thinking budget via `ThinkingConfig.thinkingBudget`. 4. **Native Bedrock-Converse** — AWS SDK v2 `bedrockruntime`. Non-Anthropic models only. diff --git a/connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md b/connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md index 508595a60b2..56612cead3d 100644 --- a/connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md +++ b/connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md @@ -560,9 +560,10 @@ fields. ## Provider Configuration Restructure Configurations are restructured by wire format. The `ProviderConfiguration` sealed type -retains six members (matching today's count) but each gains a discriminator for backend or -api-family choice. Element template UI groups conditional fields under each discriminator -value. +reduces from six to **four** members. The three separate OpenAI-family configs (`openai`, +`azureOpenAi`, `openaiCompatible`) are merged into a single `OpenAiProviderConfiguration` +with a backend discriminator, mirroring the Anthropic pattern. Element template UI groups +conditional fields under each discriminator value. ``` ProviderConfiguration @@ -575,23 +576,26 @@ ProviderConfiguration │ auth (existing AWS chain) │ model: BedrockModel ├── OpenAiProviderConfiguration api: openai-{responses|completions} -│ apiFamily: { responses | completions } ← NEW (default: responses) -│ auth (API key) +│ backend: { openai | foundry | custom } ← NEW (replaces 3 separate configs) +│ apiFamily: { responses | completions } ← present for all backends +│ auth conditional on backend: +│ openai: apiKey (+ optional organizationId, projectId) +│ foundry: apiKey | clientCredentials (Entra ID / Client Credentials) +│ custom: apiKey (optional) +│ endpoint: required for foundry and custom backends +│ headers, queryParameters, customParameters: Map (custom backend only) │ model: OpenAiModel -├── AzureOpenAiProviderConfiguration api: openai-{responses|completions} -│ apiFamily: { responses | completions } ← NEW -│ auth (existing — endpoint, key) -│ model: AzureModel -├── OpenAiCompatibleProviderConfiguration api: openai-{responses|completions} -│ apiFamily: { responses | completions } ← NEW (default: completions) -│ auth (existing — endpoint, optional key) -│ model: OpenAiCompatibleModel └── GoogleGenAiProviderConfiguration api: google-genai backend: { developer-api | vertex } ← NEW auth fields conditional on backend model: GeminiModel ``` +`AzureOpenAiProviderConfiguration` and `OpenAiCompatibleProviderConfiguration` are removed +as sealed interface members. Their saved shapes are rewritten by the migration deserializer +(see table below), and their discriminators (`azureOpenAi`, `openaiCompatible`) are dropped +from `@JsonSubTypes`. + `GoogleVertexAiProviderConfiguration` is renamed to `GoogleGenAiProviderConfiguration` since the same configuration now covers both backends. @@ -605,19 +609,29 @@ A custom `StdDeserializer` rewrites legacy configuration the new structure at deserialization time. Existing process instances continue to work without manual intervention. -| Saved shape | Rewritten to | Detection | -|----------------------------------------------------------------------------|-------------------------------------------------------------------------------------------|-----------------------| -| `{type: bedrock, bedrock.model.model: "anthropic.claude-..."}` | `{type: anthropic, anthropic.backend: bedrock, anthropic.model.model: "anthropic..."}` | model-id prefix | -| `{type: bedrock, bedrock.model.model: "amazon.nova-..." \| "meta..." }` | passthrough | model-id prefix | -| `{type: googleVertexAi, ...}` | `{type: googleGenAi, googleGenAi.backend: vertex, ...}` | discriminator rename | -| `{type: anthropic, ...}` (no backend) | `{type: anthropic, anthropic.backend: direct, ...}` | missing field default | -| `{type: openai, ...}` (no apiFamily) | `{type: openai, openai.apiFamily: completions, ...}` | missing field default | -| `{type: azureOpenAi, ...}` (no apiFamily) | same with `apiFamily: completions` | missing field default | -| `{type: openAiCompatible, ...}` (no apiFamily) | same with `apiFamily: completions` | missing field default | - -Defaults during migration preserve current behavior (Chat Completions for OpenAI variants, +| Saved shape | Rewritten to | Detection | +|----------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------|-----------------------| +| `{type: bedrock, bedrock.model.model: "anthropic.claude-..."}` | `{type: anthropic, anthropic.backend: bedrock, anthropic.model.model: "anthropic..."}` | model-id prefix | +| `{type: bedrock, bedrock.model.model: "amazon.nova-..." \| "meta..." }` | passthrough | model-id prefix | +| `{type: googleVertexAi, ...}` | `{type: googleGenAi, googleGenAi.backend: vertex, ...}` | discriminator rename | +| `{type: anthropic, ...}` (no backend) | `{type: anthropic, anthropic.backend: direct, ...}` | missing field default | +| `{type: anthropic, anthropic.authentication: {apiKey: "..."}}` (no auth `type`) | `{type: anthropic, anthropic.authentication: {type: apiKey, apiKey: "..."}, ...}` | missing nested discriminator | +| `{type: openai, ...}` (no backend) | `{type: openai, openai.backend: openai, openai.apiFamily: completions, ...}` | missing field default | +| `{type: azureOpenAi, ...}` | `{type: openai, openai.backend: foundry, openai.apiFamily: completions, ...}` with auth + endpoint field mapping | discriminator rename | +| `{type: openaiCompatible, ...}` | `{type: openai, openai.backend: custom, openai.apiFamily: completions, ...}` with auth + custom field mapping | discriminator rename | + +Auth field mapping for `azureOpenAi` → `foundry`: the `azureOpenAi.auth` sub-tree (apiKey +or clientCredentials shape) maps directly onto `openai.auth`; `azureOpenAi.endpoint` maps +to `openai.endpoint`. + +Auth and custom field mapping for `openaiCompatible` → `custom`: `openaiCompatible.auth.apiKey` +maps to `openai.auth.apiKey`; `openaiCompatible.endpoint` maps to `openai.endpoint`; +`openaiCompatible.headers`, `openaiCompatible.queryParameters`, and +`openaiCompatible.customParameters` map to the same-named fields under `openai`. + +Defaults during migration preserve current behavior (Chat Completions for all OpenAI variants, direct for Anthropic). Newly created configurations pick up the new defaults (Responses API -for OpenAI direct). +for the openai backend). The migration deserializer is permanent infrastructure — kept indefinitely so that stale process variables remain readable. diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java index d82df7ccde9..ae50a91cc6a 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/configuration/AgenticAiLangchain4JFrameworkConfiguration.java @@ -21,9 +21,9 @@ import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolCallConverterImpl; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverter; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.tool.ToolSpecificationConverterImpl; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.BedrockProviderConfiguration; import io.camunda.connector.agenticai.aiagent.model.request.provider.GoogleGenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; import io.camunda.connector.runtime.annotation.ConnectorsObjectMapper; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -98,16 +98,16 @@ public ChatModelApiFactory langchain4JBedrockChatM } @Bean - @ConditionalOnMissingBean(name = "langchain4JAzureOpenAiChatModelApiFactory") - public ChatModelApiFactory - langchain4JAzureOpenAiChatModelApiFactory( - ChatModelProvider provider, - ChatMessageConverter chatMessageConverter, - ToolSpecificationConverter toolSpecificationConverter, - JsonSchemaConverter jsonSchemaConverter) { + @ConditionalOnMissingBean( + name = {"langchain4JAzureOpenAiChatModelApiFactory", "langchain4JOpenAiChatModelApiFactory"}) + public ChatModelApiFactory langchain4JAzureOpenAiChatModelApiFactory( + ChatModelProvider provider, + ChatMessageConverter chatMessageConverter, + ToolSpecificationConverter toolSpecificationConverter, + JsonSchemaConverter jsonSchemaConverter) { return new Langchain4JChatModelApiFactory<>( - AzureOpenAiProviderConfiguration.AZURE_OPENAI_ID, - AzureOpenAiProviderConfiguration.class, + OpenAiProviderConfiguration.OPENAI_ID, + OpenAiProviderConfiguration.class, provider, chatMessageConverter, toolSpecificationConverter, diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/AzureOpenAiChatModelProvider.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/AzureOpenAiChatModelProvider.java index aaa81c2dfeb..c49ea9d420f 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/AzureOpenAiChatModelProvider.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/AzureOpenAiChatModelProvider.java @@ -12,17 +12,23 @@ import dev.langchain4j.model.azure.AzureOpenAiChatModel; import dev.langchain4j.model.chat.ChatModel; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelHttpProxySupport; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration.AzureAuthentication.AzureApiKeyAuthentication; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration.AzureAuthentication.AzureClientCredentialsAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiAuthentication.OpenAiApiKeyAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiAuthentication.OpenAiClientCredentialsAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiBackend; import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsConfigurationProperties.ChatModelProperties; import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * LangChain4j bridge provider for the Azure AI Foundry (FOUNDRY) backend of the unified {@link + * OpenAiProviderConfiguration}. Handles OpenAiProviderConfiguration instances where {@code backend + * == FOUNDRY} by building an {@link AzureOpenAiChatModel}. + */ public class AzureOpenAiChatModelProvider - implements ChatModelProvider { + implements ChatModelProvider { private static final Logger LOGGER = LoggerFactory.getLogger(AzureOpenAiChatModelProvider.class); @@ -37,16 +43,21 @@ public AzureOpenAiChatModelProvider( @Override public String type() { - return AzureOpenAiProviderConfiguration.AZURE_OPENAI_ID; + return OpenAiProviderConfiguration.OPENAI_ID; } @Override - public ChatModel createChatModel(AzureOpenAiProviderConfiguration azureOpenAi) { - final var connection = azureOpenAi.azureOpenAi(); + public ChatModel createChatModel(OpenAiProviderConfiguration openAi) { + if (openAi.openai().backend() != OpenAiBackend.FOUNDRY) { + throw new IllegalArgumentException( + "AzureOpenAiChatModelProvider only supports OpenAiBackend.FOUNDRY, got: " + + openAi.openai().backend()); + } + final var connection = openAi.openai(); final var builder = AzureOpenAiChatModel.builder() .endpoint(connection.endpoint()) - .deploymentName(connection.model().deploymentName()) + .deploymentName(connection.model().model()) .timeout( deriveTimeoutSetting( "Azure OpenAI model call", config, connection.timeouts(), LOGGER)); @@ -54,9 +65,8 @@ public ChatModel createChatModel(AzureOpenAiProviderConfiguration azureOpenAi) { proxySupport.createAzureProxyOptions(connection.endpoint()).ifPresent(builder::proxyOptions); switch (connection.authentication()) { - case AzureApiKeyAuthentication azureApiKeyAuthentication -> - builder.apiKey(azureApiKeyAuthentication.apiKey()); - case AzureClientCredentialsAuthentication auth -> { + case OpenAiApiKeyAuthentication apiKeyAuth -> builder.apiKey(apiKeyAuth.apiKey()); + case OpenAiClientCredentialsAuthentication auth -> { ClientSecretCredentialBuilder clientSecretCredentialBuilder = new ClientSecretCredentialBuilder() .clientId(auth.clientId()) @@ -71,7 +81,7 @@ public ChatModel createChatModel(AzureOpenAiProviderConfiguration azureOpenAi) { final var modelParameters = connection.model().parameters(); if (modelParameters != null) { - Optional.ofNullable(modelParameters.maxTokens()).ifPresent(builder::maxTokens); + Optional.ofNullable(modelParameters.maxCompletionTokens()).ifPresent(builder::maxTokens); Optional.ofNullable(modelParameters.temperature()).ifPresent(builder::temperature); Optional.ofNullable(modelParameters.topP()).ifPresent(builder::topP); } diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiChatModelProvider.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiChatModelProvider.java index 3d6511faaca..bb4c09e7177 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiChatModelProvider.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiChatModelProvider.java @@ -13,6 +13,7 @@ import dev.langchain4j.model.openai.OpenAiChatRequestParameters; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelHttpProxySupport; import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiAuthentication.OpenAiApiKeyAuthentication; import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsConfigurationProperties.ChatModelProperties; import java.util.Optional; import org.slf4j.Logger; @@ -42,15 +43,16 @@ public ChatModel createChatModel(OpenAiProviderConfiguration openai) { final var builder = OpenAiChatModel.builder() - .apiKey(connection.authentication().apiKey()) .modelName(connection.model().model()) .timeout( deriveTimeoutSetting("OpenAI model call", config, connection.timeouts(), LOGGER)) .httpClientBuilder(proxySupport.createJdkHttpClientBuilder()); - Optional.ofNullable(connection.authentication().organizationId()) - .ifPresent(builder::organizationId); - Optional.ofNullable(connection.authentication().projectId()).ifPresent(builder::projectId); + if (connection.authentication() instanceof OpenAiApiKeyAuthentication apiKeyAuth) { + builder.apiKey(apiKeyAuth.apiKey()); + Optional.ofNullable(apiKeyAuth.organizationId()).ifPresent(builder::organizationId); + Optional.ofNullable(apiKeyAuth.projectId()).ifPresent(builder::projectId); + } final var modelParameters = connection.model().parameters(); if (modelParameters != null) { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiCompatibleChatModelProvider.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiCompatibleChatModelProvider.java index a3227e0e0c1..78a9811729f 100644 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiCompatibleChatModelProvider.java +++ b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiCompatibleChatModelProvider.java @@ -12,16 +12,23 @@ import dev.langchain4j.model.openai.OpenAiChatModel; import dev.langchain4j.model.openai.OpenAiChatRequestParameters; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelHttpProxySupport; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration.OpenAiCompatibleAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiAuthentication.OpenAiApiKeyAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiBackend; import io.camunda.connector.agenticai.autoconfigure.AgenticAiConnectorsConfigurationProperties.ChatModelProperties; import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * LangChain4j bridge provider for the CUSTOM backend of the unified {@link + * OpenAiProviderConfiguration}. Handles OpenAiProviderConfiguration instances where {@code backend + * == CUSTOM} by building an OpenAI-compatible chat model with a custom base URL, optional API key, + * and custom headers / query params. + */ public class OpenAiCompatibleChatModelProvider - implements ChatModelProvider { + implements ChatModelProvider { private static final Logger LOGGER = LoggerFactory.getLogger(OpenAiCompatibleChatModelProvider.class); @@ -37,12 +44,17 @@ public OpenAiCompatibleChatModelProvider( @Override public String type() { - return OpenAiCompatibleProviderConfiguration.OPENAI_COMPATIBLE_ID; + return OpenAiProviderConfiguration.OPENAI_ID; } @Override - public ChatModel createChatModel(OpenAiCompatibleProviderConfiguration openaiCompatible) { - final var connection = openaiCompatible.openaiCompatible(); + public ChatModel createChatModel(OpenAiProviderConfiguration openAi) { + if (openAi.openai().backend() != OpenAiBackend.CUSTOM) { + throw new IllegalArgumentException( + "OpenAiCompatibleChatModelProvider only supports OpenAiBackend.CUSTOM, got: " + + openAi.openai().backend()); + } + final var connection = openAi.openai(); final var builder = OpenAiChatModel.builder() @@ -54,7 +66,8 @@ public ChatModel createChatModel(OpenAiCompatibleProviderConfiguration openaiCom .httpClientBuilder(proxySupport.createJdkHttpClientBuilder()); Optional.ofNullable(connection.authentication()) - .map(OpenAiCompatibleAuthentication::apiKey) + .filter(auth -> auth instanceof OpenAiApiKeyAuthentication) + .map(auth -> ((OpenAiApiKeyAuthentication) auth).apiKey()) .filter(StringUtils::isNotBlank) .ifPresent( apiKey -> { diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiCompatibleChatModelApiFactory.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiCompatibleChatModelApiFactory.java deleted file mode 100644 index a9eaedbaa09..00000000000 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/framework/openai/OpenAiCompatibleChatModelApiFactory.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH - * under one or more contributor license agreements. Licensed under a proprietary license. - * See the License.txt file for more information. You may not use this file - * except in compliance with the proprietary license. - */ -package io.camunda.connector.agenticai.aiagent.framework.openai; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.openai.client.OpenAIClient; -import com.openai.client.okhttp.OpenAIOkHttpClient; -import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApi; -import io.camunda.connector.agenticai.aiagent.framework.api.ChatModelApiFactory; -import io.camunda.connector.agenticai.aiagent.framework.capabilities.ModelCapabilitiesResolver; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration.OpenAiCompatibleConnection; -import java.time.Duration; -import java.util.Optional; -import org.apache.commons.lang3.StringUtils; -import org.springframework.lang.Nullable; - -/** - * Native factory for the {@code openaiCompatible} discriminator. Same {@link - * OpenAiChatCompletionsChatModelApi} impl as the OpenAI-direct factory; only the OkHttp client - * construction differs (custom baseUrl, optional API key, custom headers / query params). - */ -public class OpenAiCompatibleChatModelApiFactory - implements ChatModelApiFactory { - - public static final String API_FAMILY = "openai-completions"; - - private final ObjectMapper objectMapper; - private final ModelCapabilitiesResolver capabilitiesResolver; - @Nullable private final Duration defaultTimeout; - - public OpenAiCompatibleChatModelApiFactory( - ObjectMapper objectMapper, - ModelCapabilitiesResolver capabilitiesResolver, - @Nullable Duration defaultTimeout) { - this.objectMapper = objectMapper; - this.capabilitiesResolver = capabilitiesResolver; - this.defaultTimeout = defaultTimeout; - } - - @Override - public String providerType() { - return OpenAiCompatibleProviderConfiguration.OPENAI_COMPATIBLE_ID; - } - - @Override - public String apiFamily() { - return API_FAMILY; - } - - @Override - public Class configurationType() { - return OpenAiCompatibleProviderConfiguration.class; - } - - @Override - public ChatModelApi create(OpenAiCompatibleProviderConfiguration configuration) { - final var connection = configuration.openaiCompatible(); - final var client = buildClient(connection); - final var parameters = connection.model().parameters(); - final var capabilities = - capabilitiesResolver.resolve(API_FAMILY, connection.model().model(), Optional.empty()); - return new OpenAiChatCompletionsChatModelApi( - client, - connection.model().model(), - objectMapper, - capabilities, - parameters != null && parameters.maxCompletionTokens() != null - ? parameters.maxCompletionTokens().longValue() - : null, - parameters != null ? parameters.temperature() : null, - parameters != null ? parameters.topP() : null); - } - - private OpenAIClient buildClient(OpenAiCompatibleConnection connection) { - final var builder = OpenAIOkHttpClient.builder().baseUrl(connection.endpoint()); - - final var auth = connection.authentication(); - final var apiKey = - auth != null && StringUtils.isNotBlank(auth.apiKey()) ? auth.apiKey() : "no-key"; - builder.apiKey(apiKey); - - if (connection.headers() != null && !connection.headers().isEmpty()) { - connection.headers().forEach(builder::putHeader); - } - if (connection.queryParameters() != null && !connection.queryParameters().isEmpty()) { - connection.queryParameters().forEach(builder::putQueryParam); - } - - final var timeout = resolveTimeout(connection); - if (timeout != null) { - builder.timeout(timeout); - } - - return builder.build(); - } - - @Nullable - private Duration resolveTimeout(OpenAiCompatibleConnection connection) { - return Optional.ofNullable(connection.timeouts()).map(t -> t.timeout()).orElse(defaultTimeout); - } -} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/AzureOpenAiProviderConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/AzureOpenAiProviderConfiguration.java deleted file mode 100644 index 908192082be..00000000000 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/AzureOpenAiProviderConfiguration.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH - * under one or more contributor license agreements. Licensed under a proprietary license. - * See the License.txt file for more information. You may not use this file - * except in compliance with the proprietary license. - */ -package io.camunda.connector.agenticai.aiagent.model.request.provider; - -import static io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration.AZURE_OPENAI_ID; - -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.camunda.connector.agenticai.aiagent.model.request.provider.shared.HttpUrl; -import io.camunda.connector.agenticai.aiagent.model.request.provider.shared.TimeoutConfiguration; -import io.camunda.connector.api.annotation.FEEL; -import io.camunda.connector.generator.java.annotation.FeelMode; -import io.camunda.connector.generator.java.annotation.TemplateDiscriminatorProperty; -import io.camunda.connector.generator.java.annotation.TemplateProperty; -import io.camunda.connector.generator.java.annotation.TemplateSubType; -import jakarta.validation.Valid; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; - -@TemplateSubType(id = AZURE_OPENAI_ID, label = "Azure OpenAI") -public record AzureOpenAiProviderConfiguration(@Valid @NotNull AzureOpenAiConnection azureOpenAi) - implements ProviderConfiguration { - - @TemplateProperty(ignore = true) - public static final String AZURE_OPENAI_ID = "azureOpenAi"; - - @Override - public String providerType() { - return AZURE_OPENAI_ID; - } - - public record AzureOpenAiConnection( - @NotBlank - @HttpUrl - @FEEL - @TemplateProperty( - group = "provider", - description = - "Specify Azure OpenAI endpoint. Details in the documentation.", - constraints = @TemplateProperty.PropertyConstraints(notEmpty = true)) - String endpoint, - @Valid @NotNull AzureAuthentication authentication, - @Valid TimeoutConfiguration timeouts, - @Valid @NotNull AzureOpenAiModel model) {} - - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") - @JsonSubTypes({ - @JsonSubTypes.Type( - value = AzureAuthentication.AzureApiKeyAuthentication.class, - name = "apiKey"), - @JsonSubTypes.Type( - value = AzureAuthentication.AzureClientCredentialsAuthentication.class, - name = "clientCredentials") - }) - @TemplateDiscriminatorProperty( - label = "Authentication", - group = "provider", - name = "type", - defaultValue = "apiKey", - description = "Specify the Azure OpenAI authentication strategy.") - public sealed interface AzureAuthentication { - @TemplateSubType(id = "apiKey", label = "API key") - record AzureApiKeyAuthentication( - @NotBlank - @TemplateProperty( - group = "provider", - label = "API key", - type = TemplateProperty.PropertyType.String, - feel = FeelMode.optional, - constraints = @TemplateProperty.PropertyConstraints(notEmpty = true)) - String apiKey) - implements AzureAuthentication { - - @Override - public @NotNull String toString() { - return "AzureApiKeyAuthentication{apiKey=[REDACTED]}"; - } - } - - @TemplateSubType(id = "clientCredentials", label = "Client credentials") - record AzureClientCredentialsAuthentication( - @NotBlank - @TemplateProperty( - group = "provider", - label = "Client ID", - description = "ID of a Microsoft Entra application", - type = TemplateProperty.PropertyType.String, - feel = FeelMode.optional, - constraints = @TemplateProperty.PropertyConstraints(notEmpty = true)) - String clientId, - @NotBlank - @TemplateProperty( - group = "provider", - label = "Client secret", - description = "Secret of a Microsoft Entra application", - type = TemplateProperty.PropertyType.String, - feel = FeelMode.optional, - constraints = @TemplateProperty.PropertyConstraints(notEmpty = true)) - String clientSecret, - @NotBlank - @TemplateProperty( - group = "provider", - label = "Tenant ID", - description = - "ID of a Microsoft Entra tenant. Details in the documentation.", - type = TemplateProperty.PropertyType.String, - feel = FeelMode.optional) - String tenantId, - @TemplateProperty( - group = "provider", - label = "Authority host", - description = - "Authority host URL for the Microsoft Entra application. Defaults to https://login.microsoftonline.com. This can also contain an OAuth 2.0 token endpoint.", - type = TemplateProperty.PropertyType.String, - feel = FeelMode.optional, - optional = true) - String authorityHost) - implements AzureAuthentication { - - @Override - public String toString() { - return "AzureClientCredentialsAuthentication{clientId=%s, clientSecret=[REDACTED], tenantId=%s, authorityHost=%s}" - .formatted(clientId, tenantId, authorityHost); - } - } - } - - public record AzureOpenAiModel( - @NotBlank - @TemplateProperty( - group = "model", - label = "Model deployment name", - description = - "Specify the model deployment name. Details in the documentation.", - type = TemplateProperty.PropertyType.String, - feel = FeelMode.optional, - constraints = @TemplateProperty.PropertyConstraints(notEmpty = true)) - String deploymentName, - @Valid AzureOpenAiModel.AzureOpenAiModelParameters parameters) { - - public record AzureOpenAiModelParameters( - @Min(0) - @TemplateProperty( - group = "model", - label = "Maximum tokens", - tooltip = - "The maximum number of tokens per request to generate before stopping.

        Details in the documentation.", - type = TemplateProperty.PropertyType.Number, - feel = FeelMode.required, - optional = true) - Integer maxTokens, - @Min(0) - @TemplateProperty( - group = "model", - label = "Temperature", - tooltip = - "Floating point number between 0 and 2. The higher the number, the more randomness will be injected into the response.

        Details in the documentation.", - type = TemplateProperty.PropertyType.Number, - feel = FeelMode.required, - optional = true) - Double temperature, - @Min(0) - @TemplateProperty( - group = "model", - label = "top P", - tooltip = - "Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", - type = TemplateProperty.PropertyType.Number, - feel = FeelMode.required, - optional = true) - Double topP) {} - } -} diff --git a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiCompatibleProviderConfiguration.java b/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiCompatibleProviderConfiguration.java deleted file mode 100644 index 32465e91931..00000000000 --- a/connectors/agentic-ai/src/main/java/io/camunda/connector/agenticai/aiagent/model/request/provider/OpenAiCompatibleProviderConfiguration.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH - * under one or more contributor license agreements. Licensed under a proprietary license. - * See the License.txt file for more information. You may not use this file - * except in compliance with the proprietary license. - */ -package io.camunda.connector.agenticai.aiagent.model.request.provider; - -import static io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration.OPENAI_COMPATIBLE_ID; - -import io.camunda.connector.agenticai.aiagent.model.request.provider.shared.HttpUrl; -import io.camunda.connector.agenticai.aiagent.model.request.provider.shared.TimeoutConfiguration; -import io.camunda.connector.api.annotation.FEEL; -import io.camunda.connector.generator.java.annotation.FeelMode; -import io.camunda.connector.generator.java.annotation.TemplateProperty; -import io.camunda.connector.generator.java.annotation.TemplateSubType; -import jakarta.validation.Valid; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import java.util.Map; - -@TemplateSubType(id = OPENAI_COMPATIBLE_ID, label = "OpenAI Compatible") -public record OpenAiCompatibleProviderConfiguration( - @Valid @NotNull OpenAiCompatibleConnection openaiCompatible) implements ProviderConfiguration { - - @TemplateProperty(ignore = true) - public static final String OPENAI_COMPATIBLE_ID = "openaiCompatible"; - - @Override - public String providerType() { - return OPENAI_COMPATIBLE_ID; - } - - public record OpenAiCompatibleConnection( - @NotBlank - @HttpUrl - @TemplateProperty( - group = "provider", - label = "API endpoint", - tooltip = "Specify an endpoint to use the connector with an OpenAI compatible API. ", - type = TemplateProperty.PropertyType.String, - feel = FeelMode.optional, - constraints = @TemplateProperty.PropertyConstraints(notEmpty = true)) - String endpoint, - @Valid OpenAiCompatibleAuthentication authentication, - @FEEL - @TemplateProperty( - group = "provider", - label = "Headers", - description = "Map of HTTP headers to add to the request.", - feel = FeelMode.required, - optional = true) - Map headers, - @FEEL - @TemplateProperty( - group = "provider", - label = "Query Parameters", - description = "Map of query parameters to add to the request URL.", - feel = FeelMode.required, - optional = true) - @Valid - Map<@NotBlank String, String> queryParameters, - @Valid TimeoutConfiguration timeouts, - @Valid @NotNull OpenAiCompatibleModel model) {} - - public record OpenAiCompatibleAuthentication( - @TemplateProperty( - group = "provider", - label = "API key", - tooltip = - "Leave blank if using HTTP headers for authentication.
        If an Authorization header is specified in the headers, then the API key is ignored.", - type = TemplateProperty.PropertyType.String, - feel = FeelMode.optional, - optional = true) - String apiKey) { - - @Override - public String toString() { - return "OpenAiCompatibleAuthentication{apiKey=[REDACTED]}"; - } - } - - public record OpenAiCompatibleModel( - @NotBlank - @TemplateProperty( - group = "model", - label = "Model", - description = - "Specify the model ID. Details in the documentation.", - type = TemplateProperty.PropertyType.String, - feel = FeelMode.optional, - defaultValue = "gpt-4o", - defaultValueType = TemplateProperty.DefaultValueType.String, - constraints = @TemplateProperty.PropertyConstraints(notEmpty = true)) - String model, - @Valid OpenAiCompatibleModel.OpenAiCompatibleModelParameters parameters) { - - public record OpenAiCompatibleModelParameters( - @Min(0) - @TemplateProperty( - group = "model", - label = "Maximum completion tokens", - tooltip = - "The maximum number of tokens per request to generate before stopping.

        Details in the documentation.", - type = TemplateProperty.PropertyType.Number, - feel = FeelMode.required, - optional = true) - Integer maxCompletionTokens, - @Min(0) - @TemplateProperty( - group = "model", - label = "Temperature", - tooltip = - "Floating point number between 0 and 2. The higher the number, the more randomness will be injected into the response.

        Details in the documentation.", - type = TemplateProperty.PropertyType.Number, - feel = FeelMode.required, - optional = true) - Double temperature, - @Min(0) - @TemplateProperty( - group = "model", - label = "top P", - tooltip = - "Recommended for advanced use cases only (you usually only need to use temperature).

        Details in the documentation.", - type = TemplateProperty.PropertyType.Number, - feel = FeelMode.required, - optional = true) - Double topP, - @FEEL - @TemplateProperty( - group = "model", - label = "Custom parameters", - description = "Map of additional request parameters to include.", - feel = FeelMode.required, - optional = true) - Map customParameters) {} - } -} diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/AzureOpenAiChatModelProviderTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/AzureOpenAiChatModelProviderTest.java index d08bbfb1ffb..87c28f4ed9a 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/AzureOpenAiChatModelProviderTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/AzureOpenAiChatModelProviderTest.java @@ -23,11 +23,14 @@ import dev.langchain4j.model.azure.AzureOpenAiChatModel; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelHttpProxySupport; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProviderTestSupport.ResultCaptor; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration.AzureAuthentication.AzureApiKeyAuthentication; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration.AzureAuthentication.AzureClientCredentialsAuthentication; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration.AzureOpenAiConnection; -import io.camunda.connector.agenticai.aiagent.model.request.provider.AzureOpenAiProviderConfiguration.AzureOpenAiModel.AzureOpenAiModelParameters; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.ApiFamily; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiAuthentication.OpenAiApiKeyAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiAuthentication.OpenAiClientCredentialsAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiBackend; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiConnection; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiModel; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiModel.OpenAiModelParameters; import io.camunda.connector.agenticai.aiagent.model.request.provider.shared.TimeoutConfiguration; import io.camunda.connector.http.client.client.jdk.proxy.JdkHttpClientProxyConfigurator; import io.camunda.connector.http.client.proxy.ProxyConfiguration; @@ -56,8 +59,8 @@ class AzureOpenAiChatModelProviderTest { private static final String CLIENT_SECRET = "clientSecret"; private static final String TENANT_ID = "tenantId"; - private static final AzureOpenAiModelParameters DEFAULT_MODEL_PARAMETERS = - new AzureOpenAiModelParameters(10, 1.0, 0.8); + private static final OpenAiModelParameters DEFAULT_MODEL_PARAMETERS = + new OpenAiModelParameters(10, 1.0, 0.8, null); private final ProxyConfiguration proxyConfiguration = ProxyConfiguration.NONE; private final ChatModelHttpProxySupport proxySupport = @@ -73,20 +76,23 @@ class AzureOpenAiChatModelProviderTest { @Test void createsAzureOpenAiChatModelWithApiKey() { final var providerConfig = - new AzureOpenAiProviderConfiguration( - new AzureOpenAiConnection( - AZURE_OPENAI_ENDPOINT, - new AzureApiKeyAuthentication(AZURE_OPENAI_API_KEY), + new OpenAiProviderConfiguration( + new OpenAiConnection( + OpenAiBackend.FOUNDRY, + new OpenAiApiKeyAuthentication(AZURE_OPENAI_API_KEY, null, null), MODEL_TIMEOUT, - new AzureOpenAiProviderConfiguration.AzureOpenAiModel( - AZURE_OPENAI_DEPLOYMENT_NAME, DEFAULT_MODEL_PARAMETERS))); + new OpenAiModel(AZURE_OPENAI_DEPLOYMENT_NAME, DEFAULT_MODEL_PARAMETERS), + ApiFamily.COMPLETIONS, + AZURE_OPENAI_ENDPOINT, + null, + null)); testAzureOpenAiChatModelBuilder( providerConfig, (builder) -> { verify(builder).timeout(MODEL_TIMEOUT.timeout()); verify(builder).apiKey(AZURE_OPENAI_API_KEY); - verify(builder).maxTokens(DEFAULT_MODEL_PARAMETERS.maxTokens()); + verify(builder).maxTokens(DEFAULT_MODEL_PARAMETERS.maxCompletionTokens()); verify(builder).temperature(DEFAULT_MODEL_PARAMETERS.temperature()); verify(builder).topP(DEFAULT_MODEL_PARAMETERS.topP()); verify(builder, never()).tokenCredential(any()); @@ -98,20 +104,23 @@ void createsAzureOpenAiChatModelWithApiKey() { @ValueSource(strings = {"https://some-authortiy-host"}) void createsAzureOpenAiChatModelWithClientCredentials(String authorityHost) { final var providerConfig = - new AzureOpenAiProviderConfiguration( - new AzureOpenAiConnection( - AZURE_OPENAI_ENDPOINT, - new AzureClientCredentialsAuthentication( + new OpenAiProviderConfiguration( + new OpenAiConnection( + OpenAiBackend.FOUNDRY, + new OpenAiClientCredentialsAuthentication( CLIENT_ID, CLIENT_SECRET, TENANT_ID, authorityHost), MODEL_TIMEOUT, - new AzureOpenAiProviderConfiguration.AzureOpenAiModel( - AZURE_OPENAI_DEPLOYMENT_NAME, DEFAULT_MODEL_PARAMETERS))); + new OpenAiModel(AZURE_OPENAI_DEPLOYMENT_NAME, DEFAULT_MODEL_PARAMETERS), + ApiFamily.COMPLETIONS, + AZURE_OPENAI_ENDPOINT, + null, + null)); testAzureOpenAiChatModelBuilder( providerConfig, (builder) -> { verify(builder, never()).apiKey(any()); - verify(builder).maxTokens(DEFAULT_MODEL_PARAMETERS.maxTokens()); + verify(builder).maxTokens(DEFAULT_MODEL_PARAMETERS.maxCompletionTokens()); verify(builder).temperature(DEFAULT_MODEL_PARAMETERS.temperature()); verify(builder).topP(DEFAULT_MODEL_PARAMETERS.topP()); verify(builder).tokenCredential(tokenCredentialsCapture.capture()); @@ -123,16 +132,19 @@ void createsAzureOpenAiChatModelWithClientCredentials(String authorityHost) { @ParameterizedTest @NullSource @MethodSource("nullModelParameters") - void createsAzureOpenAiChatModelWithNullModelParameters( - AzureOpenAiModelParameters modelParameters) { + void createsAzureOpenAiChatModelWithNullModelParameters(OpenAiModelParameters modelParameters) { final var providerConfig = - new AzureOpenAiProviderConfiguration( - new AzureOpenAiConnection( - AZURE_OPENAI_ENDPOINT, - new AzureClientCredentialsAuthentication(CLIENT_ID, CLIENT_SECRET, TENANT_ID, null), + new OpenAiProviderConfiguration( + new OpenAiConnection( + OpenAiBackend.FOUNDRY, + new OpenAiClientCredentialsAuthentication( + CLIENT_ID, CLIENT_SECRET, TENANT_ID, null), MODEL_TIMEOUT, - new AzureOpenAiProviderConfiguration.AzureOpenAiModel( - AZURE_OPENAI_DEPLOYMENT_NAME, modelParameters))); + new OpenAiModel(AZURE_OPENAI_DEPLOYMENT_NAME, modelParameters), + ApiFamily.COMPLETIONS, + AZURE_OPENAI_ENDPOINT, + null, + null)); testAzureOpenAiChatModelBuilder( providerConfig, @@ -149,20 +161,24 @@ void createsAzureOpenAiChatModelWithNullModelParameters( "io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProviderTestSupport#defaultTimeoutYieldingConfigs") void createsAzureOpenAiChatModelWithUnspecifiedTimeouts(TimeoutConfiguration timeouts) { final var providerConfig = - new AzureOpenAiProviderConfiguration( - new AzureOpenAiConnection( - AZURE_OPENAI_ENDPOINT, - new AzureClientCredentialsAuthentication(CLIENT_ID, CLIENT_SECRET, TENANT_ID, null), + new OpenAiProviderConfiguration( + new OpenAiConnection( + OpenAiBackend.FOUNDRY, + new OpenAiClientCredentialsAuthentication( + CLIENT_ID, CLIENT_SECRET, TENANT_ID, null), timeouts, - new AzureOpenAiProviderConfiguration.AzureOpenAiModel( - AZURE_OPENAI_DEPLOYMENT_NAME, null))); + new OpenAiModel(AZURE_OPENAI_DEPLOYMENT_NAME, null), + ApiFamily.COMPLETIONS, + AZURE_OPENAI_ENDPOINT, + null, + null)); testAzureOpenAiChatModelBuilder( providerConfig, (builder) -> verify(builder).timeout(Duration.ofMinutes(3))); } private void testAzureOpenAiChatModelBuilder( - AzureOpenAiProviderConfiguration providerConfig, + OpenAiProviderConfiguration providerConfig, ThrowingConsumer builderAssertions) { final var chatModelBuilder = spy(AzureOpenAiChatModel.builder()); final var chatModelResultCaptor = new ResultCaptor(); @@ -183,7 +199,7 @@ private void testAzureOpenAiChatModelBuilder( } } - static Stream nullModelParameters() { - return Stream.of(new AzureOpenAiModelParameters(null, null, null)); + static Stream nullModelParameters() { + return Stream.of(new OpenAiModelParameters(null, null, null, null)); } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiChatModelProviderTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiChatModelProviderTest.java index cbc36d1b727..19339d13e76 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiChatModelProviderTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiChatModelProviderTest.java @@ -22,6 +22,7 @@ import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelHttpProxySupport; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProviderTestSupport.ResultCaptor; import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiAuthentication.OpenAiApiKeyAuthentication; import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiConnection; import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiModel.OpenAiModelParameters; import io.camunda.connector.agenticai.aiagent.model.request.provider.shared.TimeoutConfiguration; @@ -48,7 +49,7 @@ class OpenAiChatModelProviderTest { private static final String OPEN_AI_MODEL = "openAiModel"; private static final OpenAiModelParameters DEFAULT_MODEL_PARAMETERS = - new OpenAiModelParameters(10, 1.0, 0.8); + new OpenAiModelParameters(10, 1.0, 0.8, null); private final ProxyConfiguration proxyConfiguration = ProxyConfiguration.NONE; private final ChatModelHttpProxySupport proxySupport = @@ -66,7 +67,7 @@ void createsOpenAiChatModel() { final var providerConfig = new OpenAiProviderConfiguration( new OpenAiConnection( - new OpenAiProviderConfiguration.OpenAiAuthentication(OPEN_AI_API_KEY, null, null), + new OpenAiApiKeyAuthentication(OPEN_AI_API_KEY, null, null), MODEL_TIMEOUT, new OpenAiProviderConfiguration.OpenAiModel( OPEN_AI_MODEL, DEFAULT_MODEL_PARAMETERS))); @@ -97,8 +98,7 @@ void createsOpenAiChatModelWithCustomOrganizationAndProjectIds() { final var providerConfig = new OpenAiProviderConfiguration( new OpenAiConnection( - new OpenAiProviderConfiguration.OpenAiAuthentication( - OPEN_AI_API_KEY, "MY_ORG_ID", "MY_PROJECT_ID"), + new OpenAiApiKeyAuthentication(OPEN_AI_API_KEY, "MY_ORG_ID", "MY_PROJECT_ID"), MODEL_TIMEOUT, new OpenAiProviderConfiguration.OpenAiModel( OPEN_AI_MODEL, DEFAULT_MODEL_PARAMETERS))); @@ -118,7 +118,7 @@ void createsOpenAiChatModelWithNullModelParameters(OpenAiModelParameters modelPa final var providerConfig = new OpenAiProviderConfiguration( new OpenAiConnection( - new OpenAiProviderConfiguration.OpenAiAuthentication(OPEN_AI_API_KEY, null, null), + new OpenAiApiKeyAuthentication(OPEN_AI_API_KEY, null, null), MODEL_TIMEOUT, new OpenAiProviderConfiguration.OpenAiModel(OPEN_AI_MODEL, modelParameters))); @@ -148,7 +148,7 @@ void createsOpenAiChatModelWithUnspecifiedTimeouts(TimeoutConfiguration timeouts final var providerConfig = new OpenAiProviderConfiguration( new OpenAiConnection( - new OpenAiProviderConfiguration.OpenAiAuthentication(OPEN_AI_API_KEY, null, null), + new OpenAiApiKeyAuthentication(OPEN_AI_API_KEY, null, null), timeouts, new OpenAiProviderConfiguration.OpenAiModel(OPEN_AI_MODEL, null))); @@ -177,6 +177,6 @@ private void testOpenAiChatModelBuilder( } static Stream nullModelParameters() { - return Stream.of(new OpenAiModelParameters(null, null, null)); + return Stream.of(new OpenAiModelParameters(null, null, null, null)); } } diff --git a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiCompatibleChatModelProviderTest.java b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiCompatibleChatModelProviderTest.java index 8c460a60809..04638fc63f0 100644 --- a/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiCompatibleChatModelProviderTest.java +++ b/connectors/agentic-ai/src/test/java/io/camunda/connector/agenticai/aiagent/framework/langchain4j/provider/OpenAiCompatibleChatModelProviderTest.java @@ -23,9 +23,13 @@ import dev.langchain4j.model.openai.OpenAiChatRequestParameters; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.ChatModelHttpProxySupport; import io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProviderTestSupport.ResultCaptor; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration.OpenAiCompatibleConnection; -import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiCompatibleProviderConfiguration.OpenAiCompatibleModel.OpenAiCompatibleModelParameters; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.ApiFamily; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiAuthentication.OpenAiApiKeyAuthentication; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiBackend; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiConnection; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiModel; +import io.camunda.connector.agenticai.aiagent.model.request.provider.OpenAiProviderConfiguration.OpenAiModel.OpenAiModelParameters; import io.camunda.connector.agenticai.aiagent.model.request.provider.shared.TimeoutConfiguration; import io.camunda.connector.http.client.client.jdk.proxy.JdkHttpClientProxyConfigurator; import io.camunda.connector.http.client.proxy.ProxyConfiguration; @@ -52,8 +56,8 @@ class OpenAiCompatibleChatModelProviderTest { private static final String ENDPOINT = "https://compatible.local/v1"; private static final String MODEL = "some-compatible-model"; - private static final OpenAiCompatibleModelParameters DEFAULT_MODEL_PARAMETERS = - new OpenAiCompatibleModelParameters(10, 1.0, 0.8, Map.of("my-param", "my-value")); + private static final OpenAiModelParameters DEFAULT_MODEL_PARAMETERS = + new OpenAiModelParameters(10, 1.0, 0.8, Map.of("my-param", "my-value")); private final ProxyConfiguration proxyConfiguration = ProxyConfiguration.NONE; private final ChatModelHttpProxySupport proxySupport = @@ -66,18 +70,33 @@ class OpenAiCompatibleChatModelProviderTest { @Captor private ArgumentCaptor modelParametersArgumentCaptor; + private OpenAiProviderConfiguration makeConfig( + String apiKey, + Map headers, + Map queryParams, + TimeoutConfiguration timeouts, + OpenAiModelParameters modelParameters) { + return new OpenAiProviderConfiguration( + new OpenAiConnection( + OpenAiBackend.CUSTOM, + new OpenAiApiKeyAuthentication(apiKey, null, null), + timeouts, + new OpenAiModel(MODEL, modelParameters), + ApiFamily.COMPLETIONS, + ENDPOINT, + headers, + queryParams)); + } + @Test void createsOpenAiCompatibleChatModelWithApiKeyAndHeaders() { final var providerConfig = - new OpenAiCompatibleProviderConfiguration( - new OpenAiCompatibleConnection( - ENDPOINT, - new OpenAiCompatibleProviderConfiguration.OpenAiCompatibleAuthentication(API_KEY), - Map.of("my-header", "my-value"), - null, - MODEL_TIMEOUT, - new OpenAiCompatibleProviderConfiguration.OpenAiCompatibleModel( - MODEL, DEFAULT_MODEL_PARAMETERS))); + makeConfig( + API_KEY, + Map.of("my-header", "my-value"), + null, + MODEL_TIMEOUT, + DEFAULT_MODEL_PARAMETERS); testOpenAiCompatibleChatModelBuilder( providerConfig, @@ -108,15 +127,7 @@ void createsOpenAiCompatibleChatModelWithApiKeyAndHeaders() { @Test void createsOpenAiCompatibleChatModelWithoutApiKey() { final var providerConfig = - new OpenAiCompatibleProviderConfiguration( - new OpenAiCompatibleConnection( - ENDPOINT, - new OpenAiCompatibleProviderConfiguration.OpenAiCompatibleAuthentication(null), - null, - null, - MODEL_TIMEOUT, - new OpenAiCompatibleProviderConfiguration.OpenAiCompatibleModel( - MODEL, DEFAULT_MODEL_PARAMETERS))); + makeConfig(null, null, null, MODEL_TIMEOUT, DEFAULT_MODEL_PARAMETERS); testOpenAiCompatibleChatModelBuilder( providerConfig, @@ -132,17 +143,9 @@ void createsOpenAiCompatibleChatModelWithoutApiKey() { @NullSource @MethodSource("nullModelParameters") void createsOpenAiCompatibleChatModelWithNullModelParameters( - OpenAiCompatibleModelParameters modelParameters) { + OpenAiModelParameters modelParameters) { final var providerConfig = - new OpenAiCompatibleProviderConfiguration( - new OpenAiCompatibleConnection( - ENDPOINT, - new OpenAiCompatibleProviderConfiguration.OpenAiCompatibleAuthentication(API_KEY), - Map.of(), - Map.of(), - MODEL_TIMEOUT, - new OpenAiCompatibleProviderConfiguration.OpenAiCompatibleModel( - MODEL, modelParameters))); + makeConfig(API_KEY, Map.of(), Map.of(), MODEL_TIMEOUT, modelParameters); testOpenAiCompatibleChatModelBuilder( providerConfig, @@ -168,15 +171,7 @@ void createsOpenAiCompatibleChatModelWithNullModelParameters( @MethodSource( "io.camunda.connector.agenticai.aiagent.framework.langchain4j.provider.ChatModelProviderTestSupport#defaultTimeoutYieldingConfigs") void createsOpenAiCompatibleChatModelWithUnspecifiedTimeouts(TimeoutConfiguration timeouts) { - final var providerConfig = - new OpenAiCompatibleProviderConfiguration( - new OpenAiCompatibleConnection( - ENDPOINT, - new OpenAiCompatibleProviderConfiguration.OpenAiCompatibleAuthentication(API_KEY), - Map.of(), - Map.of(), - timeouts, - new OpenAiCompatibleProviderConfiguration.OpenAiCompatibleModel(MODEL, null))); + final var providerConfig = makeConfig(API_KEY, Map.of(), Map.of(), timeouts, null); testOpenAiCompatibleChatModelBuilder( providerConfig, (builder) -> verify(builder).timeout(Duration.ofMinutes(3))); @@ -186,15 +181,12 @@ void createsOpenAiCompatibleChatModelWithUnspecifiedTimeouts(TimeoutConfiguratio void createsOpenAiCompatibleChatModelWithApiKeyAndAuthorizationHeader() { final var authHeaderValue = "Bearer token123"; final var providerConfig = - new OpenAiCompatibleProviderConfiguration( - new OpenAiCompatibleConnection( - ENDPOINT, - new OpenAiCompatibleProviderConfiguration.OpenAiCompatibleAuthentication(API_KEY), - Map.of("Authorization", authHeaderValue), - Collections.emptyMap(), - MODEL_TIMEOUT, - new OpenAiCompatibleProviderConfiguration.OpenAiCompatibleModel( - MODEL, DEFAULT_MODEL_PARAMETERS))); + makeConfig( + API_KEY, + Map.of("Authorization", authHeaderValue), + Collections.emptyMap(), + MODEL_TIMEOUT, + DEFAULT_MODEL_PARAMETERS); testOpenAiCompatibleChatModelBuilder( providerConfig, @@ -218,15 +210,12 @@ void createsOpenAiCompatibleChatModelWithApiKeyAndHeaderAndQueryParameters() { final var authHeaderValue = "Bearer token123"; final var customQueryParameters = Map.of("foo", "bar", "foo2", "bar2"); final var providerConfig = - new OpenAiCompatibleProviderConfiguration( - new OpenAiCompatibleConnection( - ENDPOINT, - new OpenAiCompatibleProviderConfiguration.OpenAiCompatibleAuthentication(API_KEY), - Map.of("Authorization", authHeaderValue), - customQueryParameters, - null, - new OpenAiCompatibleProviderConfiguration.OpenAiCompatibleModel( - MODEL, DEFAULT_MODEL_PARAMETERS))); + makeConfig( + API_KEY, + Map.of("Authorization", authHeaderValue), + customQueryParameters, + null, + DEFAULT_MODEL_PARAMETERS); testOpenAiCompatibleChatModelBuilder( providerConfig, @@ -248,7 +237,7 @@ void createsOpenAiCompatibleChatModelWithApiKeyAndHeaderAndQueryParameters() { } private void testOpenAiCompatibleChatModelBuilder( - OpenAiCompatibleProviderConfiguration providerConfig, + OpenAiProviderConfiguration providerConfig, ThrowingConsumer builderAssertions) { final var chatModelBuilder = spy(OpenAiChatModel.builder()); final var chatModelResultCaptor = new ResultCaptor(); @@ -267,7 +256,7 @@ private void testOpenAiCompatibleChatModelBuilder( } } - static Stream nullModelParameters() { - return Stream.of(new OpenAiCompatibleModelParameters(null, null, null, null)); + static Stream nullModelParameters() { + return Stream.of(new OpenAiModelParameters(null, null, null, null)); } } From 7b9061206c4833267bd51f17a1c6fffc84a63d90 Mon Sep 17 00:00:00 2001 From: Mathias Geat Date: Wed, 13 May 2026 13:05:47 +0200 Subject: [PATCH 81/81] docs(agentic-ai): catch up ADR-005 plan with E3+E4 and F as built MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase E3+E4 (ToolCallResultStrategy + native multimodal emission) and Phase F (provider configuration restructure + Jackson migration deserializer + element template v11→v12) have both landed on this branch since the last plan revision. Update the Actual state section, mark the affected per-phase headers as done, refresh the Critical files table, and note the one wiring deviation in Phase F (type-level @JsonDeserialize instead of a Jackson Module, because the connector runtime's @ConnectorsObjectMapper does not do Module bean discovery). ADR status gains a prototype note pointing at this PR while staying Proposed; the design is being re-landed on main as smaller follow-up issues, and Status will move to Implemented once Phase G + Phase H land there. --- .../docs/adr-005-implementation-plan.md | 107 +++++++++++++----- .../adr/005-replace-langchain4j-framework.md | 6 +- 2 files changed, 84 insertions(+), 29 deletions(-) diff --git a/connectors/agentic-ai/docs/adr-005-implementation-plan.md b/connectors/agentic-ai/docs/adr-005-implementation-plan.md index 4fcbd1b71bd..e5a3bdae041 100644 --- a/connectors/agentic-ai/docs/adr-005-implementation-plan.md +++ b/connectors/agentic-ai/docs/adr-005-implementation-plan.md @@ -13,11 +13,18 @@ before generalising leads to a smaller, better-shaped abstraction in Phase E. Re caching, Azure OpenAI, Anthropic cloud backends, Google GenAI and Bedrock-Converse are all explicitly **deferred** out of the first native cut. -Phase E is split into three sub-phases (E1, E2 done; E3+E4 combined — see §"Phase E" below). Phase E +**Plan revision (2026-05-13)** — branch state catch-up. Phase E3+E4 and Phase F have both +landed on `agentic-ai/custom-llm-layer`. The L4J bridge has been narrowed accordingly: native +factories own the `anthropic` (DIRECT backend) and `openai` (apiKey on all three backends) +discriminators; the bridge still serves `bedrock` and `googleGenAi`, plus the +`openai`/FOUNDRY+clientCredentials slot. Phase G (remaining cloud backends and native impls) +and Phase H (default-off bridge demotion) are still open. + +Phase E is split into three sub-phases (E1, E2, E3+E4 all done — see §"Phase E" below). Phase E is layered on top of PR #6999 (`agentic-ai-document-tool-call-results`); our branch was rebased onto that PR while it was in active review. -**Actual starting state** (`agentic-ai/custom-llm-layer`): +**Actual state** (`agentic-ai/custom-llm-layer`, 2026-05-13): - Phase 0 done: `AssistantMessage` gains `modelId`/`apiId`/`stopReason`/`usage`; `TokenUsage` gains cache and reasoning fields; `ReasoningContent` added to `Content` hierarchy; `contentBlocks` added to `ToolCallResult`; `AiFrameworkChatResponse#rawChatResponse()` dropped. @@ -25,21 +32,48 @@ onto that PR while it was in active review. `ChatClient`; LangChain4j wired as the bridge `ChatModelApi` for all six provider discriminators (one factory bean per discriminator). `AiFrameworkAdapter` and `AiFrameworkChatResponse` already removed from the source tree (ahead of the original Phase F schedule). -- **Phases B / C / D done**: native `AnthropicMessagesChatModelApi` (text-only), - `OpenAiChatCompletionsChatModelApi` (text-only), `OpenAiResponsesChatModelApi` - (text-only) plus the `apiFamily` switch on the `openai` discriminator and element template - bump 10 → 11. +- **Phases B / C / D done**: native `AnthropicMessagesChatModelApi` (text + image + PDF after E4), + `OpenAiChatCompletionsChatModelApi`, `OpenAiResponsesChatModelApi`, plus the + `apiFamily` switch on the `openai` discriminator and element template bump 10 → 11. - **Phase E1 / E2 done**: capability matrix loaded as Spring Boot config (bundled - `model-capabilities.yaml` registered via `EnvironmentPostProcessor`); each native impl now - consumes a `ModelCapabilities` resolved at factory time. -- Wire-format e2e regression tests added for the Anthropic Messages API, OpenAI Chat - Completions API and OpenAI Responses API (WireMock-based, currently exercising the native - impls). -- ADR-005 document committed. - -**End state**: `BaseAgentRequestHandler` calls `ChatClient`. Native `ChatModelApi` impls ship for -Anthropic Messages (direct), OpenAI Chat Completions, OpenAI Responses, Azure OpenAI, Anthropic -cloud backends, Google GenAI and Bedrock-Converse. LangChain4j survives only as an opt-in bridge. + `model-capabilities.yaml` registered as a low-precedence `PropertySource`); each native impl + now consumes a `ModelCapabilities` resolved at factory time. +- **Phase E3+E4 done**: `ToolCallResultStrategy` ships single-pass per-block routing for every + document in a chat request; native multimodal emission (images + PDFs) wired in all three + native impls; `AgentMessagesHandlerImpl` no longer extracts documents; modality detection + via `DocumentModality` MIME mapping (`Modality.PDF` renamed to `Modality.DOCUMENT`). +- **Phase F done**: `ProviderConfiguration` restructured by wire format. Anthropic gains a + sealed `AnthropicAuthentication` (`apiKey` / `clientCredentials`) plus + `AnthropicBackend { DIRECT, BEDROCK, VERTEX, FOUNDRY }`. OpenAI consolidates the three legacy + configs into a single `OpenAiProviderConfiguration` with `OpenAiBackend { OPENAI, FOUNDRY, + CUSTOM }`, FOUNDRY/CUSTOM `endpoint` required, headers/queryParameters/customParameters + scoped to CUSTOM. Google renamed `GoogleVertexAi` → `GoogleGenAi` with + `GoogleBackend { DEVELOPER_API, VERTEX }`. Bedrock validates non-Anthropic model IDs at + construction. `ProviderConfigurationDeserializer` wired via type-level `@JsonDeserialize` + (not via Jackson `Module`, because the connector runtime's `@ConnectorsObjectMapper` doesn't + do module discovery) rewrites all legacy shapes per the ADR migration table. Element + template bumped 11 → 12 with backend dropdowns + conditional auth groups. +- Wire-format e2e regression tests for the Anthropic Messages API, OpenAI Chat Completions + API, and OpenAI Responses API (WireMock-based) exercise the native impls. +- ADR-005 document + this implementation plan committed. + +**Still open**: +- **Phase G** — native impls for Anthropic cloud backends (BEDROCK / VERTEX / FOUNDRY), + OpenAI FOUNDRY with clientCredentials auth, native Google GenAI, native Bedrock-Converse. + The L4J bridge currently registers factories for `bedrock` and `googleGenAi`; the native + `OpenAiChatModelApiFactory` throws a "Phase G Azure SDK integration" placeholder when + building a FOUNDRY client with clientCredentials. The native + `AnthropicMessagesChatModelApiFactory` rejects any backend other than DIRECT. +- **Phase H** — demote the L4J bridge to opt-in (`camunda.connector.agenticai.framework=langchain4j` + is currently the default; it should default off). ADR status → Implemented. +- Deferred capabilities: **reasoning** (signed thinking blocks, encrypted reasoning items), + **prompt caching** (`cache_control` markers, `prompt_cache_key`), **audio + video modalities**, + **JDK `java.net.http.HttpClient` adapter** (replacing OkHttp transport). + +**Target end state**: `BaseAgentRequestHandler` calls `ChatClient`. Native `ChatModelApi` impls +ship for Anthropic Messages (DIRECT + cloud backends), OpenAI Chat Completions, OpenAI +Responses (OpenAI + Foundry + Custom), Google GenAI, and Bedrock-Converse. LangChain4j survives +only as an opt-in bridge. --- @@ -281,7 +315,7 @@ pass against the native impl. --- -## Phase E — Capability matrix + tool-result strategy + multimodality +## Phase E — Capability matrix + tool-result strategy + multimodality (done) **Plan revision (2026-05-07)** — Phase E is split into three sub-phases. E1 + E2 ship as independent commits; **E3 and E4 ship as one combined commit** because the strategy and the @@ -291,6 +325,13 @@ silently dropped between phases). Reasoning and prompt caching are explicitly ** of Phase E** to keep the cut focused; the matrix already declares the flags so the slot is reserved. +**Branch state (2026-05-13)**: all three sub-phases are landed on +`agentic-ai/custom-llm-layer`. E1 + E2 sections kept as built. E3+E4 description below +matches what actually shipped: single-pass `ToolCallResultStrategyImpl`, native multimodal +emission for images + PDFs on all three native impls, removal of the +`createDocumentMessageForToolResults` extraction site in `AgentMessagesHandlerImpl`, +`Modality.PDF` renamed to `Modality.DOCUMENT`. + Phase E is layered on top of the work from PR #6999 (`agentic-ai-document-tool-call-results`), which contributes the `ToolCallResultDocumentExtractor` (recursive walker over lists / maps / MCP-shaped content), per-handler extraction hooks on `GatewayToolHandler`, the synthetic @@ -360,7 +401,7 @@ always resolves under `openai-completions`. The L4J bridge keeps its own conserv (it stays as a fallback path; replacing it through the resolver is not worth the change at this point). -### Sub-phase E3 + E4 — `ToolCallResultStrategy` + native multimodal emission (combined) +### Sub-phase E3 + E4 — `ToolCallResultStrategy` + native multimodal emission (done) **Goal**: single-pass per-block routing for every document in a chat request (E3), plus the native multimodal emission paths in each provider impl that consume the routed @@ -495,11 +536,18 @@ inline-native path in E4. --- -## Phase F — `ProviderConfiguration` restructure + Jackson migration +## Phase F — `ProviderConfiguration` restructure + Jackson migration (done) **Goal**: introduce the canonical discriminator scheme without breaking saved process state. Element template version bump 11 → 12 (after D's bump). +**Branch state (2026-05-13)**: landed across commits `04150588f` → `77b5c9163`. The section +below describes the as-built result; everything aligns with the original design except for one +deliberate wiring choice: the migration deserializer is registered via type-level +`@JsonDeserialize` on `ProviderConfiguration` instead of via a Jackson `Module`, because the +connector runtime's `@ConnectorsObjectMapper` composes a hard-coded module list (no `Module` +bean discovery) — see the wiring note in the deserializer block below. + **Files to modify / create**: - `AnthropicProviderConfiguration`: add `AnthropicBackend { DIRECT, BEDROCK, VERTEX, FOUNDRY }`; conditional auth fields per backend. `AnthropicAuthentication` is converted from a flat record @@ -642,15 +690,18 @@ mvn test -pl connectors-e2e-test/connectors-e2e-test-agentic-ai # full suite ( |------|-------| | `BaseAgentRequestHandler.java:172–176` | A (cutover site, done) | | `framework/api/` SPI package | A (done) | -| `framework/ChatClientImpl.java` | A (done), E (capability + strategy wiring) | -| `framework/anthropic/` (new) | B, G (cloud backends) | -| `framework/openai/` (new) | C (Completions + factories), D (Responses), G (Azure) | -| `OpenAiProviderConfiguration.java` | D (apiFamily field) | -| `model/request/provider/*ProviderConfiguration.java` | F (canonical restructure) | -| `element-templates/agenticai-aiagent-outbound-connector.json` | D (v11), F (v12) | -| `element-templates/README.md` | D, F | -| `AgenticAiConnectorsAutoConfiguration.java` | A (done), B–G (provider imports) | -| `docs/adr/005-replace-langchain4j-framework.md` | H (status update) | +| `framework/ChatClientImpl.java` | A (done), E (capability + strategy wiring, done) | +| `framework/strategy/ToolCallResultStrategyImpl.java` | E3+E4 (done) | +| `framework/multimodal/DocumentModality.java` | E3+E4 (done — MIME → Modality mapping) | +| `framework/anthropic/` | B (done), G (cloud backends, open) | +| `framework/openai/` | C (done), D (done), G (FOUNDRY + clientCredentials, open) | +| `OpenAiProviderConfiguration.java` | D (apiFamily, done), F (backend + auth restructure, done) | +| `model/request/provider/*ProviderConfiguration.java` | F (done) | +| `model/request/provider/ProviderConfigurationDeserializer.java` | F (done — type-level `@JsonDeserialize`) | +| `element-templates/agenticai-aiagent-outbound-connector.json` | D (v11, done), F (v12, done) | +| `element-templates/README.md` | D (done), F (done) | +| `AgenticAiConnectorsAutoConfiguration.java` | A (done), B–F (provider imports, done), G (open) | +| `docs/adr/005-replace-langchain4j-framework.md` | H (status → Implemented when G+H land) | ## Reusable existing code diff --git a/connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md b/connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md index 56612cead3d..6dcdf7d6c15 100644 --- a/connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md +++ b/connectors/agentic-ai/docs/adr/005-replace-langchain4j-framework.md @@ -5,7 +5,11 @@ ## Status -**Proposed** +**Proposed** — design accepted; a working prototype implementing Phases 0, A–D, E (E1+E2+E3+E4), +and F lives on the hackdays branch `agentic-ai/custom-llm-layer` (PR #7151) for reference. The +prototype branch is not for merge; the design is being re-landed on `main` in smaller, +reviewable units tracked under the ADR-005 parent issue. Status will move to **Implemented** +once Phase G + Phase H land. ## Context and Problem Statement