feat(agent): add streaming and multimodal tool result support (#3912)

spring-ai-alibaba-agent-framework/README.md

@@ -11,6 +11,9 @@ Spring AI Alibaba Agent Framework is created for Java developers to quickly and
 * **Human In The Loop**
 * **A2A**
 * **Rich Model, Tool and MCP Support**
+* **Async Tool Execution** - AsyncToolCallback with parallel execution and cancellation support
+* **Streaming Tool Execution** - StreamingToolCallback for incremental result delivery via Flux<ToolResult>
+* **Multimodal Tool Results** - ToolResult supporting text, images, audio, video, and files
 
 ## Related Projects
 Spring AI Alibaba Agent Framework depends on the following projects:

spring-ai-alibaba-agent-framework/pom.xml

@@ -88,6 +88,12 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>io.projectreactor</groupId>
+            <artifactId>reactor-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>com.alibaba.cloud.ai</groupId>
             <artifactId>spring-ai-alibaba-starter-dashscope</artifactId>

spring-ai-alibaba-agent-framework/src/main/java/com/alibaba/cloud/ai/graph/agent/MessageToolCallResultConverter.java

@@ -15,42 +15,108 @@
  */
 package com.alibaba.cloud.ai.graph.agent;
 
+import com.alibaba.cloud.ai.graph.agent.tool.ToolResult;
 import org.springframework.ai.chat.messages.AssistantMessage;
+import org.springframework.ai.content.Media;
 import org.springframework.ai.tool.execution.ToolCallResultConverter;
 import org.springframework.ai.util.json.JsonParser;
 
 import org.springframework.lang.Nullable;
 import org.springframework.util.StringUtils;
 
 import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
 
 import org.apache.commons.collections4.CollectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+/**
+ * Converter for tool call results that supports text, multimodal content, and ToolResult.
+ *
+ * <p>Handles the following result types:</p>
+ * <ul>
+ *   <li>{@link ToolResult} - Rich results with text and/or media</li>
+ *   <li>{@link Media} - Single media content</li>
+ *   <li>{@link Collection} of {@link Media} - Multiple media content</li>
+ *   <li>{@link AssistantMessage} - Text or media content</li>
+ *   <li>Other objects - Serialized to JSON</li>
+ * </ul>
+ *
+ * @author disaster
+ * @since 1.0.0
+ */
 public class MessageToolCallResultConverter implements ToolCallResultConverter {
 
 	private static final Logger logger = LoggerFactory.getLogger(MessageToolCallResultConverter.class);
 
 	/**
-	 * Currently Spring AI ToolResponseMessage only supports text type, that's why the return type of this method is String.
-	 * More types like image/audio/video/file can be supported in the future.
+	 * Converts tool result to a string representation.
+	 * Supports ToolResult, Media, and other types.
 	 */
 	public String convert(@Nullable Object result, @Nullable Type returnType) {
 		if (returnType == Void.TYPE) {
 			logger.debug("The tool has no return type. Converting to conventional response.");
 			return JsonParser.toJson("Done");
-		} else if (result instanceof AssistantMessage assistantMessage) {
+		}
+
+		if (result == null) {
+			return "";
+		}
+
+		if (result instanceof String str) {
+			return str;
+		}
+
+		// Handle ToolResult - rich result model
+		if (result instanceof ToolResult toolResult) {
+			return toolResult.toStringResult();
+		}
+
+		// Handle single Media
+		if (result instanceof Media media) {
+			return serializeMedia(media);
+		}
+
+		// Handle collection of Media
+		if (result instanceof Collection<?> collection && !collection.isEmpty()) {
+			Object first = collection.iterator().next();
+			if (first instanceof Media) {
+				@SuppressWarnings("unchecked")
+				List<Media> mediaList = new ArrayList<>((Collection<Media>) collection);
+				return ToolResult.media(mediaList).toStringResult();
+			}
+		}
+
+		// Handle AssistantMessage
+		if (result instanceof AssistantMessage assistantMessage) {
 			if (StringUtils.hasLength(assistantMessage.getText())) {
+				// Check if there's also media content
+				if (CollectionUtils.isNotEmpty(assistantMessage.getMedia())) {
+					return ToolResult.mixed(assistantMessage.getText(), assistantMessage.getMedia()).toStringResult();
+				}
 				return assistantMessage.getText();
-			} else if (CollectionUtils.isNotEmpty(assistantMessage.getMedia())) {
-				throw new UnsupportedOperationException("Currently Spring AI ToolResponseMessage only supports text type, that's why the return type of this method is String. More types like image/audio/video/file can be supported in the future.");
+			}
+			else if (CollectionUtils.isNotEmpty(assistantMessage.getMedia())) {
+				// Media-only result
+				return ToolResult.media(assistantMessage.getMedia()).toStringResult();
 			}
 			logger.warn("The tool returned an empty AssistantMessage. Converting to conventional response.");
 			return JsonParser.toJson("Done");
-		} else {
-			logger.debug("Converting tool result to JSON.");
-			return JsonParser.toJson(result);
 		}
+
+		// Default: try JSON serialization
+		logger.debug("Converting tool result to JSON.");
+		return JsonParser.toJson(result);
 	}
+
+	/**
+	 * Serializes a single Media to ToolResult format.
+	 */
+	private String serializeMedia(Media media) {
+		return ToolResult.media(List.of(media)).toStringResult();
+	}
+
 }

spring-ai-alibaba-agent-framework/src/main/java/com/alibaba/cloud/ai/graph/agent/interceptor/ToolCallResponse.java

@@ -15,6 +15,7 @@
  */
 package com.alibaba.cloud.ai.graph.agent.interceptor;
 
+import com.alibaba.cloud.ai.graph.agent.tool.ToolResult;
 import org.springframework.ai.chat.messages.ToolResponseMessage;
 
 import java.util.Collections;
@@ -23,31 +24,57 @@
 
 /**
  * Response object for tool calls.
+ * Supports both simple string results and rich {@link ToolResult} with multimodal content.
  */
 public class ToolCallResponse {
 
 	private final String result;
+
 	private final String toolName;
+
 	private final String toolCallId;
+
 	private final String status;
+
 	private final Map<String, Object> metadata;
 
+	private final ToolResult richResult;
+
 	public ToolCallResponse(String result, String toolName, String toolCallId) {
-		this(result, toolName, toolCallId, null, null);
+		this(result, toolName, toolCallId, null, null, null);
+	}
+
+	public ToolCallResponse(String result, String toolName, String toolCallId, String status,
+			Map<String, Object> metadata) {
+		this(result, toolName, toolCallId, status, metadata, null);
 	}
 
-	public ToolCallResponse(String result, String toolName, String toolCallId, String status, Map<String, Object> metadata) {
+	public ToolCallResponse(String result, String toolName, String toolCallId, String status,
+			Map<String, Object> metadata, ToolResult richResult) {
 		this.result = result;
 		this.toolName = toolName;
 		this.toolCallId = toolCallId;
 		this.status = status;
 		this.metadata = metadata != null ? new HashMap<>(metadata) : Collections.emptyMap();
+		this.richResult = richResult;
 	}
 
 	public static ToolCallResponse of(String toolCallId, String toolName, String result) {
 		return new ToolCallResponse(result, toolName, toolCallId);
 	}
 
+	/**
+	 * Creates a response with a rich ToolResult. The string result is automatically
+	 * generated from the ToolResult for backward compatibility.
+	 * @param toolCallId the tool call ID
+	 * @param toolName the tool name
+	 * @param result the rich result
+	 * @return a new ToolCallResponse with rich result support
+	 */
+	public static ToolCallResponse ofRich(String toolCallId, String toolName, ToolResult result) {
+		return new ToolCallResponse(result.toStringResult(), toolName, toolCallId, "success", null, result);
+	}
+
 	/**
 	 * Creates an error response for a failed tool execution.
 	 * @param toolCallId the tool call ID
@@ -97,6 +124,22 @@ public Map<String, Object> getMetadata() {
 		return Collections.unmodifiableMap(metadata);
 	}
 
+	/**
+	 * Gets the rich result if available.
+	 * @return the ToolResult or null if not a rich result
+	 */
+	public ToolResult getRichResult() {
+		return richResult;
+	}
+
+	/**
+	 * Checks if this response has a rich result.
+	 * @return true if rich result is available
+	 */
+	public boolean hasRichResult() {
+		return richResult != null;
+	}
+
 	/**
 	 * Checks if this response represents an error.
 	 * @return true if this is an error response
@@ -110,12 +153,19 @@ public ToolResponseMessage.ToolResponse toToolResponse() {
 	}
 
 	public static class Builder {
+
 		private String content;
+
 		private String toolName;
+
 		private String toolCallId;
+
 		private String status;
+
 		private Map<String, Object> metadata;
 
+		private ToolResult richResult;
+
 		public Builder content(String content) {
 			this.content = content;
 			return this;
@@ -141,9 +191,23 @@ public Builder metadata(Map<String, Object> metadata) {
 			return this;
 		}
 
+		/**
+		 * Sets the rich result. Also updates content with the serialized result.
+		 * @param result the rich result
+		 * @return this builder
+		 */
+		public Builder richResult(ToolResult result) {
+			this.richResult = result;
+			if (result != null) {
+				this.content = result.toStringResult();
+			}
+			return this;
+		}
+
 		public ToolCallResponse build() {
-			return new ToolCallResponse(content, toolName, toolCallId, status, metadata);
+			return new ToolCallResponse(content, toolName, toolCallId, status, metadata, richResult);
 		}
+
 	}
 }