Merge remote-tracking branch 'upstream/main'

Author: vorticalbox

agent.go

@@ -636,94 +636,11 @@ func (a *agent) executeTools(ctx context.Context, allTools []AgentTool, toolCall
 	results := make([]ToolResultContent, 0, len(toolCalls))
 
 	for _, toolCall := range toolCalls {
-		// Skip invalid tool calls - create error result
-		if toolCall.Invalid {
-			result := ToolResultContent{
-				ToolCallID: toolCall.ToolCallID,
-				ToolName:   toolCall.ToolName,
-				Result: ToolResultOutputContentError{
-					Error: toolCall.ValidationError,
-				},
-				ProviderExecuted: false,
-			}
-			results = append(results, result)
-			if toolResultCallback != nil {
-				if err := toolResultCallback(result); err != nil {
-					return nil, err
-				}
-			}
-			continue
-		}
-
-		tool, exists := toolMap[toolCall.ToolName]
-		if !exists {
-			result := ToolResultContent{
-				ToolCallID: toolCall.ToolCallID,
-				ToolName:   toolCall.ToolName,
-				Result: ToolResultOutputContentError{
-					Error: errors.New("Error: Tool not found: " + toolCall.ToolName),
-				},
-				ProviderExecuted: false,
-			}
-			results = append(results, result)
-			if toolResultCallback != nil {
-				if err := toolResultCallback(result); err != nil {
-					return nil, err
-				}
-			}
-			continue
-		}
-
-		// Execute the tool
-		toolResult, err := tool.Run(ctx, ToolCall{
-			ID:    toolCall.ToolCallID,
-			Name:  toolCall.ToolName,
-			Input: toolCall.Input,
-		})
-		if err != nil {
-			result := ToolResultContent{
-				ToolCallID: toolCall.ToolCallID,
-				ToolName:   toolCall.ToolName,
-				Result: ToolResultOutputContentError{
-					Error: err,
-				},
-				ClientMetadata:   toolResult.Metadata,
-				ProviderExecuted: false,
-			}
-			if toolResultCallback != nil {
-				if cbErr := toolResultCallback(result); cbErr != nil {
-					return nil, cbErr
-				}
-			}
-			return nil, err
-		}
-
-		var result ToolResultContent
-		if toolResult.IsError {
-			result = ToolResultContent{
-				ToolCallID: toolCall.ToolCallID,
-				ToolName:   toolCall.ToolName,
-				Result: ToolResultOutputContentError{
-					Error: errors.New(toolResult.Content),
-				},
-				ClientMetadata:   toolResult.Metadata,
-				ProviderExecuted: false,
-			}
-		} else {
-			result = ToolResultContent{
-				ToolCallID: toolCall.ToolCallID,
-				ToolName:   toolCall.ToolName,
-				Result: ToolResultOutputContentText{
-					Text: toolResult.Content,
-				},
-				ClientMetadata:   toolResult.Metadata,
-				ProviderExecuted: false,
-			}
-		}
+		result, isCriticalError := a.executeSingleTool(ctx, toolMap, toolCall, toolResultCallback)
 		results = append(results, result)
-		if toolResultCallback != nil {
-			if err := toolResultCallback(result); err != nil {
-				return nil, err
+		if isCriticalError {
+			if errorResult, ok := result.Result.(ToolResultOutputContentError); ok && errorResult.Error != nil {
+				return nil, errorResult.Error
 			}
 		}
 	}
@@ -775,7 +692,6 @@ func (a *agent) executeSingleTool(ctx context.Context, toolMap map[string]AgentT
 		if toolResultCallback != nil {
 			_ = toolResultCallback(result)
 		}
-		// This is a critical error - tool.Run() failed
 		return result, true
 	}
 
@@ -784,6 +700,12 @@ func (a *agent) executeSingleTool(ctx context.Context, toolMap map[string]AgentT
 		result.Result = ToolResultOutputContentError{
 			Error: errors.New(toolResult.Content),
 		}
+	} else if toolResult.Type == "image" || toolResult.Type == "media" {
+		result.Result = ToolResultOutputContentMedia{
+			Data:      string(toolResult.Data),
+			MediaType: toolResult.MediaType,
+			Text:      toolResult.Content,
+		}
 	} else {
 		result.Result = ToolResultOutputContentText{
 			Text: toolResult.Content,
@@ -792,7 +714,6 @@ func (a *agent) executeSingleTool(ctx context.Context, toolMap map[string]AgentT
 	if toolResultCallback != nil {
 		_ = toolResultCallback(result)
 	}
-	// Not a critical error - tool ran successfully (even if it reported an error state)
 	return result, false
 }
 

agent_test.go

@@ -1535,3 +1535,236 @@ func TestToolCallRepair(t *testing.T) {
 		require.Contains(t, toolCalls[0].ValidationError.Error(), "invalid JSON input")
 	})
 }
+
+// Test media and image tool responses
+func TestAgent_MediaToolResponses(t *testing.T) {
+	t.Parallel()
+
+	imageData := []byte{0x89, 0x50, 0x4E, 0x47} // PNG header bytes
+	audioData := []byte{0x52, 0x49, 0x46, 0x46} // RIFF header bytes
+
+	t.Run("Image tool response", func(t *testing.T) {
+		t.Parallel()
+
+		imageTool := &mockTool{
+			name:        "generate_image",
+			description: "Generates an image",
+			executeFunc: func(ctx context.Context, call ToolCall) (ToolResponse, error) {
+				return NewImageResponse(imageData, "image/png"), nil
+			},
+		}
+
+		model := &mockLanguageModel{
+			generateFunc: func(ctx context.Context, call Call) (*Response, error) {
+				if len(call.Prompt) == 1 {
+					// First call - request image tool
+					return &Response{
+						Content: []Content{
+							ToolCallContent{
+								ToolCallID: "img-1",
+								ToolName:   "generate_image",
+								Input:      `{}`,
+							},
+						},
+						Usage:        Usage{TotalTokens: 10},
+						FinishReason: FinishReasonToolCalls,
+					}, nil
+				}
+				// Second call - after tool execution
+				return &Response{
+					Content:      []Content{TextContent{Text: "Image generated"}},
+					Usage:        Usage{TotalTokens: 20},
+					FinishReason: FinishReasonStop,
+				}, nil
+			},
+		}
+
+		agent := NewAgent(model, WithTools(imageTool), WithStopConditions(StepCountIs(3)))
+
+		result, err := agent.Generate(context.Background(), AgentCall{
+			Prompt: "Generate an image",
+		})
+
+		require.NoError(t, err)
+		require.NotNil(t, result)
+		require.Len(t, result.Steps, 2) // Tool call step + final response
+
+		// Check tool results in first step
+		toolResults := result.Steps[0].Content.ToolResults()
+		require.Len(t, toolResults, 1)
+
+		mediaResult, ok := toolResults[0].Result.(ToolResultOutputContentMedia)
+		require.True(t, ok, "Expected media result")
+		require.Equal(t, string(imageData), mediaResult.Data)
+		require.Equal(t, "image/png", mediaResult.MediaType)
+	})
+
+	t.Run("Media tool response (audio)", func(t *testing.T) {
+		t.Parallel()
+
+		audioTool := &mockTool{
+			name:        "generate_audio",
+			description: "Generates audio",
+			executeFunc: func(ctx context.Context, call ToolCall) (ToolResponse, error) {
+				return NewMediaResponse(audioData, "audio/wav"), nil
+			},
+		}
+
+		model := &mockLanguageModel{
+			generateFunc: func(ctx context.Context, call Call) (*Response, error) {
+				if len(call.Prompt) == 1 {
+					return &Response{
+						Content: []Content{
+							ToolCallContent{
+								ToolCallID: "audio-1",
+								ToolName:   "generate_audio",
+								Input:      `{}`,
+							},
+						},
+						Usage:        Usage{TotalTokens: 10},
+						FinishReason: FinishReasonToolCalls,
+					}, nil
+				}
+				return &Response{
+					Content:      []Content{TextContent{Text: "Audio generated"}},
+					Usage:        Usage{TotalTokens: 20},
+					FinishReason: FinishReasonStop,
+				}, nil
+			},
+		}
+
+		agent := NewAgent(model, WithTools(audioTool), WithStopConditions(StepCountIs(3)))
+
+		result, err := agent.Generate(context.Background(), AgentCall{
+			Prompt: "Generate audio",
+		})
+
+		require.NoError(t, err)
+		require.NotNil(t, result)
+
+		toolResults := result.Steps[0].Content.ToolResults()
+		require.Len(t, toolResults, 1)
+
+		mediaResult, ok := toolResults[0].Result.(ToolResultOutputContentMedia)
+		require.True(t, ok, "Expected media result")
+		require.Equal(t, string(audioData), mediaResult.Data)
+		require.Equal(t, "audio/wav", mediaResult.MediaType)
+	})
+
+	t.Run("Media response with text", func(t *testing.T) {
+		t.Parallel()
+
+		imageTool := &mockTool{
+			name:        "screenshot",
+			description: "Takes a screenshot",
+			executeFunc: func(ctx context.Context, call ToolCall) (ToolResponse, error) {
+				resp := NewImageResponse(imageData, "image/png")
+				resp.Content = "Screenshot captured successfully"
+				return resp, nil
+			},
+		}
+
+		model := &mockLanguageModel{
+			generateFunc: func(ctx context.Context, call Call) (*Response, error) {
+				if len(call.Prompt) == 1 {
+					return &Response{
+						Content: []Content{
+							ToolCallContent{
+								ToolCallID: "screen-1",
+								ToolName:   "screenshot",
+								Input:      `{}`,
+							},
+						},
+						Usage:        Usage{TotalTokens: 10},
+						FinishReason: FinishReasonToolCalls,
+					}, nil
+				}
+				return &Response{
+					Content:      []Content{TextContent{Text: "Done"}},
+					Usage:        Usage{TotalTokens: 20},
+					FinishReason: FinishReasonStop,
+				}, nil
+			},
+		}
+
+		agent := NewAgent(model, WithTools(imageTool), WithStopConditions(StepCountIs(3)))
+
+		result, err := agent.Generate(context.Background(), AgentCall{
+			Prompt: "Take a screenshot",
+		})
+
+		require.NoError(t, err)
+		require.NotNil(t, result)
+
+		toolResults := result.Steps[0].Content.ToolResults()
+		require.Len(t, toolResults, 1)
+
+		mediaResult, ok := toolResults[0].Result.(ToolResultOutputContentMedia)
+		require.True(t, ok, "Expected media result")
+		require.Equal(t, string(imageData), mediaResult.Data)
+		require.Equal(t, "image/png", mediaResult.MediaType)
+		require.Equal(t, "Screenshot captured successfully", mediaResult.Text)
+	})
+
+	t.Run("Media response preserves metadata", func(t *testing.T) {
+		t.Parallel()
+
+		type ImageMetadata struct {
+			Width  int `json:"width"`
+			Height int `json:"height"`
+		}
+
+		imageTool := &mockTool{
+			name:        "generate_image",
+			description: "Generates an image",
+			executeFunc: func(ctx context.Context, call ToolCall) (ToolResponse, error) {
+				resp := NewImageResponse(imageData, "image/png")
+				return WithResponseMetadata(resp, ImageMetadata{Width: 800, Height: 600}), nil
+			},
+		}
+
+		model := &mockLanguageModel{
+			generateFunc: func(ctx context.Context, call Call) (*Response, error) {
+				if len(call.Prompt) == 1 {
+					return &Response{
+						Content: []Content{
+							ToolCallContent{
+								ToolCallID: "img-1",
+								ToolName:   "generate_image",
+								Input:      `{}`,
+							},
+						},
+						Usage:        Usage{TotalTokens: 10},
+						FinishReason: FinishReasonToolCalls,
+					}, nil
+				}
+				return &Response{
+					Content:      []Content{TextContent{Text: "Done"}},
+					Usage:        Usage{TotalTokens: 20},
+					FinishReason: FinishReasonStop,
+				}, nil
+			},
+		}
+
+		agent := NewAgent(model, WithTools(imageTool), WithStopConditions(StepCountIs(3)))
+
+		result, err := agent.Generate(context.Background(), AgentCall{
+			Prompt: "Generate image",
+		})
+
+		require.NoError(t, err)
+		require.NotNil(t, result)
+
+		toolResults := result.Steps[0].Content.ToolResults()
+		require.Len(t, toolResults, 1)
+
+		// Check metadata was preserved
+		require.NotEmpty(t, toolResults[0].ClientMetadata)
+
+		var metadata ImageMetadata
+		err = json.Unmarshal([]byte(toolResults[0].ClientMetadata), &metadata)
+		require.NoError(t, err)
+		require.Equal(t, 800, metadata.Width)
+		require.Equal(t, 600, metadata.Height)
+	})
+}

content.go

@@ -306,8 +306,9 @@ func (t ToolResultOutputContentError) GetType() ToolResultContentType {
 
 // ToolResultOutputContentMedia represents media output content of a tool result.
 type ToolResultOutputContentMedia struct {
-	Data      string `json:"data"`       // for media type (base64)
-	MediaType string `json:"media_type"` // for media type
+	Data      string `json:"data"`           // for media type (base64)
+	MediaType string `json:"media_type"`     // for media type
+	Text      string `json:"text,omitempty"` // optional text content accompanying the media
 }
 
 // GetType returns the type of the tool result output content media.