From 1b5f04ba61ee1c91da06fe9bef8478232544a379 Mon Sep 17 00:00:00 2001 From: Andrey Nering Date: Tue, 9 Sep 2025 17:58:18 -0300 Subject: [PATCH 1/6] WIP test: add tests for thinking --- providertests/provider_test.go | 60 ++++++++++++++++++ .../TestThinking/anthropic-claude-sonnet.yaml | 3 + .../TestThinking/google-gemini-2.5-flash.yaml | 59 +++++++++++++++++ .../TestThinking/google-gemini-2.5-pro.yaml | 59 +++++++++++++++++ .../TestThinking/openai-gpt-4o-mini.yaml | 63 +++++++++++++++++++ .../testdata/TestThinking/openai-gpt-4o.yaml | 63 +++++++++++++++++++ 6 files changed, 307 insertions(+) create mode 100644 providertests/testdata/TestThinking/anthropic-claude-sonnet.yaml create mode 100644 providertests/testdata/TestThinking/google-gemini-2.5-flash.yaml create mode 100644 providertests/testdata/TestThinking/google-gemini-2.5-pro.yaml create mode 100644 providertests/testdata/TestThinking/openai-gpt-4o-mini.yaml create mode 100644 providertests/testdata/TestThinking/openai-gpt-4o.yaml diff --git a/providertests/provider_test.go b/providertests/provider_test.go index 96ebf64da..3dbddbe1e 100644 --- a/providertests/provider_test.go +++ b/providertests/provider_test.go @@ -85,6 +85,66 @@ func TestTool(t *testing.T) { } } +func TestThinking(t *testing.T) { + for _, pair := range languageModelBuilders { + t.Run(pair.name, func(t *testing.T) { + r := newRecorder(t) + + languageModel, err := pair.builder(r) + if err != nil { + t.Fatalf("failed to build language model: %v", err) + } + + type WeatherInput struct { + Location string `json:"location" description:"the city"` + } + + weatherTool := ai.NewAgentTool( + "weather", + "Get weather information for a location", + func(ctx context.Context, input WeatherInput, _ ai.ToolCall) (ai.ToolResponse, error) { + return ai.NewTextResponse("40 C"), nil + }, + ) + + agent := ai.NewAgent( + languageModel, + ai.WithSystemPrompt("You are a helpful assistant"), + ai.WithTools(weatherTool), + ) + result, err := agent.Generate(t.Context(), ai.AgentCall{ + Prompt: "What's the weather in Florence?", + ProviderOptions: ai.ProviderOptions{ + "anthropic": { + "thinking": map[string]any{ + "budget_tokens": 10_000, + }, + }, + "google": { + "thinking_config": map[string]any{ + "thinking_budget": 100, + "include_thoughts": true, + }, + }, + "openai": { + "reasoning_effort": "medium", + }, + }, + }) + if err != nil { + t.Fatalf("failed to generate: %v", err) + } + + want1 := "Florence" + want2 := "40" + got := result.Response.Content.Text() + if !strings.Contains(got, want1) || !strings.Contains(got, want2) { + t.Fatalf("unexpected response: got %q, want %q %q", got, want1, want2) + } + }) + } +} + func TestStream(t *testing.T) { for _, pair := range languageModelBuilders { t.Run(pair.name, func(t *testing.T) { diff --git a/providertests/testdata/TestThinking/anthropic-claude-sonnet.yaml b/providertests/testdata/TestThinking/anthropic-claude-sonnet.yaml new file mode 100644 index 000000000..2797c38e0 --- /dev/null +++ b/providertests/testdata/TestThinking/anthropic-claude-sonnet.yaml @@ -0,0 +1,3 @@ +--- +version: 2 +interactions: [] diff --git a/providertests/testdata/TestThinking/google-gemini-2.5-flash.yaml b/providertests/testdata/TestThinking/google-gemini-2.5-flash.yaml new file mode 100644 index 000000000..95f07865d --- /dev/null +++ b/providertests/testdata/TestThinking/google-gemini-2.5-flash.yaml @@ -0,0 +1,59 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 481 + host: generativelanguage.googleapis.com + body: "{\"contents\":[{\"parts\":[{\"text\":\"What's the weather in Florence?\"}],\"role\":\"user\"}],\"generationConfig\":{},\"systemInstruction\":{\"parts\":[{\"text\":\"You are a helpful assistant\"}],\"role\":\"user\"},\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"AUTO\"}},\"tools\":[{\"functionDeclarations\":[{\"description\":\"Get weather information for a location\",\"name\":\"weather\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"STRING\"}},\"required\":[\"location\"],\"type\":\"OBJECT\"}}]}]}\n" + headers: + Content-Type: + - application/json + User-Agent: + - google-genai-sdk/1.23.0 gl-go/go1.24.5 + url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": [\n {\n \"functionCall\": {\n \"name\": \"weather\",\n \"args\": {\n \"location\": \"Florence\"\n }\n },\n \"thoughtSignature\": \"CtwBAdHtim+LzmvI8rtsj5p2RhFdBKtg0CShperfeu8CtOIU63TuPtAOTwedY7xzLHw+8Xn9y3Hk5vE8O42WBkS9f61smbc+VIp/Qj3JZ+3xdW1C54gb47V2IxNqDyKuDGuU+HpLq3++ad1mq8lAHWrjWZj5oOGdH75T4hHv+XsZPuYKRPjg2tklhcsG7uaM8GZbXwdG8V6Vhc1YFo3opHxXywhoZl4Tg7ddMmfVQJD6SkVQRuTr1p/r/mtEYL5aDUmgIX0aLgje9u6qEhOEf53ibwcDhI9/aHgVCIYv0A==\"\n }\n ],\n \"role\": \"model\"\n },\n \"finishReason\": \"STOP\",\n \"index\": 0\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": 52,\n \"candidatesTokenCount\": 13,\n \"totalTokenCount\": 108,\n \"promptTokensDetails\": [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 52\n }\n ],\n \"thoughtsTokenCount\": 43\n },\n \"modelVersion\": \"gemini-2.5-flash\",\n \"responseId\": \"05TAaOnfH-fzqtsP1KeSqAM\"\n}\n" + headers: + Content-Type: + - application/json; charset=UTF-8 + status: 200 OK + code: 200 + duration: 1.623831958s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 700 + host: generativelanguage.googleapis.com + body: "{\"contents\":[{\"parts\":[{\"text\":\"What's the weather in Florence?\"}],\"role\":\"user\"},{\"parts\":[{\"functionCall\":{\"args\":{\"location\":\"Florence\"},\"id\":\"weather\",\"name\":\"weather\"}}],\"role\":\"model\"},{\"parts\":[{\"functionResponse\":{\"id\":\"weather\",\"name\":\"weather\",\"response\":{\"result\":\"40 C\"}}}],\"role\":\"user\"}],\"generationConfig\":{},\"systemInstruction\":{\"parts\":[{\"text\":\"You are a helpful assistant\"}],\"role\":\"user\"},\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"AUTO\"}},\"tools\":[{\"functionDeclarations\":[{\"description\":\"Get weather information for a location\",\"name\":\"weather\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"STRING\"}},\"required\":[\"location\"],\"type\":\"OBJECT\"}}]}]}\n" + headers: + Content-Type: + - application/json + User-Agent: + - google-genai-sdk/1.23.0 gl-go/go1.24.5 + url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": [\n {\n \"text\": \"It is 40 C in Florence.\"\n }\n ],\n \"role\": \"model\"\n },\n \"finishReason\": \"STOP\",\n \"index\": 0\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": 80,\n \"candidatesTokenCount\": 9,\n \"totalTokenCount\": 89,\n \"promptTokensDetails\": [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 80\n }\n ]\n },\n \"modelVersion\": \"gemini-2.5-flash\",\n \"responseId\": \"1JTAaOnKNczUz7IPns3_-Aw\"\n}\n" + headers: + Content-Type: + - application/json; charset=UTF-8 + status: 200 OK + code: 200 + duration: 1.352961042s diff --git a/providertests/testdata/TestThinking/google-gemini-2.5-pro.yaml b/providertests/testdata/TestThinking/google-gemini-2.5-pro.yaml new file mode 100644 index 000000000..9dc26201d --- /dev/null +++ b/providertests/testdata/TestThinking/google-gemini-2.5-pro.yaml @@ -0,0 +1,59 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 481 + host: generativelanguage.googleapis.com + body: "{\"contents\":[{\"parts\":[{\"text\":\"What's the weather in Florence?\"}],\"role\":\"user\"}],\"generationConfig\":{},\"systemInstruction\":{\"parts\":[{\"text\":\"You are a helpful assistant\"}],\"role\":\"user\"},\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"AUTO\"}},\"tools\":[{\"functionDeclarations\":[{\"description\":\"Get weather information for a location\",\"name\":\"weather\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"STRING\"}},\"required\":[\"location\"],\"type\":\"OBJECT\"}}]}]}\n" + headers: + Content-Type: + - application/json + User-Agent: + - google-genai-sdk/1.23.0 gl-go/go1.24.5 + url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": [\n {\n \"functionCall\": {\n \"name\": \"weather\",\n \"args\": {\n \"location\": \"Florence\"\n }\n },\n \"thoughtSignature\": \"CpcEAdHtim9KlLGBYSvtxGJRv/W4YaFoyub/zNb/YOxexIl3K3C7nKmcIRo/WbgAlzxj/eqhNnkoE+eRZxtzK+mArUcNxvCeufuvXQIj2hyrEDsXOp7vXN65Aum2KMzqcTw8kEKTdnvpfhNnzawa0kS2oLHG0TnGX7RTdz0k4pOLNGPhhV0dH8dbF9SU7OwQw8qoJEonIPuDR+lTUIJLLFbW/bA0CAQsN2+DrcjCfU494pWMaomZHZ6SlFH6i7vu4t+47VrGF1NuXxlxIvp8o7nQf2olUfgO1nNBDVRvjOeGCTHrjZnWDHv7XxKbXHOrEifTJJ2xp2M/R7oCCV+gHe4NK4vBPrwhDXS3vclMccq03u+b5xofW3yM/yE3aL1/xoJYoR87/MQn4WkyBi4EqdZZwysP7TYhFmNBi64bqyXUX89Gdo2+L7OweBPuM9fBG5CRq3HcdU2Doyg+Bv/5B0iwacxpETPtsNHD1pKBt/jWnfb2V+BcZKY3ipnuCcKt4cFO8u+KH/98WTN/o4u2gPtEoof3reflTuIT/MJYl+MPWsPuZAOB48jMBPy9KrFyzGUrF1gT0qjGIosa/XMNxXCJP50n2J25JmSgu1UwB8IwVXaRvgBDX6o/tf0qDtvoRC2uqUJqzRjqO+m6FNXL6vUAbFpfJnadUjllv+8ONPDJIrolTWyHu0JlVG+EY1ni3PJ5e+NUT2jZWg==\"\n }\n ],\n \"role\": \"model\"\n },\n \"finishReason\": \"STOP\",\n \"index\": 0\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": 52,\n \"candidatesTokenCount\": 13,\n \"totalTokenCount\": 184,\n \"promptTokensDetails\": [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 52\n }\n ],\n \"thoughtsTokenCount\": 119\n },\n \"modelVersion\": \"gemini-2.5-pro\",\n \"responseId\": \"2JTAaODDDbGqmtkPzprG6AI\"\n}\n" + headers: + Content-Type: + - application/json; charset=UTF-8 + status: 200 OK + code: 200 + duration: 3.433144416s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 700 + host: generativelanguage.googleapis.com + body: "{\"contents\":[{\"parts\":[{\"text\":\"What's the weather in Florence?\"}],\"role\":\"user\"},{\"parts\":[{\"functionCall\":{\"args\":{\"location\":\"Florence\"},\"id\":\"weather\",\"name\":\"weather\"}}],\"role\":\"model\"},{\"parts\":[{\"functionResponse\":{\"id\":\"weather\",\"name\":\"weather\",\"response\":{\"result\":\"40 C\"}}}],\"role\":\"user\"}],\"generationConfig\":{},\"systemInstruction\":{\"parts\":[{\"text\":\"You are a helpful assistant\"}],\"role\":\"user\"},\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"AUTO\"}},\"tools\":[{\"functionDeclarations\":[{\"description\":\"Get weather information for a location\",\"name\":\"weather\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"STRING\"}},\"required\":[\"location\"],\"type\":\"OBJECT\"}}]}]}\n" + headers: + Content-Type: + - application/json + User-Agent: + - google-genai-sdk/1.23.0 gl-go/go1.24.5 + url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": [\n {\n \"text\": \"I'm not sure which Florence you are referring to. Can you please clarify?\"\n }\n ],\n \"role\": \"model\"\n },\n \"finishReason\": \"STOP\",\n \"index\": 0\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": 80,\n \"candidatesTokenCount\": 17,\n \"totalTokenCount\": 97,\n \"promptTokensDetails\": [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 80\n }\n ]\n },\n \"modelVersion\": \"gemini-2.5-pro\",\n \"responseId\": \"2ZTAaNaCGtHRz7IPtKmAgAo\"\n}\n" + headers: + Content-Type: + - application/json; charset=UTF-8 + status: 200 OK + code: 200 + duration: 1.229572208s diff --git a/providertests/testdata/TestThinking/openai-gpt-4o-mini.yaml b/providertests/testdata/TestThinking/openai-gpt-4o-mini.yaml new file mode 100644 index 000000000..0b07af847 --- /dev/null +++ b/providertests/testdata/TestThinking/openai-gpt-4o-mini.yaml @@ -0,0 +1,63 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 429 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence?\",\"role\":\"user\"}],\"model\":\"gpt-4o-mini\",\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"id\": \"chatcmpl-CDzhPaOtWo71P4laqIZgHVRRNDK8T\",\n \"object\": \"chat.completion\",\n \"created\": 1757451471,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_9bQygg6UA54JS87CZdVNe7Z2\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"weather\",\n \"arguments\": \"{\\\"location\\\":\\\"Florence\\\"}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 59,\n \"completion_tokens\": 14,\n \"total_tokens\": 73,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_8bda4d3a2c\"\n}\n" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 1.091469542s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 674 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence?\",\"role\":\"user\"},{\"tool_calls\":[{\"id\":\"call_9bQygg6UA54JS87CZdVNe7Z2\",\"function\":{\"arguments\":\"{\\\"location\\\":\\\"Florence\\\"}\",\"name\":\"weather\"},\"type\":\"function\"}],\"role\":\"assistant\"},{\"content\":\"40 C\",\"tool_call_id\":\"call_9bQygg6UA54JS87CZdVNe7Z2\",\"role\":\"tool\"}],\"model\":\"gpt-4o-mini\",\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"id\": \"chatcmpl-CDzhRSBOmuB2XOkIwZCLCJPopguEi\",\n \"object\": \"chat.completion\",\n \"created\": 1757451473,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The weather in Florence is currently 40°C.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 82,\n \"completion_tokens\": 11,\n \"total_tokens\": 93,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_8bda4d3a2c\"\n}\n" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 1.36089475s diff --git a/providertests/testdata/TestThinking/openai-gpt-4o.yaml b/providertests/testdata/TestThinking/openai-gpt-4o.yaml new file mode 100644 index 000000000..3b807efe2 --- /dev/null +++ b/providertests/testdata/TestThinking/openai-gpt-4o.yaml @@ -0,0 +1,63 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 424 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence?\",\"role\":\"user\"}],\"model\":\"gpt-4o\",\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"id\": \"chatcmpl-CDzhOgDUUJOdb6hJ3p9TqHdMo8mUL\",\n \"object\": \"chat.completion\",\n \"created\": 1757451470,\n \"model\": \"gpt-4o-2024-08-06\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_ebxPjF1PF5rNURKZb8VZXEj3\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"weather\",\n \"arguments\": \"{\\\"location\\\":\\\"Florence\\\"}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 59,\n \"completion_tokens\": 14,\n \"total_tokens\": 73,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_1827dd0c55\"\n}\n" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 765.026125ms +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 669 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence?\",\"role\":\"user\"},{\"tool_calls\":[{\"id\":\"call_ebxPjF1PF5rNURKZb8VZXEj3\",\"function\":{\"arguments\":\"{\\\"location\\\":\\\"Florence\\\"}\",\"name\":\"weather\"},\"type\":\"function\"}],\"role\":\"assistant\"},{\"content\":\"40 C\",\"tool_call_id\":\"call_ebxPjF1PF5rNURKZb8VZXEj3\",\"role\":\"tool\"}],\"model\":\"gpt-4o\",\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"id\": \"chatcmpl-CDzhPcAzAgnJ6NNeu3tAGe4SOkl0h\",\n \"object\": \"chat.completion\",\n \"created\": 1757451471,\n \"model\": \"gpt-4o-2024-08-06\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"In Florence, the weather is currently 40°C.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 82,\n \"completion_tokens\": 12,\n \"total_tokens\": 94,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_1827dd0c55\"\n}\n" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 537.69375ms From 3fce20b754ab0e401739420a0e2a23f5b1a5207a Mon Sep 17 00:00:00 2001 From: Kujtim Hoxha Date: Wed, 17 Sep 2025 13:02:25 +0200 Subject: [PATCH 2/6] chore: fix thinking test - add chrush.md - add check to google provider - make all tests use testify --- CRUSH.md | 29 ++ ai/tool_test.go | 311 +++++------------- google/google.go | 30 +- providertests/builders_test.go | 14 + providertests/provider_test.go | 170 +++++----- .../TestThinking/anthropic-claude-sonnet.yaml | 62 +++- .../TestThinking/google-gemini-2.5-flash.yaml | 59 ---- .../TestThinking/google-gemini-2.5-pro.yaml | 20 +- .../TestThinking/openai-gpt-4o-mini.yaml | 63 ---- .../testdata/TestThinking/openai-gpt-4o.yaml | 63 ---- .../testdata/TestThinking/openai-gpt-5.yaml | 63 ++++ .../anthropic-claude-sonnet.yaml | 61 ++++ .../google-gemini-2.5-pro.yaml | 63 ++++ .../TestThinkingStreaming/openai-gpt-5.yaml | 61 ++++ 14 files changed, 555 insertions(+), 514 deletions(-) create mode 100644 CRUSH.md delete mode 100644 providertests/testdata/TestThinking/google-gemini-2.5-flash.yaml delete mode 100644 providertests/testdata/TestThinking/openai-gpt-4o-mini.yaml delete mode 100644 providertests/testdata/TestThinking/openai-gpt-4o.yaml create mode 100644 providertests/testdata/TestThinking/openai-gpt-5.yaml create mode 100644 providertests/testdata/TestThinkingStreaming/anthropic-claude-sonnet.yaml create mode 100644 providertests/testdata/TestThinkingStreaming/google-gemini-2.5-pro.yaml create mode 100644 providertests/testdata/TestThinkingStreaming/openai-gpt-5.yaml diff --git a/CRUSH.md b/CRUSH.md new file mode 100644 index 000000000..ad5079839 --- /dev/null +++ b/CRUSH.md @@ -0,0 +1,29 @@ +# CRUSH.md - Fantasy AI SDK + +## Build/Test/Lint Commands +- **Build**: `go build ./...` +- **Test all**: `task test` or `go test ./... -count=1` +- **Test single**: `go test -run TestName ./package -v` +- **Test with args**: `task test -- -v -run TestName` +- **Lint**: `task lint` or `golangci-lint run` +- **Format**: `task fmt` or `gofmt -s -w .` +- **Modernize**: `task modernize` or `modernize -fix ./...` + +## Code Style Guidelines +- **Package naming**: lowercase, single word (ai, openai, anthropic, google) +- **Imports**: standard library first, then third-party, then local packages +- **Error handling**: Use custom error types with structured fields, wrap with context +- **Types**: Use type aliases for function signatures (`type Option = func(*options)`) +- **Naming**: CamelCase for exported, camelCase for unexported +- **Constants**: Use const blocks with descriptive names (ProviderName, DefaultURL) +- **Structs**: Embed anonymous structs for composition (APICallError embeds *AIError) +- **Functions**: Return error as last parameter, use context.Context as first param +- **Testing**: Use testify/assert, table-driven tests, recorder pattern for HTTP mocking +- **Comments**: Godoc format for exported functions, explain complex logic inline +- **JSON**: Use struct tags for marshaling, handle empty values gracefully + +## Project Structure +- `/ai` - Core AI abstractions and agent logic +- `/openai`, `/anthropic`, `/google` - Provider implementations +- `/providertests` - Cross-provider integration tests with VCR recordings +- `/examples` - Usage examples for different patterns \ No newline at end of file diff --git a/ai/tool_test.go b/ai/tool_test.go index 2d11d8991..8539c38f1 100644 --- a/ai/tool_test.go +++ b/ai/tool_test.go @@ -4,8 +4,9 @@ import ( "context" "fmt" "reflect" - "strings" "testing" + + "github.com/stretchr/testify/require" ) // Example of a simple typed tool using the function approach @@ -28,12 +29,9 @@ func TestTypedToolFuncExample(t *testing.T) { // Check the tool info info := tool.Info() - if info.Name != "calculator" { - t.Errorf("Expected tool name 'calculator', got %s", info.Name) - } - if len(info.Required) != 1 || info.Required[0] != "expression" { - t.Errorf("Expected required field 'expression', got %v", info.Required) - } + require.Equal(t, "calculator", info.Name) + require.Len(t, info.Required, 1) + require.Equal(t, "expression", info.Required[0]) // Test execution call := ToolCall{ @@ -43,15 +41,9 @@ func TestTypedToolFuncExample(t *testing.T) { } result, err := tool.Run(context.Background(), call) - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - if result.Content != "4" { - t.Errorf("Expected result '4', got %s", result.Content) - } - if result.IsError { - t.Errorf("Expected successful result, got error") - } + require.NoError(t, err) + require.Equal(t, "4", result.Content) + require.False(t, result.IsError) } func TestEnumToolExample(t *testing.T) { @@ -76,13 +68,10 @@ func TestEnumToolExample(t *testing.T) { // Check that the schema includes enum values info := tool.Info() unitsParam, ok := info.Parameters["units"].(map[string]any) - if !ok { - t.Fatal("Expected units parameter to exist") - } + require.True(t, ok, "Expected units parameter to exist") enumValues, ok := unitsParam["enum"].([]any) - if !ok || len(enumValues) != 2 { - t.Errorf("Expected 2 enum values, got %v", enumValues) - } + require.True(t, ok) + require.Len(t, enumValues, 2) // Test execution with enum value call := ToolCall{ @@ -92,15 +81,9 @@ func TestEnumToolExample(t *testing.T) { } result, err := tool.Run(context.Background(), call) - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - if !strings.Contains(result.Content, "San Francisco") { - t.Errorf("Expected result to contain 'San Francisco', got %s", result.Content) - } - if !strings.Contains(result.Content, "72°F") { - t.Errorf("Expected result to contain '72°F', got %s", result.Content) - } + require.NoError(t, err) + require.Contains(t, result.Content, "San Francisco") + require.Contains(t, result.Content, "72°F") } func TestEnumSupport(t *testing.T) { @@ -113,30 +96,20 @@ func TestEnumSupport(t *testing.T) { schema := generateSchema(reflect.TypeOf(WeatherInput{})) - if schema.Type != "object" { - t.Errorf("Expected schema type 'object', got %s", schema.Type) - } + require.Equal(t, "object", schema.Type) // Check units field has enum values unitsSchema := schema.Properties["units"] - if unitsSchema == nil { - t.Fatal("Expected units property to exist") - } - if len(unitsSchema.Enum) != 3 { - t.Errorf("Expected 3 enum values for units, got %d", len(unitsSchema.Enum)) - } + require.NotNil(t, unitsSchema, "Expected units property to exist") + require.Len(t, unitsSchema.Enum, 3) expectedUnits := []string{"celsius", "fahrenheit", "kelvin"} for i, expected := range expectedUnits { - if unitsSchema.Enum[i] != expected { - t.Errorf("Expected enum value %s, got %v", expected, unitsSchema.Enum[i]) - } + require.Equal(t, expected, unitsSchema.Enum[i]) } // Check required fields (format should not be required due to omitempty) expectedRequired := []string{"location", "units"} - if len(schema.Required) != len(expectedRequired) { - t.Errorf("Expected %d required fields, got %d", len(expectedRequired), len(schema.Required)) - } + require.Len(t, schema.Required, len(expectedRequired)) } func TestSchemaToParameters(t *testing.T) { @@ -170,43 +143,24 @@ func TestSchemaToParameters(t *testing.T) { // Check name parameter nameParam, ok := params["name"].(map[string]any) - if !ok { - t.Fatal("Expected name parameter to exist") - } - if nameParam["type"] != "string" { - t.Errorf("Expected name type 'string', got %v", nameParam["type"]) - } - if nameParam["description"] != "The name field" { - t.Errorf("Expected name description 'The name field', got %v", nameParam["description"]) - } + require.True(t, ok, "Expected name parameter to exist") + require.Equal(t, "string", nameParam["type"]) + require.Equal(t, "The name field", nameParam["description"]) // Check age parameter with min/max ageParam, ok := params["age"].(map[string]any) - if !ok { - t.Fatal("Expected age parameter to exist") - } - if ageParam["type"] != "integer" { - t.Errorf("Expected age type 'integer', got %v", ageParam["type"]) - } - if ageParam["minimum"] != 0.0 { - t.Errorf("Expected age minimum 0, got %v", ageParam["minimum"]) - } - if ageParam["maximum"] != 120.0 { - t.Errorf("Expected age maximum 120, got %v", ageParam["maximum"]) - } + require.True(t, ok, "Expected age parameter to exist") + require.Equal(t, "integer", ageParam["type"]) + require.Equal(t, 0.0, ageParam["minimum"]) + require.Equal(t, 120.0, ageParam["maximum"]) // Check priority parameter with enum priorityParam, ok := params["priority"].(map[string]any) - if !ok { - t.Fatal("Expected priority parameter to exist") - } - if priorityParam["type"] != "string" { - t.Errorf("Expected priority type 'string', got %v", priorityParam["type"]) - } + require.True(t, ok, "Expected priority parameter to exist") + require.Equal(t, "string", priorityParam["type"]) enumValues, ok := priorityParam["enum"].([]any) - if !ok || len(enumValues) != 3 { - t.Errorf("Expected 3 enum values, got %v", enumValues) - } + require.True(t, ok) + require.Len(t, enumValues, 3) } func TestGenerateSchemaBasicTypes(t *testing.T) { @@ -258,9 +212,7 @@ func TestGenerateSchemaBasicTypes(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() schema := generateSchema(reflect.TypeOf(tt.input)) - if schema.Type != tt.expected.Type { - t.Errorf("Expected type %s, got %s", tt.expected.Type, schema.Type) - } + require.Equal(t, tt.expected.Type, schema.Type) }) } } @@ -303,15 +255,9 @@ func TestGenerateSchemaArrayTypes(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() schema := generateSchema(reflect.TypeOf(tt.input)) - if schema.Type != tt.expected.Type { - t.Errorf("Expected type %s, got %s", tt.expected.Type, schema.Type) - } - if schema.Items == nil { - t.Fatal("Expected items schema to exist") - } - if schema.Items.Type != tt.expected.Items.Type { - t.Errorf("Expected items type %s, got %s", tt.expected.Items.Type, schema.Items.Type) - } + require.Equal(t, tt.expected.Type, schema.Type) + require.NotNil(t, schema.Items, "Expected items schema to exist") + require.Equal(t, tt.expected.Items.Type, schema.Items.Type) }) } } @@ -345,9 +291,7 @@ func TestGenerateSchemaMapTypes(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() schema := generateSchema(reflect.TypeOf(tt.input)) - if schema.Type != tt.expected { - t.Errorf("Expected type %s, got %s", tt.expected, schema.Type) - } + require.Equal(t, tt.expected, schema.Type) }) } } @@ -384,60 +328,36 @@ func TestGenerateSchemaStructTypes(t *testing.T) { name: "simple struct", input: SimpleStruct{}, validate: func(t *testing.T, schema Schema) { - if schema.Type != "object" { - t.Errorf("Expected type object, got %s", schema.Type) - } - if len(schema.Properties) != 2 { - t.Errorf("Expected 2 properties, got %d", len(schema.Properties)) - } - if schema.Properties["name"] == nil { - t.Error("Expected name property to exist") - } - if schema.Properties["name"].Description != "The name field" { - t.Errorf("Expected description 'The name field', got %s", schema.Properties["name"].Description) - } - if len(schema.Required) != 2 { - t.Errorf("Expected 2 required fields, got %d", len(schema.Required)) - } + require.Equal(t, "object", schema.Type) + require.Len(t, schema.Properties, 2) + require.NotNil(t, schema.Properties["name"], "Expected name property to exist") + require.Equal(t, "The name field", schema.Properties["name"].Description) + require.Len(t, schema.Required, 2) }, }, { name: "struct with omitempty", input: StructWithOmitEmpty{}, validate: func(t *testing.T, schema Schema) { - if len(schema.Required) != 1 { - t.Errorf("Expected 1 required field, got %d", len(schema.Required)) - } - if schema.Required[0] != "required" { - t.Errorf("Expected required field 'required', got %s", schema.Required[0]) - } + require.Len(t, schema.Required, 1) + require.Equal(t, "required", schema.Required[0]) }, }, { name: "struct with json ignore", input: StructWithJSONIgnore{}, validate: func(t *testing.T, schema Schema) { - if len(schema.Properties) != 1 { - t.Errorf("Expected 1 property, got %d", len(schema.Properties)) - } - if schema.Properties["visible"] == nil { - t.Error("Expected visible property to exist") - } - if schema.Properties["hidden"] != nil { - t.Error("Expected hidden property to not exist") - } + require.Len(t, schema.Properties, 1) + require.NotNil(t, schema.Properties["visible"], "Expected visible property to exist") + require.Nil(t, schema.Properties["hidden"], "Expected hidden property to not exist") }, }, { name: "struct without json tags", input: StructWithoutJSONTags{}, validate: func(t *testing.T, schema Schema) { - if schema.Properties["first_name"] == nil { - t.Error("Expected first_name property to exist") - } - if schema.Properties["last_name"] == nil { - t.Error("Expected last_name property to exist") - } + require.NotNil(t, schema.Properties["first_name"], "Expected first_name property to exist") + require.NotNil(t, schema.Properties["last_name"], "Expected last_name property to exist") }, }, } @@ -461,23 +381,13 @@ func TestGenerateSchemaPointerTypes(t *testing.T) { schema := generateSchema(reflect.TypeOf(StructWithPointers{})) - if schema.Type != "object" { - t.Errorf("Expected type object, got %s", schema.Type) - } + require.Equal(t, "object", schema.Type) - if schema.Properties["name"] == nil { - t.Fatal("Expected name property to exist") - } - if schema.Properties["name"].Type != "string" { - t.Errorf("Expected name type string, got %s", schema.Properties["name"].Type) - } + require.NotNil(t, schema.Properties["name"], "Expected name property to exist") + require.Equal(t, "string", schema.Properties["name"].Type) - if schema.Properties["age"] == nil { - t.Fatal("Expected age property to exist") - } - if schema.Properties["age"].Type != "integer" { - t.Errorf("Expected age type integer, got %s", schema.Properties["age"].Type) - } + require.NotNil(t, schema.Properties["age"], "Expected age property to exist") + require.Equal(t, "integer", schema.Properties["age"].Type) } func TestGenerateSchemaNestedStructs(t *testing.T) { @@ -495,25 +405,15 @@ func TestGenerateSchemaNestedStructs(t *testing.T) { schema := generateSchema(reflect.TypeOf(Person{})) - if schema.Type != "object" { - t.Errorf("Expected type object, got %s", schema.Type) - } + require.Equal(t, "object", schema.Type) - if schema.Properties["address"] == nil { - t.Fatal("Expected address property to exist") - } + require.NotNil(t, schema.Properties["address"], "Expected address property to exist") addressSchema := schema.Properties["address"] - if addressSchema.Type != "object" { - t.Errorf("Expected address type object, got %s", addressSchema.Type) - } + require.Equal(t, "object", addressSchema.Type) - if addressSchema.Properties["street"] == nil { - t.Error("Expected street property in address to exist") - } - if addressSchema.Properties["city"] == nil { - t.Error("Expected city property in address to exist") - } + require.NotNil(t, addressSchema.Properties["street"], "Expected street property in address to exist") + require.NotNil(t, addressSchema.Properties["city"], "Expected city property in address to exist") } func TestGenerateSchemaRecursiveStructs(t *testing.T) { @@ -526,23 +426,15 @@ func TestGenerateSchemaRecursiveStructs(t *testing.T) { schema := generateSchema(reflect.TypeOf(Node{})) - if schema.Type != "object" { - t.Errorf("Expected type object, got %s", schema.Type) - } + require.Equal(t, "object", schema.Type) - if schema.Properties["value"] == nil { - t.Error("Expected value property to exist") - } + require.NotNil(t, schema.Properties["value"], "Expected value property to exist") - if schema.Properties["next"] == nil { - t.Error("Expected next property to exist") - } + require.NotNil(t, schema.Properties["next"], "Expected next property to exist") // The recursive reference should be handled gracefully nextSchema := schema.Properties["next"] - if nextSchema.Type != "object" { - t.Errorf("Expected next type object, got %s", nextSchema.Type) - } + require.Equal(t, "object", nextSchema.Type) } func TestGenerateSchemaWithEnumTags(t *testing.T) { @@ -558,33 +450,21 @@ func TestGenerateSchemaWithEnumTags(t *testing.T) { // Check level field levelSchema := schema.Properties["level"] - if levelSchema == nil { - t.Fatal("Expected level property to exist") - } - if len(levelSchema.Enum) != 4 { - t.Errorf("Expected 4 enum values for level, got %d", len(levelSchema.Enum)) - } + require.NotNil(t, levelSchema, "Expected level property to exist") + require.Len(t, levelSchema.Enum, 4) expectedLevels := []string{"debug", "info", "warn", "error"} for i, expected := range expectedLevels { - if levelSchema.Enum[i] != expected { - t.Errorf("Expected enum value %s, got %v", expected, levelSchema.Enum[i]) - } + require.Equal(t, expected, levelSchema.Enum[i]) } // Check format field formatSchema := schema.Properties["format"] - if formatSchema == nil { - t.Fatal("Expected format property to exist") - } - if len(formatSchema.Enum) != 2 { - t.Errorf("Expected 2 enum values for format, got %d", len(formatSchema.Enum)) - } + require.NotNil(t, formatSchema, "Expected format property to exist") + require.Len(t, formatSchema.Enum, 2) // Check required fields (optional should not be required due to omitempty) expectedRequired := []string{"level", "format"} - if len(schema.Required) != len(expectedRequired) { - t.Errorf("Expected %d required fields, got %d", len(expectedRequired), len(schema.Required)) - } + require.Len(t, schema.Required, len(expectedRequired)) } func TestGenerateSchemaComplexTypes(t *testing.T) { @@ -601,45 +481,25 @@ func TestGenerateSchemaComplexTypes(t *testing.T) { // Check string slice stringSliceSchema := schema.Properties["string_slice"] - if stringSliceSchema == nil { - t.Fatal("Expected string_slice property to exist") - } - if stringSliceSchema.Type != "array" { - t.Errorf("Expected string_slice type array, got %s", stringSliceSchema.Type) - } - if stringSliceSchema.Items.Type != "string" { - t.Errorf("Expected string_slice items type string, got %s", stringSliceSchema.Items.Type) - } + require.NotNil(t, stringSliceSchema, "Expected string_slice property to exist") + require.Equal(t, "array", stringSliceSchema.Type) + require.Equal(t, "string", stringSliceSchema.Items.Type) // Check int map intMapSchema := schema.Properties["int_map"] - if intMapSchema == nil { - t.Fatal("Expected int_map property to exist") - } - if intMapSchema.Type != "object" { - t.Errorf("Expected int_map type object, got %s", intMapSchema.Type) - } + require.NotNil(t, intMapSchema, "Expected int_map property to exist") + require.Equal(t, "object", intMapSchema.Type) // Check nested slice nestedSliceSchema := schema.Properties["nested_slice"] - if nestedSliceSchema == nil { - t.Fatal("Expected nested_slice property to exist") - } - if nestedSliceSchema.Type != "array" { - t.Errorf("Expected nested_slice type array, got %s", nestedSliceSchema.Type) - } - if nestedSliceSchema.Items.Type != "object" { - t.Errorf("Expected nested_slice items type object, got %s", nestedSliceSchema.Items.Type) - } + require.NotNil(t, nestedSliceSchema, "Expected nested_slice property to exist") + require.Equal(t, "array", nestedSliceSchema.Type) + require.Equal(t, "object", nestedSliceSchema.Items.Type) // Check interface interfaceSchema := schema.Properties["interface"] - if interfaceSchema == nil { - t.Fatal("Expected interface property to exist") - } - if interfaceSchema.Type != "object" { - t.Errorf("Expected interface type object, got %s", interfaceSchema.Type) - } + require.NotNil(t, interfaceSchema, "Expected interface property to exist") + require.Equal(t, "object", interfaceSchema.Type) } func TestToSnakeCase(t *testing.T) { @@ -664,9 +524,7 @@ func TestToSnakeCase(t *testing.T) { t.Run(tt.input, func(t *testing.T) { t.Parallel() result := toSnakeCase(tt.input) - if result != tt.expected { - t.Errorf("toSnakeCase(%s) = %s, expected %s", tt.input, result, tt.expected) - } + require.Equal(t, tt.expected, result, "toSnakeCase(%s)", tt.input) }) } } @@ -740,21 +598,14 @@ func TestSchemaToParametersEdgeCases(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() result := schemaToParameters(tt.schema) - if len(result) != len(tt.expected) { - t.Errorf("Expected %d parameters, got %d", len(tt.expected), len(result)) - } + require.Len(t, result, len(tt.expected)) for key, expectedValue := range tt.expected { - if result[key] == nil { - t.Errorf("Expected parameter %s to exist", key) - continue - } + require.NotNil(t, result[key], "Expected parameter %s to exist", key) // Deep comparison would be complex, so we'll check key properties resultParam := result[key].(map[string]any) expectedParam := expectedValue.(map[string]any) for propKey, propValue := range expectedParam { - if resultParam[propKey] != propValue { - t.Errorf("Expected %s.%s = %v, got %v", key, propKey, propValue, resultParam[propKey]) - } + require.Equal(t, propValue, resultParam[propKey], "Expected %s.%s", key, propKey) } } }) diff --git a/google/google.go b/google/google.go index b0a380e83..b32161328 100644 --- a/google/google.go +++ b/google/google.go @@ -120,16 +120,26 @@ func (a languageModel) prepareParams(call ai.Call) (*genai.GenerateContentConfig systemInstructions, content, warnings := toGooglePrompt(call.Prompt) - if providerOptions.ThinkingConfig != nil && - providerOptions.ThinkingConfig.IncludeThoughts != nil && - *providerOptions.ThinkingConfig.IncludeThoughts && - strings.HasPrefix(a.provider, "google.vertex.") { - warnings = append(warnings, ai.CallWarning{ - Type: ai.CallWarningTypeOther, - Message: "The 'includeThoughts' option is only supported with the Google Vertex provider " + - "and might not be supported or could behave unexpectedly with the current Google provider " + - fmt.Sprintf("(%s)", a.provider), - }) + if providerOptions.ThinkingConfig != nil { + if providerOptions.ThinkingConfig.IncludeThoughts != nil && + *providerOptions.ThinkingConfig.IncludeThoughts && + strings.HasPrefix(a.provider, "google.vertex.") { + warnings = append(warnings, ai.CallWarning{ + Type: ai.CallWarningTypeOther, + Message: "The 'includeThoughts' option is only supported with the Google Vertex provider " + + "and might not be supported or could behave unexpectedly with the current Google provider " + + fmt.Sprintf("(%s)", a.provider), + }) + } + + if providerOptions.ThinkingConfig.ThinkingBudget != nil && + *providerOptions.ThinkingConfig.ThinkingBudget < 128 { + warnings = append(warnings, ai.CallWarning{ + Type: ai.CallWarningTypeOther, + Message: "The 'thinking_budget' option can not be under 128 and will be set to 128 by default", + }) + providerOptions.ThinkingConfig.ThinkingBudget = ai.IntOption(128) + } } isGemmaModel := strings.HasPrefix(strings.ToLower(a.modelID), "gemma-") diff --git a/providertests/builders_test.go b/providertests/builders_test.go index 6a6c7652d..c7e5add73 100644 --- a/providertests/builders_test.go +++ b/providertests/builders_test.go @@ -27,6 +27,12 @@ var languageModelBuilders = []builderPair{ {"google-gemini-2.5-pro", builderGoogleGemini25Pro}, } +var thinkingLanguageModelBuilders = []builderPair{ + {"openai-gpt-5", builderOpenaiGpt5}, + {"anthropic-claude-sonnet", builderAnthropicClaudeSonnet4}, + {"google-gemini-2.5-pro", builderGoogleGemini25Pro}, +} + func builderOpenaiGpt4o(r *recorder.Recorder) (ai.LanguageModel, error) { provider := openai.New( openai.WithAPIKey(os.Getenv("OPENAI_API_KEY")), @@ -43,6 +49,14 @@ func builderOpenaiGpt4oMini(r *recorder.Recorder) (ai.LanguageModel, error) { return provider.LanguageModel("gpt-4o-mini") } +func builderOpenaiGpt5(r *recorder.Recorder) (ai.LanguageModel, error) { + provider := openai.New( + openai.WithAPIKey(os.Getenv("OPENAI_API_KEY")), + openai.WithHTTPClient(&http.Client{Transport: r}), + ) + return provider.LanguageModel("gpt-5") +} + func builderAnthropicClaudeSonnet4(r *recorder.Recorder) (ai.LanguageModel, error) { provider := anthropic.New( anthropic.WithAPIKey(os.Getenv("ANTHROPIC_API_KEY")), diff --git a/providertests/provider_test.go b/providertests/provider_test.go index 3dbddbe1e..c788f20bb 100644 --- a/providertests/provider_test.go +++ b/providertests/provider_test.go @@ -7,6 +7,10 @@ import ( "testing" "github.com/charmbracelet/fantasy/ai" + "github.com/charmbracelet/fantasy/anthropic" + "github.com/charmbracelet/fantasy/google" + "github.com/charmbracelet/fantasy/openai" + "github.com/stretchr/testify/require" _ "github.com/joho/godotenv/autoload" ) @@ -16,9 +20,7 @@ func TestSimple(t *testing.T) { r := newRecorder(t) languageModel, err := pair.builder(r) - if err != nil { - t.Fatalf("failed to build language model: %v", err) - } + require.NoError(t, err, "failed to build language model") agent := ai.NewAgent( languageModel, @@ -27,16 +29,12 @@ func TestSimple(t *testing.T) { result, err := agent.Generate(t.Context(), ai.AgentCall{ Prompt: "Say hi in Portuguese", }) - if err != nil { - t.Fatalf("failed to generate: %v", err) - } + require.NoError(t, err, "failed to generate") option1 := "Oi" option2 := "Olá" got := result.Response.Content.Text() - if !strings.Contains(got, option1) && !strings.Contains(got, option2) { - t.Fatalf("unexpected response: got %q, want %q or %q", got, option1, option2) - } + require.True(t, strings.Contains(got, option1) || strings.Contains(got, option2), "unexpected response: got %q, want %q or %q", got, option1, option2) }) } } @@ -47,9 +45,7 @@ func TestTool(t *testing.T) { r := newRecorder(t) languageModel, err := pair.builder(r) - if err != nil { - t.Fatalf("failed to build language model: %v", err) - } + require.NoError(t, err, "failed to build language model") type WeatherInput struct { Location string `json:"location" description:"the city"` @@ -71,29 +67,23 @@ func TestTool(t *testing.T) { result, err := agent.Generate(t.Context(), ai.AgentCall{ Prompt: "What's the weather in Florence?", }) - if err != nil { - t.Fatalf("failed to generate: %v", err) - } + require.NoError(t, err, "failed to generate") want1 := "Florence" want2 := "40" got := result.Response.Content.Text() - if !strings.Contains(got, want1) || !strings.Contains(got, want2) { - t.Fatalf("unexpected response: got %q, want %q %q", got, want1, want2) - } + require.True(t, strings.Contains(got, want1) && strings.Contains(got, want2), "unexpected response: got %q, want %q %q", got, want1, want2) }) } } func TestThinking(t *testing.T) { - for _, pair := range languageModelBuilders { + for _, pair := range thinkingLanguageModelBuilders { t.Run(pair.name, func(t *testing.T) { r := newRecorder(t) languageModel, err := pair.builder(r) - if err != nil { - t.Fatalf("failed to build language model: %v", err) - } + require.NoError(t, err, "failed to build language model") type WeatherInput struct { Location string `json:"location" description:"the city"` @@ -113,34 +103,84 @@ func TestThinking(t *testing.T) { ai.WithTools(weatherTool), ) result, err := agent.Generate(t.Context(), ai.AgentCall{ - Prompt: "What's the weather in Florence?", + Prompt: "What's the weather in Florence, Italy?", ProviderOptions: ai.ProviderOptions{ - "anthropic": { - "thinking": map[string]any{ - "budget_tokens": 10_000, + "anthropic": &anthropic.ProviderOptions{ + Thinking: &anthropic.ThinkingProviderOption{ + BudgetTokens: 10_000, }, }, - "google": { - "thinking_config": map[string]any{ - "thinking_budget": 100, - "include_thoughts": true, + "google": &google.ProviderOptions{ + ThinkingConfig: &google.ThinkingConfig{ + ThinkingBudget: ai.IntOption(100), + IncludeThoughts: ai.BoolOption(true), }, }, - "openai": { - "reasoning_effort": "medium", + "openai": &openai.ProviderOptions{ + ReasoningEffort: openai.ReasoningEffortOption(openai.ReasoningEffortMedium), }, }, }) - if err != nil { - t.Fatalf("failed to generate: %v", err) - } + require.NoError(t, err, "failed to generate") want1 := "Florence" want2 := "40" got := result.Response.Content.Text() - if !strings.Contains(got, want1) || !strings.Contains(got, want2) { - t.Fatalf("unexpected response: got %q, want %q %q", got, want1, want2) + require.True(t, strings.Contains(got, want1) && strings.Contains(got, want2), "unexpected response: got %q, want %q %q", got, want1, want2) + }) + } +} + +func TestThinkingStreaming(t *testing.T) { + for _, pair := range thinkingLanguageModelBuilders { + t.Run(pair.name, func(t *testing.T) { + r := newRecorder(t) + + languageModel, err := pair.builder(r) + require.NoError(t, err, "failed to build language model") + + type WeatherInput struct { + Location string `json:"location" description:"the city"` } + + weatherTool := ai.NewAgentTool( + "weather", + "Get weather information for a location", + func(ctx context.Context, input WeatherInput, _ ai.ToolCall) (ai.ToolResponse, error) { + return ai.NewTextResponse("40 C"), nil + }, + ) + + agent := ai.NewAgent( + languageModel, + ai.WithSystemPrompt("You are a helpful assistant"), + ai.WithTools(weatherTool), + ) + result, err := agent.Stream(t.Context(), ai.AgentStreamCall{ + Prompt: "What's the weather in Florence, Italy?", + ProviderOptions: ai.ProviderOptions{ + "anthropic": &anthropic.ProviderOptions{ + Thinking: &anthropic.ThinkingProviderOption{ + BudgetTokens: 10_000, + }, + }, + "google": &google.ProviderOptions{ + ThinkingConfig: &google.ThinkingConfig{ + ThinkingBudget: ai.IntOption(100), + IncludeThoughts: ai.BoolOption(true), + }, + }, + "openai": &openai.ProviderOptions{ + ReasoningEffort: openai.ReasoningEffortOption(openai.ReasoningEffortMedium), + }, + }, + }) + require.NoError(t, err, "failed to generate") + + want1 := "Florence" + want2 := "40" + got := result.Response.Content.Text() + require.True(t, strings.Contains(got, want1) && strings.Contains(got, want2), "unexpected response: got %q, want %q %q", got, want1, want2) }) } } @@ -151,9 +191,7 @@ func TestStream(t *testing.T) { r := newRecorder(t) languageModel, err := pair.builder(r) - if err != nil { - t.Fatalf("failed to build language model: %v", err) - } + require.NoError(t, err, "failed to build language model") agent := ai.NewAgent( languageModel, @@ -178,32 +216,20 @@ func TestStream(t *testing.T) { } result, err := agent.Stream(t.Context(), streamCall) - if err != nil { - t.Fatalf("failed to stream: %v", err) - } + require.NoError(t, err, "failed to stream") finalText := result.Response.Content.Text() - if finalText == "" { - t.Fatal("expected non-empty response") - } + require.NotEmpty(t, finalText, "expected non-empty response") - if !strings.Contains(strings.ToLower(finalText), "uno") || - !strings.Contains(strings.ToLower(finalText), "dos") || - !strings.Contains(strings.ToLower(finalText), "tres") { - t.Fatalf("unexpected response: %q", finalText) - } + require.True(t, strings.Contains(strings.ToLower(finalText), "uno") && + strings.Contains(strings.ToLower(finalText), "dos") && + strings.Contains(strings.ToLower(finalText), "tres"), "unexpected response: %q", finalText) - if textDeltaCount == 0 { - t.Fatal("expected at least one text delta callback") - } + require.Greater(t, textDeltaCount, 0, "expected at least one text delta callback") - if stepCount == 0 { - t.Fatal("expected at least one step finish callback") - } + require.Greater(t, stepCount, 0, "expected at least one step finish callback") - if collectedText.String() == "" { - t.Fatal("expected collected text from deltas to be non-empty") - } + require.NotEmpty(t, collectedText.String(), "expected collected text from deltas to be non-empty") }) } } @@ -214,9 +240,7 @@ func TestStreamWithTools(t *testing.T) { r := newRecorder(t) languageModel, err := pair.builder(r) - if err != nil { - t.Fatalf("failed to build language model: %v", err) - } + require.NoError(t, err, "failed to build language model") type CalculatorInput struct { A int `json:"a" description:"first number"` @@ -250,9 +274,7 @@ func TestStreamWithTools(t *testing.T) { }, OnToolCall: func(toolCall ai.ToolCallContent) error { toolCallCount++ - if toolCall.ToolName != "add" { - t.Errorf("unexpected tool name: %s", toolCall.ToolName) - } + require.Equal(t, "add", toolCall.ToolName, "unexpected tool name") return nil }, OnToolResult: func(result ai.ToolResultContent) error { @@ -262,22 +284,14 @@ func TestStreamWithTools(t *testing.T) { } result, err := agent.Stream(t.Context(), streamCall) - if err != nil { - t.Fatalf("failed to stream: %v", err) - } + require.NoError(t, err, "failed to stream") finalText := result.Response.Content.Text() - if !strings.Contains(finalText, "42") { - t.Fatalf("expected response to contain '42', got: %q", finalText) - } + require.Contains(t, finalText, "42", "expected response to contain '42', got: %q", finalText) - if toolCallCount == 0 { - t.Fatal("expected at least one tool call") - } + require.Greater(t, toolCallCount, 0, "expected at least one tool call") - if toolResultCount == 0 { - t.Fatal("expected at least one tool result") - } + require.Greater(t, toolResultCount, 0, "expected at least one tool result") }) } } diff --git a/providertests/testdata/TestThinking/anthropic-claude-sonnet.yaml b/providertests/testdata/TestThinking/anthropic-claude-sonnet.yaml index 2797c38e0..1a554d7b3 100644 --- a/providertests/testdata/TestThinking/anthropic-claude-sonnet.yaml +++ b/providertests/testdata/TestThinking/anthropic-claude-sonnet.yaml @@ -1,3 +1,63 @@ --- version: 2 -interactions: [] +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 550 + host: "" + body: "{\"max_tokens\":14096,\"messages\":[{\"content\":[{\"text\":\"What's the weather in Florence, Italy?\",\"type\":\"text\"}],\"role\":\"user\"}],\"model\":\"claude-sonnet-4-20250514\",\"system\":[{\"text\":\"You are a helpful assistant\",\"type\":\"text\"}],\"thinking\":{\"budget_tokens\":10000,\"type\":\"enabled\"},\"tool_choice\":{\"disable_parallel_tool_use\":false,\"type\":\"auto\"},\"tools\":[{\"input_schema\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"},\"name\":\"weather\",\"description\":\"Get weather information for a location\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - Anthropic/Go 1.10.0 + url: https://api.anthropic.com/v1/messages + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\"id\":\"msg_01NmYsbcWZbtPpmV1aMa5WWT\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-20250514\",\"content\":[{\"type\":\"thinking\",\"thinking\":\"The user is asking for weather information for Florence, Italy. I have a weather function available that takes a location parameter. The user has provided the location as \\\"Florence, Italy\\\" which is specific enough for the weather function.\\n\\nLet me call the weather function with \\\"Florence, Italy\\\" as the location parameter.\",\"signature\":\"EuwDCkYIBxgCKkChgOvL+rOlboiQFkEOC20rmj1/Xs3mTGfMFk5lIVU0H0drGyFYAl+5JU5PoWng2ZU7J9EpJrLUonCw9KBjS78oEgzAegs6pV953eMRkQAaDKJXIOEXcqfXFXnNayIwweUSskDSybgjCXZOKTQBm5xBlvThzhK75k4zycqwZpx3zeDZrdaV/+MIjgK1GAVqKtMCWp9QcmFNxVmwMGsORlN0zS3KY+3Xgd1D489b1lMG+FT8t1Xy2HxDBLlk9XY6HUQK7nN3HNXu/liglYnLT0weuYHsrzp8QgVrmgSWKLtX2pCI6SB8Df+9oQLzppw81d9+Vm3o7aJeI4nzwMxmZRekUu2j3LJiBFq5iQEAYnaGchWJ5B60mT5dk3UhnjTJYjVfaqgTHqybIwZ0ZrkAho4cybEwmQV7fCNsVIDom3v2XwDQF2TLeOGp/uFNElP4mpzQsB7k9x4asSb/kMsW8N34E5oWevGYyWDsX6c1NkTcJ+afmVN0df8i77bzFwtkrSz7/N85vX85rUxNxXCUfUiX5RkXq1ZHEL/y34ecpa9lP2CikFATgYKTfFQfc1x84LAC2aiBsDTKZFaZZocJcTHbO/PC1Ui+n4Ef8z33epy+AmGELkXG0CPgp6cqB08+AgoFlH78GAE=\"},{\"type\":\"tool_use\",\"id\":\"toolu_01M2Zq5AL9cCGbQxtDYLVQQ4\",\"name\":\"weather\",\"input\":{\"location\":\"Florence, Italy\"}}],\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"usage\":{\"input_tokens\":423,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":125,\"service_tier\":\"standard\"}}" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 2.398491s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 1879 + host: "" + body: "{\"max_tokens\":14096,\"messages\":[{\"content\":[{\"text\":\"What's the weather in Florence, Italy?\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"signature\":\"EuwDCkYIBxgCKkChgOvL+rOlboiQFkEOC20rmj1/Xs3mTGfMFk5lIVU0H0drGyFYAl+5JU5PoWng2ZU7J9EpJrLUonCw9KBjS78oEgzAegs6pV953eMRkQAaDKJXIOEXcqfXFXnNayIwweUSskDSybgjCXZOKTQBm5xBlvThzhK75k4zycqwZpx3zeDZrdaV/+MIjgK1GAVqKtMCWp9QcmFNxVmwMGsORlN0zS3KY+3Xgd1D489b1lMG+FT8t1Xy2HxDBLlk9XY6HUQK7nN3HNXu/liglYnLT0weuYHsrzp8QgVrmgSWKLtX2pCI6SB8Df+9oQLzppw81d9+Vm3o7aJeI4nzwMxmZRekUu2j3LJiBFq5iQEAYnaGchWJ5B60mT5dk3UhnjTJYjVfaqgTHqybIwZ0ZrkAho4cybEwmQV7fCNsVIDom3v2XwDQF2TLeOGp/uFNElP4mpzQsB7k9x4asSb/kMsW8N34E5oWevGYyWDsX6c1NkTcJ+afmVN0df8i77bzFwtkrSz7/N85vX85rUxNxXCUfUiX5RkXq1ZHEL/y34ecpa9lP2CikFATgYKTfFQfc1x84LAC2aiBsDTKZFaZZocJcTHbO/PC1Ui+n4Ef8z33epy+AmGELkXG0CPgp6cqB08+AgoFlH78GAE=\",\"thinking\":\"The user is asking for weather information for Florence, Italy. I have a weather function available that takes a location parameter. The user has provided the location as \\\"Florence, Italy\\\" which is specific enough for the weather function.\\n\\nLet me call the weather function with \\\"Florence, Italy\\\" as the location parameter.\",\"type\":\"thinking\"},{\"id\":\"toolu_01M2Zq5AL9cCGbQxtDYLVQQ4\",\"input\":{\"location\":\"Florence, Italy\"},\"name\":\"weather\",\"type\":\"tool_use\"}],\"role\":\"assistant\"},{\"content\":[{\"tool_use_id\":\"toolu_01M2Zq5AL9cCGbQxtDYLVQQ4\",\"content\":[{\"text\":\"40 C\",\"type\":\"text\"}],\"type\":\"tool_result\"}],\"role\":\"user\"}],\"model\":\"claude-sonnet-4-20250514\",\"system\":[{\"text\":\"You are a helpful assistant\",\"type\":\"text\"}],\"thinking\":{\"budget_tokens\":10000,\"type\":\"enabled\"},\"tool_choice\":{\"disable_parallel_tool_use\":false,\"type\":\"auto\"},\"tools\":[{\"input_schema\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"},\"name\":\"weather\",\"description\":\"Get weather information for a location\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - Anthropic/Go 1.10.0 + url: https://api.anthropic.com/v1/messages + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\"id\":\"msg_01DyEBuCyDtP4cjnhtqdbqrh\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-20250514\",\"content\":[{\"type\":\"text\",\"text\":\"The current weather in Florence, Italy is 40°C (104°F). That's quite hot! It seems like a very warm day in Florence.\"}],\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"usage\":{\"input_tokens\":563,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":35,\"service_tier\":\"standard\"}}" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 1.744995167s diff --git a/providertests/testdata/TestThinking/google-gemini-2.5-flash.yaml b/providertests/testdata/TestThinking/google-gemini-2.5-flash.yaml deleted file mode 100644 index 95f07865d..000000000 --- a/providertests/testdata/TestThinking/google-gemini-2.5-flash.yaml +++ /dev/null @@ -1,59 +0,0 @@ ---- -version: 2 -interactions: -- id: 0 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 481 - host: generativelanguage.googleapis.com - body: "{\"contents\":[{\"parts\":[{\"text\":\"What's the weather in Florence?\"}],\"role\":\"user\"}],\"generationConfig\":{},\"systemInstruction\":{\"parts\":[{\"text\":\"You are a helpful assistant\"}],\"role\":\"user\"},\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"AUTO\"}},\"tools\":[{\"functionDeclarations\":[{\"description\":\"Get weather information for a location\",\"name\":\"weather\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"STRING\"}},\"required\":[\"location\"],\"type\":\"OBJECT\"}}]}]}\n" - headers: - Content-Type: - - application/json - User-Agent: - - google-genai-sdk/1.23.0 gl-go/go1.24.5 - url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent - method: POST - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - content_length: -1 - uncompressed: true - body: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": [\n {\n \"functionCall\": {\n \"name\": \"weather\",\n \"args\": {\n \"location\": \"Florence\"\n }\n },\n \"thoughtSignature\": \"CtwBAdHtim+LzmvI8rtsj5p2RhFdBKtg0CShperfeu8CtOIU63TuPtAOTwedY7xzLHw+8Xn9y3Hk5vE8O42WBkS9f61smbc+VIp/Qj3JZ+3xdW1C54gb47V2IxNqDyKuDGuU+HpLq3++ad1mq8lAHWrjWZj5oOGdH75T4hHv+XsZPuYKRPjg2tklhcsG7uaM8GZbXwdG8V6Vhc1YFo3opHxXywhoZl4Tg7ddMmfVQJD6SkVQRuTr1p/r/mtEYL5aDUmgIX0aLgje9u6qEhOEf53ibwcDhI9/aHgVCIYv0A==\"\n }\n ],\n \"role\": \"model\"\n },\n \"finishReason\": \"STOP\",\n \"index\": 0\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": 52,\n \"candidatesTokenCount\": 13,\n \"totalTokenCount\": 108,\n \"promptTokensDetails\": [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 52\n }\n ],\n \"thoughtsTokenCount\": 43\n },\n \"modelVersion\": \"gemini-2.5-flash\",\n \"responseId\": \"05TAaOnfH-fzqtsP1KeSqAM\"\n}\n" - headers: - Content-Type: - - application/json; charset=UTF-8 - status: 200 OK - code: 200 - duration: 1.623831958s -- id: 1 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 700 - host: generativelanguage.googleapis.com - body: "{\"contents\":[{\"parts\":[{\"text\":\"What's the weather in Florence?\"}],\"role\":\"user\"},{\"parts\":[{\"functionCall\":{\"args\":{\"location\":\"Florence\"},\"id\":\"weather\",\"name\":\"weather\"}}],\"role\":\"model\"},{\"parts\":[{\"functionResponse\":{\"id\":\"weather\",\"name\":\"weather\",\"response\":{\"result\":\"40 C\"}}}],\"role\":\"user\"}],\"generationConfig\":{},\"systemInstruction\":{\"parts\":[{\"text\":\"You are a helpful assistant\"}],\"role\":\"user\"},\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"AUTO\"}},\"tools\":[{\"functionDeclarations\":[{\"description\":\"Get weather information for a location\",\"name\":\"weather\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"STRING\"}},\"required\":[\"location\"],\"type\":\"OBJECT\"}}]}]}\n" - headers: - Content-Type: - - application/json - User-Agent: - - google-genai-sdk/1.23.0 gl-go/go1.24.5 - url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent - method: POST - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - content_length: -1 - uncompressed: true - body: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": [\n {\n \"text\": \"It is 40 C in Florence.\"\n }\n ],\n \"role\": \"model\"\n },\n \"finishReason\": \"STOP\",\n \"index\": 0\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": 80,\n \"candidatesTokenCount\": 9,\n \"totalTokenCount\": 89,\n \"promptTokensDetails\": [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 80\n }\n ]\n },\n \"modelVersion\": \"gemini-2.5-flash\",\n \"responseId\": \"1JTAaOnKNczUz7IPns3_-Aw\"\n}\n" - headers: - Content-Type: - - application/json; charset=UTF-8 - status: 200 OK - code: 200 - duration: 1.352961042s diff --git a/providertests/testdata/TestThinking/google-gemini-2.5-pro.yaml b/providertests/testdata/TestThinking/google-gemini-2.5-pro.yaml index 9dc26201d..b927c7edd 100644 --- a/providertests/testdata/TestThinking/google-gemini-2.5-pro.yaml +++ b/providertests/testdata/TestThinking/google-gemini-2.5-pro.yaml @@ -6,14 +6,14 @@ interactions: proto: HTTP/1.1 proto_major: 1 proto_minor: 1 - content_length: 481 + content_length: 550 host: generativelanguage.googleapis.com - body: "{\"contents\":[{\"parts\":[{\"text\":\"What's the weather in Florence?\"}],\"role\":\"user\"}],\"generationConfig\":{},\"systemInstruction\":{\"parts\":[{\"text\":\"You are a helpful assistant\"}],\"role\":\"user\"},\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"AUTO\"}},\"tools\":[{\"functionDeclarations\":[{\"description\":\"Get weather information for a location\",\"name\":\"weather\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"STRING\"}},\"required\":[\"location\"],\"type\":\"OBJECT\"}}]}]}\n" + body: "{\"contents\":[{\"parts\":[{\"text\":\"What's the weather in Florence, Italy?\"}],\"role\":\"user\"}],\"generationConfig\":{\"thinkingConfig\":{\"includeThoughts\":true,\"thinkingBudget\":128}},\"systemInstruction\":{\"parts\":[{\"text\":\"You are a helpful assistant\"}],\"role\":\"user\"},\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"AUTO\"}},\"tools\":[{\"functionDeclarations\":[{\"description\":\"Get weather information for a location\",\"name\":\"weather\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"STRING\"}},\"required\":[\"location\"],\"type\":\"OBJECT\"}}]}]}\n" headers: Content-Type: - application/json User-Agent: - - google-genai-sdk/1.23.0 gl-go/go1.24.5 + - google-genai-sdk/1.23.0 gl-go/go1.25.0 url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent method: POST response: @@ -22,26 +22,26 @@ interactions: proto_minor: 0 content_length: -1 uncompressed: true - body: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": [\n {\n \"functionCall\": {\n \"name\": \"weather\",\n \"args\": {\n \"location\": \"Florence\"\n }\n },\n \"thoughtSignature\": \"CpcEAdHtim9KlLGBYSvtxGJRv/W4YaFoyub/zNb/YOxexIl3K3C7nKmcIRo/WbgAlzxj/eqhNnkoE+eRZxtzK+mArUcNxvCeufuvXQIj2hyrEDsXOp7vXN65Aum2KMzqcTw8kEKTdnvpfhNnzawa0kS2oLHG0TnGX7RTdz0k4pOLNGPhhV0dH8dbF9SU7OwQw8qoJEonIPuDR+lTUIJLLFbW/bA0CAQsN2+DrcjCfU494pWMaomZHZ6SlFH6i7vu4t+47VrGF1NuXxlxIvp8o7nQf2olUfgO1nNBDVRvjOeGCTHrjZnWDHv7XxKbXHOrEifTJJ2xp2M/R7oCCV+gHe4NK4vBPrwhDXS3vclMccq03u+b5xofW3yM/yE3aL1/xoJYoR87/MQn4WkyBi4EqdZZwysP7TYhFmNBi64bqyXUX89Gdo2+L7OweBPuM9fBG5CRq3HcdU2Doyg+Bv/5B0iwacxpETPtsNHD1pKBt/jWnfb2V+BcZKY3ipnuCcKt4cFO8u+KH/98WTN/o4u2gPtEoof3reflTuIT/MJYl+MPWsPuZAOB48jMBPy9KrFyzGUrF1gT0qjGIosa/XMNxXCJP50n2J25JmSgu1UwB8IwVXaRvgBDX6o/tf0qDtvoRC2uqUJqzRjqO+m6FNXL6vUAbFpfJnadUjllv+8ONPDJIrolTWyHu0JlVG+EY1ni3PJ5e+NUT2jZWg==\"\n }\n ],\n \"role\": \"model\"\n },\n \"finishReason\": \"STOP\",\n \"index\": 0\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": 52,\n \"candidatesTokenCount\": 13,\n \"totalTokenCount\": 184,\n \"promptTokensDetails\": [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 52\n }\n ],\n \"thoughtsTokenCount\": 119\n },\n \"modelVersion\": \"gemini-2.5-pro\",\n \"responseId\": \"2JTAaODDDbGqmtkPzprG6AI\"\n}\n" + body: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": [\n {\n \"text\": \"**Getting the Weather in Florence**\\n\\nOkay, so I see the user wants the weather for \\\"Florence, Italy.\\\" My initial thought is to grab the `weather.get_weather` tool. Perfect, it has a `location` parameter, which is exactly what I need. The user has thoughtfully provided \\\"Florence, Italy,\\\" so that's a straightforward input for the `location` parameter. This is a quick and efficient request to handle.\\n\",\n \"thought\": true\n },\n {\n \"functionCall\": {\n \"name\": \"weather\",\n \"args\": {\n \"location\": \"Florence, Italy\"\n }\n },\n \"thoughtSignature\": \"CokDAdHtim93vKtHJYI78AitJBYITb44JuVhBViGlnpnj3bSPvRBDI3GF8joEA68HpEu4qw281IW11+lQD+rSyPmhuYibh1cABkgMBMnlzHWn1FyJ6Vxv14WNDQKchoHMJpJ7yvFsga1jI2ALYJ+beV+6jrJa2/yA5VAaEKFTtxisBZzxM7U2HkHsZrAhWZtVK+GBx2bYuXRRF5THFT3ilIzPOCb02cG8Ve4abqO23J/augLfoftvDn+QK+PKyj13MdD3w/f89xjLr7MH4WXn6eEWU9TENJGiMgOoEXvNyjf/ZiAoShowtBYkhXxG0IFGai+O6x42LryDImGoXCbhdNQ7/zOVwbLBzWBnLEuVVNN7KuJXhl+FrsYJj+WDhxmTxjE1MKatCG9mNFAl3BxSEqDkNtBDng2SrFLTpt80+PFR9QB0jdBoml7e/k0kNCBH+5ObGFaKAxQaSF/QHarAF9K4YmR/7OpryU/y+etlYGHYc/tGvXBZRZ8NUxbfjEBt0N99VoGUx2WXKJo\"\n }\n ],\n \"role\": \"model\"\n },\n \"finishReason\": \"STOP\",\n \"index\": 0\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": 54,\n \"candidatesTokenCount\": 15,\n \"totalTokenCount\": 162,\n \"promptTokensDetails\": [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 54\n }\n ],\n \"thoughtsTokenCount\": 93\n },\n \"modelVersion\": \"gemini-2.5-pro\",\n \"responseId\": \"bpHKaMGVJc6YkdUP3fStCA\"\n}\n" headers: Content-Type: - application/json; charset=UTF-8 status: 200 OK code: 200 - duration: 3.433144416s + duration: 4.193896625s - id: 1 request: proto: HTTP/1.1 proto_major: 1 proto_minor: 1 - content_length: 700 + content_length: 776 host: generativelanguage.googleapis.com - body: "{\"contents\":[{\"parts\":[{\"text\":\"What's the weather in Florence?\"}],\"role\":\"user\"},{\"parts\":[{\"functionCall\":{\"args\":{\"location\":\"Florence\"},\"id\":\"weather\",\"name\":\"weather\"}}],\"role\":\"model\"},{\"parts\":[{\"functionResponse\":{\"id\":\"weather\",\"name\":\"weather\",\"response\":{\"result\":\"40 C\"}}}],\"role\":\"user\"}],\"generationConfig\":{},\"systemInstruction\":{\"parts\":[{\"text\":\"You are a helpful assistant\"}],\"role\":\"user\"},\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"AUTO\"}},\"tools\":[{\"functionDeclarations\":[{\"description\":\"Get weather information for a location\",\"name\":\"weather\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"STRING\"}},\"required\":[\"location\"],\"type\":\"OBJECT\"}}]}]}\n" + body: "{\"contents\":[{\"parts\":[{\"text\":\"What's the weather in Florence, Italy?\"}],\"role\":\"user\"},{\"parts\":[{\"functionCall\":{\"args\":{\"location\":\"Florence, Italy\"},\"id\":\"weather\",\"name\":\"weather\"}}],\"role\":\"model\"},{\"parts\":[{\"functionResponse\":{\"id\":\"weather\",\"name\":\"weather\",\"response\":{\"result\":\"40 C\"}}}],\"role\":\"user\"}],\"generationConfig\":{\"thinkingConfig\":{\"includeThoughts\":true,\"thinkingBudget\":128}},\"systemInstruction\":{\"parts\":[{\"text\":\"You are a helpful assistant\"}],\"role\":\"user\"},\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"AUTO\"}},\"tools\":[{\"functionDeclarations\":[{\"description\":\"Get weather information for a location\",\"name\":\"weather\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"STRING\"}},\"required\":[\"location\"],\"type\":\"OBJECT\"}}]}]}\n" headers: Content-Type: - application/json User-Agent: - - google-genai-sdk/1.23.0 gl-go/go1.24.5 + - google-genai-sdk/1.23.0 gl-go/go1.25.0 url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent method: POST response: @@ -50,10 +50,10 @@ interactions: proto_minor: 0 content_length: -1 uncompressed: true - body: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": [\n {\n \"text\": \"I'm not sure which Florence you are referring to. Can you please clarify?\"\n }\n ],\n \"role\": \"model\"\n },\n \"finishReason\": \"STOP\",\n \"index\": 0\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": 80,\n \"candidatesTokenCount\": 17,\n \"totalTokenCount\": 97,\n \"promptTokensDetails\": [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 80\n }\n ]\n },\n \"modelVersion\": \"gemini-2.5-pro\",\n \"responseId\": \"2ZTAaNaCGtHRz7IPtKmAgAo\"\n}\n" + body: "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": [\n {\n \"text\": \"It's 40 C in Florence, Italy. \\n\"\n }\n ],\n \"role\": \"model\"\n },\n \"finishReason\": \"STOP\",\n \"index\": 0\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": 84,\n \"candidatesTokenCount\": 12,\n \"totalTokenCount\": 96,\n \"promptTokensDetails\": [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 84\n }\n ]\n },\n \"modelVersion\": \"gemini-2.5-pro\",\n \"responseId\": \"cJHKaOO8KfrY7M8Pg8-psQo\"\n}\n" headers: Content-Type: - application/json; charset=UTF-8 status: 200 OK code: 200 - duration: 1.229572208s + duration: 2.006433542s diff --git a/providertests/testdata/TestThinking/openai-gpt-4o-mini.yaml b/providertests/testdata/TestThinking/openai-gpt-4o-mini.yaml deleted file mode 100644 index 0b07af847..000000000 --- a/providertests/testdata/TestThinking/openai-gpt-4o-mini.yaml +++ /dev/null @@ -1,63 +0,0 @@ ---- -version: 2 -interactions: -- id: 0 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 429 - host: "" - body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence?\",\"role\":\"user\"}],\"model\":\"gpt-4o-mini\",\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}]}" - headers: - Accept: - - application/json - Content-Type: - - application/json - User-Agent: - - OpenAI/Go 2.3.0 - url: https://api.openai.com/v1/chat/completions - method: POST - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - content_length: -1 - uncompressed: true - body: "{\n \"id\": \"chatcmpl-CDzhPaOtWo71P4laqIZgHVRRNDK8T\",\n \"object\": \"chat.completion\",\n \"created\": 1757451471,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_9bQygg6UA54JS87CZdVNe7Z2\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"weather\",\n \"arguments\": \"{\\\"location\\\":\\\"Florence\\\"}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 59,\n \"completion_tokens\": 14,\n \"total_tokens\": 73,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_8bda4d3a2c\"\n}\n" - headers: - Content-Type: - - application/json - status: 200 OK - code: 200 - duration: 1.091469542s -- id: 1 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 674 - host: "" - body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence?\",\"role\":\"user\"},{\"tool_calls\":[{\"id\":\"call_9bQygg6UA54JS87CZdVNe7Z2\",\"function\":{\"arguments\":\"{\\\"location\\\":\\\"Florence\\\"}\",\"name\":\"weather\"},\"type\":\"function\"}],\"role\":\"assistant\"},{\"content\":\"40 C\",\"tool_call_id\":\"call_9bQygg6UA54JS87CZdVNe7Z2\",\"role\":\"tool\"}],\"model\":\"gpt-4o-mini\",\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}]}" - headers: - Accept: - - application/json - Content-Type: - - application/json - User-Agent: - - OpenAI/Go 2.3.0 - url: https://api.openai.com/v1/chat/completions - method: POST - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - content_length: -1 - uncompressed: true - body: "{\n \"id\": \"chatcmpl-CDzhRSBOmuB2XOkIwZCLCJPopguEi\",\n \"object\": \"chat.completion\",\n \"created\": 1757451473,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The weather in Florence is currently 40°C.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 82,\n \"completion_tokens\": 11,\n \"total_tokens\": 93,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_8bda4d3a2c\"\n}\n" - headers: - Content-Type: - - application/json - status: 200 OK - code: 200 - duration: 1.36089475s diff --git a/providertests/testdata/TestThinking/openai-gpt-4o.yaml b/providertests/testdata/TestThinking/openai-gpt-4o.yaml deleted file mode 100644 index 3b807efe2..000000000 --- a/providertests/testdata/TestThinking/openai-gpt-4o.yaml +++ /dev/null @@ -1,63 +0,0 @@ ---- -version: 2 -interactions: -- id: 0 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 424 - host: "" - body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence?\",\"role\":\"user\"}],\"model\":\"gpt-4o\",\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}]}" - headers: - Accept: - - application/json - Content-Type: - - application/json - User-Agent: - - OpenAI/Go 2.3.0 - url: https://api.openai.com/v1/chat/completions - method: POST - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - content_length: -1 - uncompressed: true - body: "{\n \"id\": \"chatcmpl-CDzhOgDUUJOdb6hJ3p9TqHdMo8mUL\",\n \"object\": \"chat.completion\",\n \"created\": 1757451470,\n \"model\": \"gpt-4o-2024-08-06\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_ebxPjF1PF5rNURKZb8VZXEj3\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"weather\",\n \"arguments\": \"{\\\"location\\\":\\\"Florence\\\"}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 59,\n \"completion_tokens\": 14,\n \"total_tokens\": 73,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_1827dd0c55\"\n}\n" - headers: - Content-Type: - - application/json - status: 200 OK - code: 200 - duration: 765.026125ms -- id: 1 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 669 - host: "" - body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence?\",\"role\":\"user\"},{\"tool_calls\":[{\"id\":\"call_ebxPjF1PF5rNURKZb8VZXEj3\",\"function\":{\"arguments\":\"{\\\"location\\\":\\\"Florence\\\"}\",\"name\":\"weather\"},\"type\":\"function\"}],\"role\":\"assistant\"},{\"content\":\"40 C\",\"tool_call_id\":\"call_ebxPjF1PF5rNURKZb8VZXEj3\",\"role\":\"tool\"}],\"model\":\"gpt-4o\",\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}]}" - headers: - Accept: - - application/json - Content-Type: - - application/json - User-Agent: - - OpenAI/Go 2.3.0 - url: https://api.openai.com/v1/chat/completions - method: POST - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - content_length: -1 - uncompressed: true - body: "{\n \"id\": \"chatcmpl-CDzhPcAzAgnJ6NNeu3tAGe4SOkl0h\",\n \"object\": \"chat.completion\",\n \"created\": 1757451471,\n \"model\": \"gpt-4o-2024-08-06\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"In Florence, the weather is currently 40°C.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 82,\n \"completion_tokens\": 12,\n \"total_tokens\": 94,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_1827dd0c55\"\n}\n" - headers: - Content-Type: - - application/json - status: 200 OK - code: 200 - duration: 537.69375ms diff --git a/providertests/testdata/TestThinking/openai-gpt-5.yaml b/providertests/testdata/TestThinking/openai-gpt-5.yaml new file mode 100644 index 000000000..0c2ed0cd4 --- /dev/null +++ b/providertests/testdata/TestThinking/openai-gpt-5.yaml @@ -0,0 +1,63 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 458 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence, Italy?\",\"role\":\"user\"}],\"model\":\"gpt-5\",\"reasoning_effort\":\"medium\",\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"id\": \"chatcmpl-CGjxZHxMSr7N8cRMctGQZk1GoGaeb\",\n \"object\": \"chat.completion\",\n \"created\": 1758105953,\n \"model\": \"gpt-5-2025-08-07\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_altLYQTktFAyPzFgFXA8no5D\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"weather\",\n \"arguments\": \"{\\\"location\\\":\\\"Florence, Italy\\\"}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 145,\n \"completion_tokens\": 153,\n \"total_tokens\": 298,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 128,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": null\n}\n" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 3.138198958s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 710 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence, Italy?\",\"role\":\"user\"},{\"tool_calls\":[{\"id\":\"call_altLYQTktFAyPzFgFXA8no5D\",\"function\":{\"arguments\":\"{\\\"location\\\":\\\"Florence, Italy\\\"}\",\"name\":\"weather\"},\"type\":\"function\"}],\"role\":\"assistant\"},{\"content\":\"40 C\",\"tool_call_id\":\"call_altLYQTktFAyPzFgFXA8no5D\",\"role\":\"tool\"}],\"model\":\"gpt-5\",\"reasoning_effort\":\"medium\",\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"id\": \"chatcmpl-CGjxczIS9JrgSAuyFofVReqav7T93\",\n \"object\": \"chat.completion\",\n \"created\": 1758105956,\n \"model\": \"gpt-5-2025-08-07\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It’s currently about 40°C (104°F) in Florence, Italy.\\n\\nWant the forecast, humidity, or wind details as well?\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 176,\n \"completion_tokens\": 229,\n \"total_tokens\": 405,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 192,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": null\n}\n" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 6.448143542s diff --git a/providertests/testdata/TestThinkingStreaming/anthropic-claude-sonnet.yaml b/providertests/testdata/TestThinkingStreaming/anthropic-claude-sonnet.yaml new file mode 100644 index 000000000..eb888a85b --- /dev/null +++ b/providertests/testdata/TestThinkingStreaming/anthropic-claude-sonnet.yaml @@ -0,0 +1,61 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 564 + host: "" + body: "{\"max_tokens\":14096,\"messages\":[{\"content\":[{\"text\":\"What's the weather in Florence, Italy?\",\"type\":\"text\"}],\"role\":\"user\"}],\"model\":\"claude-sonnet-4-20250514\",\"system\":[{\"text\":\"You are a helpful assistant\",\"type\":\"text\"}],\"thinking\":{\"budget_tokens\":10000,\"type\":\"enabled\"},\"tool_choice\":{\"disable_parallel_tool_use\":false,\"type\":\"auto\"},\"tools\":[{\"input_schema\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"},\"name\":\"weather\",\"description\":\"Get weather information for a location\"}],\"stream\":true}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - Anthropic/Go 1.10.0 + url: https://api.anthropic.com/v1/messages + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_01VPcbUdEoEcxp65A65U43Cq\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-20250514\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":423,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":6,\"service_tier\":\"standard\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\",\"signature\":\"\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"The user is asking for\"} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" weather information for Florence, Italy. I have\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" access to a weather function that takes a location parameter. The\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" location is clearly specified as \\\"Florence, Italy\\\".\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" I have all the required parameters to make this function call.\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"signature_delta\",\"signature\":\"EqUDCkYIBxgCKkA1dDHptJ2iKtWZ9sMIIJPkgaj5W1miVnddSy3skrvAjGIHqew6UE71EDjqHCF5WYgEVd4SvysajvVTcguSMxHMEgzskB/aBlt0gS1BtUkaDPJSL8B++51vZ5DHlSIw+iMxyK5JptEx7nmgVEe8qK1bXt78PF7K83woxhkziWjrJcj/kndIznLO+qELQpNNKowCh+qjLD0jAIucketOZRE4uiSegDiqlzkenv9exVlEeoFvjiN1zVdgVKpWeylvA3BZYIviwFqgUVGAXjSsWcG+RNvB6SQmNk0PA5R9NmCvckI+Q/6VA9hp2hrjIskceJIsSg3mAtRQ36Rml4ie5ttHDD8f8XevtDu0NS9ymBRf5NPfZHtBPl9AQ32v15XgGS2oYzjn+vd/S/F/hkYd2e9XRnc3hdDa/AmoWbVjS6xU46gT0haidg2LkL79QeX8/2s5FfhIMOizH0XOzlwEHgvBRVBTERGEiSlpUXziHnaUVWbPtTDe42SvwuO/b2npygBoxXkHJyRS0Dz9FCZVs5XM2CoKRN0giFX8szV0rhgB\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01VSoSma6BM9Dhq3RxyiRmAr\",\"name\":\"weather\",\"input\":{}} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"loc\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"ation\\\": \\\"Flo\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"rence, It\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"aly\\\"}\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":1 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null},\"usage\":{\"input_tokens\":423,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":110} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n" + headers: + Content-Type: + - text/event-stream; charset=utf-8 + status: 200 OK + code: 200 + duration: 1.289802542s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 1721 + host: "" + body: "{\"max_tokens\":14096,\"messages\":[{\"content\":[{\"text\":\"What's the weather in Florence, Italy?\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"signature\":\"EqUDCkYIBxgCKkA1dDHptJ2iKtWZ9sMIIJPkgaj5W1miVnddSy3skrvAjGIHqew6UE71EDjqHCF5WYgEVd4SvysajvVTcguSMxHMEgzskB/aBlt0gS1BtUkaDPJSL8B++51vZ5DHlSIw+iMxyK5JptEx7nmgVEe8qK1bXt78PF7K83woxhkziWjrJcj/kndIznLO+qELQpNNKowCh+qjLD0jAIucketOZRE4uiSegDiqlzkenv9exVlEeoFvjiN1zVdgVKpWeylvA3BZYIviwFqgUVGAXjSsWcG+RNvB6SQmNk0PA5R9NmCvckI+Q/6VA9hp2hrjIskceJIsSg3mAtRQ36Rml4ie5ttHDD8f8XevtDu0NS9ymBRf5NPfZHtBPl9AQ32v15XgGS2oYzjn+vd/S/F/hkYd2e9XRnc3hdDa/AmoWbVjS6xU46gT0haidg2LkL79QeX8/2s5FfhIMOizH0XOzlwEHgvBRVBTERGEiSlpUXziHnaUVWbPtTDe42SvwuO/b2npygBoxXkHJyRS0Dz9FCZVs5XM2CoKRN0giFX8szV0rhgB\",\"thinking\":\"The user is asking for weather information for Florence, Italy. I have access to a weather function that takes a location parameter. The location is clearly specified as \\\"Florence, Italy\\\". I have all the required parameters to make this function call.\",\"type\":\"thinking\"},{\"id\":\"toolu_01VSoSma6BM9Dhq3RxyiRmAr\",\"input\":{\"location\":\"Florence, Italy\"},\"name\":\"weather\",\"type\":\"tool_use\"}],\"role\":\"assistant\"},{\"content\":[{\"tool_use_id\":\"toolu_01VSoSma6BM9Dhq3RxyiRmAr\",\"content\":[{\"text\":\"40 C\",\"type\":\"text\"}],\"type\":\"tool_result\"}],\"role\":\"user\"}],\"model\":\"claude-sonnet-4-20250514\",\"system\":[{\"text\":\"You are a helpful assistant\",\"type\":\"text\"}],\"thinking\":{\"budget_tokens\":10000,\"type\":\"enabled\"},\"tool_choice\":{\"disable_parallel_tool_use\":false,\"type\":\"auto\"},\"tools\":[{\"input_schema\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"},\"name\":\"weather\",\"description\":\"Get weather information for a location\"}],\"stream\":true}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - Anthropic/Go 1.10.0 + url: https://api.anthropic.com/v1/messages + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_01GNkVaVsDPcTec33HABjn5e\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-20250514\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":548,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"The\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" weather in Florence, Italy is currently 40°C (104°F),\"} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" which is quite hot! This is typical\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" summer weather for Florence during the warmer months. Make\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" sure to stay hydrated and seek shade if you're planning to be out\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"doors.\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null},\"usage\":{\"input_tokens\":548,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":56} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n" + headers: + Content-Type: + - text/event-stream; charset=utf-8 + status: 200 OK + code: 200 + duration: 1.676578167s diff --git a/providertests/testdata/TestThinkingStreaming/google-gemini-2.5-pro.yaml b/providertests/testdata/TestThinkingStreaming/google-gemini-2.5-pro.yaml new file mode 100644 index 000000000..2292fb205 --- /dev/null +++ b/providertests/testdata/TestThinkingStreaming/google-gemini-2.5-pro.yaml @@ -0,0 +1,63 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 550 + host: generativelanguage.googleapis.com + body: "{\"contents\":[{\"parts\":[{\"text\":\"What's the weather in Florence, Italy?\"}],\"role\":\"user\"}],\"generationConfig\":{\"thinkingConfig\":{\"includeThoughts\":true,\"thinkingBudget\":128}},\"systemInstruction\":{\"parts\":[{\"text\":\"You are a helpful assistant\"}],\"role\":\"user\"},\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"AUTO\"}},\"tools\":[{\"functionDeclarations\":[{\"description\":\"Get weather information for a location\",\"name\":\"weather\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"STRING\"}},\"required\":[\"location\"],\"type\":\"OBJECT\"}}]}]}\n" + form: + alt: + - sse + headers: + Content-Type: + - application/json + User-Agent: + - google-genai-sdk/1.23.0 gl-go/go1.25.0 + url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:streamGenerateContent?alt=sse + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"**Determining Weather Parameters**\\n\\nI've determined that the user's request, \\\"weather in Florence, Italy,\\\" perfectly aligns with the `weather.get_weather` function. It seems the `location` parameter is the key, and I'll confidently set it to \\\"Florence, Italy\\\".\\n\\n\\n\",\"thought\": true}],\"role\": \"model\"},\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 54,\"totalTokenCount\": 121,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 54}],\"thoughtsTokenCount\": 67},\"modelVersion\": \"gemini-2.5-pro\",\"responseId\": \"zZHKaL2DMrPTvdIP4aHWyAs\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"parts\": [{\"functionCall\": {\"name\": \"weather\",\"args\": {\"location\": \"Florence, Italy\"}},\"thoughtSignature\": \"CiIB0e2Kb+4o560+ejjFBllU8TlXrgq4/b/tb77clkkTfKwtCmcB0e2Kb1IrE0Kd0PSfakpKlsp4JG0j3GDD1xu8po3uW1bBSFXZqOXycsUfeT7p/narEgT6HIrPBNkwA1qVwPeAj4D/jUbhIY2Oj2lS8ysNnQeqytNstgmTw32h9vbrhhvOcaH6IH1dCocBAdHtim8NodjIJ2/KTN7Ujms6FtiMUYBQHquMPWyGvJlCrwy2HjucHBKXc4u9og3+2Sd1BKC06BSuzCMiAIF5WqLBhnpTQSBqAoJQmpa856a1FUdmusVWLunaMM1HOuLlZCGNp0KYekg5i6swmIVyAzYOn/4HrEant/3HN8U0q7cPpyfFulYZCnAB0e2Kbxs+yhyWd5uvIZBXkefbF2xrggIwwWmG5FoCMXkhf9fTrrT24QGyHtBDZtjO/+VFvchpcv3XJC11Mfx+T9NpkWFKEQ78DFVVejHgyGEDGkE41mS+da0ejxpAXnHWYsh8Nbz5w9fkGJESlaqK\"}],\"role\": \"model\"},\"finishReason\": \"STOP\",\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 54,\"candidatesTokenCount\": 14,\"totalTokenCount\": 135,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 54}],\"thoughtsTokenCount\": 67},\"modelVersion\": \"gemini-2.5-pro\",\"responseId\": \"zZHKaL2DMrPTvdIP4aHWyAs\"}\r\n\r\n" + headers: + Content-Type: + - text/event-stream + status: 200 OK + code: 200 + duration: 3.416265375s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 776 + host: generativelanguage.googleapis.com + body: "{\"contents\":[{\"parts\":[{\"text\":\"What's the weather in Florence, Italy?\"}],\"role\":\"user\"},{\"parts\":[{\"functionCall\":{\"args\":{\"location\":\"Florence, Italy\"},\"id\":\"weather\",\"name\":\"weather\"}}],\"role\":\"model\"},{\"parts\":[{\"functionResponse\":{\"id\":\"weather\",\"name\":\"weather\",\"response\":{\"result\":\"40 C\"}}}],\"role\":\"user\"}],\"generationConfig\":{\"thinkingConfig\":{\"includeThoughts\":true,\"thinkingBudget\":128}},\"systemInstruction\":{\"parts\":[{\"text\":\"You are a helpful assistant\"}],\"role\":\"user\"},\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"AUTO\"}},\"tools\":[{\"functionDeclarations\":[{\"description\":\"Get weather information for a location\",\"name\":\"weather\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"STRING\"}},\"required\":[\"location\"],\"type\":\"OBJECT\"}}]}]}\n" + form: + alt: + - sse + headers: + Content-Type: + - application/json + User-Agent: + - google-genai-sdk/1.23.0 gl-go/go1.25.0 + url: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:streamGenerateContent?alt=sse + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"The\"}],\"role\": \"model\"},\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 84,\"candidatesTokenCount\": 1,\"totalTokenCount\": 85,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 84}]},\"modelVersion\": \"gemini-2.5-pro\",\"responseId\": \"0pHKaOvQBIXrvdIP44SHgAQ\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \" weather in Florence, Italy is 40 degrees Celsius. \\n\"}],\"role\": \"model\"},\"finishReason\": \"STOP\",\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 84,\"candidatesTokenCount\": 13,\"totalTokenCount\": 97,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 84}]},\"modelVersion\": \"gemini-2.5-pro\",\"responseId\": \"0pHKaOvQBIXrvdIP44SHgAQ\"}\r\n\r\n" + headers: + Content-Type: + - text/event-stream + status: 200 OK + code: 200 + duration: 2.557538458s diff --git a/providertests/testdata/TestThinkingStreaming/openai-gpt-5.yaml b/providertests/testdata/TestThinkingStreaming/openai-gpt-5.yaml new file mode 100644 index 000000000..335246cbc --- /dev/null +++ b/providertests/testdata/TestThinkingStreaming/openai-gpt-5.yaml @@ -0,0 +1,61 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 512 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence, Italy?\",\"role\":\"user\"}],\"model\":\"gpt-5\",\"reasoning_effort\":\"medium\",\"stream_options\":{\"include_usage\":true},\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}],\"stream\":true}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "data: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"index\":0,\"id\":\"call_S4PPNE1CBSTQiyVXNMhypGp9\",\"type\":\"function\",\"function\":{\"name\":\"weather\",\"arguments\":\"\"}}],\"refusal\":null},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"0xwspG0FC\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"\"}}]},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"V1qmL7MMFxIDEJ3\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"location\"}}]},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"AM6r08CpIx\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\":\\\"\"}}]},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"LNb7muLzepVLd\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"Flor\"}}]},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"un0Jtn9FQ9Dh7e\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"ence\"}}]},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"PnIt3YJQ7Odcnm\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\",\"}}]},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"J\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\" Italy\"}}]},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"rNjAaLKtPTCR\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"ogKZ9Um1xvfM6Sm\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"tool_calls\"}],\"usage\":null,\"obfuscation\":\"\"}\n\ndata: {\"id\":\"chatcmpl-CGjzBr5epgv1mOg6v3d0aHGfMitxL\",\"object\":\"chat.completion.chunk\",\"created\":1758106053,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[],\"usage\":{\"prompt_tokens\":145,\"completion_tokens\":153,\"total_tokens\":298,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":128,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"tstpIe\"}\n\ndata: [DONE]\n\n" + headers: + Content-Type: + - text/event-stream; charset=utf-8 + status: 200 OK + code: 200 + duration: 2.807384709s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 764 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence, Italy?\",\"role\":\"user\"},{\"tool_calls\":[{\"id\":\"call_S4PPNE1CBSTQiyVXNMhypGp9\",\"function\":{\"arguments\":\"{\\\"location\\\":\\\"Florence, Italy\\\"}\",\"name\":\"weather\"},\"type\":\"function\"}],\"role\":\"assistant\"},{\"content\":\"40 C\",\"tool_call_id\":\"call_S4PPNE1CBSTQiyVXNMhypGp9\",\"role\":\"tool\"}],\"model\":\"gpt-5\",\"reasoning_effort\":\"medium\",\"stream_options\":{\"include_usage\":true},\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}],\"stream\":true}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "data: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"3dXcrYmrOy\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\"It\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"PqgQgdemta\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\"’s\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"kCefUAWvH9\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\" currently\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"R6\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\" about\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"JzHfhJ\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\" \"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"UnOIco1wqlK\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\"40\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"gxVfChj5p1\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\"°C\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"1Yfx063pRL\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\" in\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"atYPSethq\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\" Florence\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"Hm3\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"ZoMzHd1dUin\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\" Italy\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"eBSjfa\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"UuHUTv0keAz\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":null,\"obfuscation\":\"mHVZv6\"}\n\ndata: {\"id\":\"chatcmpl-CGjzDAZcL5W9B3yoNqJ1XcCg4v4WY\",\"object\":\"chat.completion.chunk\",\"created\":1758106055,\"model\":\"gpt-5-2025-08-07\",\"service_tier\":\"default\",\"system_fingerprint\":null,\"choices\":[],\"usage\":{\"prompt_tokens\":176,\"completion_tokens\":277,\"total_tokens\":453,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":256,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"MMvL8v\"}\n\ndata: [DONE]\n\n" + headers: + Content-Type: + - text/event-stream; charset=utf-8 + status: 200 OK + code: 200 + duration: 4.36330425s From fb3017c0e832f2ae44a76ac8ffc7f17a3eb1ec5e Mon Sep 17 00:00:00 2001 From: Kujtim Hoxha Date: Wed, 17 Sep 2025 14:13:30 +0200 Subject: [PATCH 3/6] chore: remove duplicate name const --- anthropic/anthropic.go | 8 ++++---- anthropic/provider_options.go | 2 -- google/google.go | 6 ++++-- google/provider_options.go | 2 -- openai/openai.go | 8 ++++---- openai/provider_options.go | 2 -- 6 files changed, 12 insertions(+), 16 deletions(-) diff --git a/anthropic/anthropic.go b/anthropic/anthropic.go index 7a6eaf02b..8598db459 100644 --- a/anthropic/anthropic.go +++ b/anthropic/anthropic.go @@ -18,8 +18,8 @@ import ( ) const ( - ProviderName = "anthropic" - DefaultURL = "https://api.anthropic.com" + Name = "anthropic" + DefaultURL = "https://api.anthropic.com" ) type options struct { @@ -45,7 +45,7 @@ func New(opts ...Option) ai.Provider { } providerOptions.baseURL = cmp.Or(providerOptions.baseURL, DefaultURL) - providerOptions.name = cmp.Or(providerOptions.name, ProviderName) + providerOptions.name = cmp.Or(providerOptions.name, Name) return &provider{options: providerOptions} } @@ -97,7 +97,7 @@ func (a *provider) LanguageModel(modelID string) (ai.LanguageModel, error) { } return languageModel{ modelID: modelID, - provider: fmt.Sprintf("%s.messages", a.options.name), + provider: a.options.name, options: a.options, client: anthropic.NewClient(anthropicClientOptions...), }, nil diff --git a/anthropic/provider_options.go b/anthropic/provider_options.go index 5ccfaf6a5..e582edf41 100644 --- a/anthropic/provider_options.go +++ b/anthropic/provider_options.go @@ -2,8 +2,6 @@ package anthropic import "github.com/charmbracelet/fantasy/ai" -const Name = "anthropic" - type ProviderOptions struct { SendReasoning *bool `json:"send_reasoning"` Thinking *ThinkingProviderOption `json:"thinking"` diff --git a/google/google.go b/google/google.go index b32161328..36648a9e2 100644 --- a/google/google.go +++ b/google/google.go @@ -17,6 +17,8 @@ import ( "google.golang.org/genai" ) +const Name = "google" + type provider struct { options options } @@ -38,7 +40,7 @@ func New(opts ...Option) ai.Provider { o(&options) } - options.name = cmp.Or(options.name, "google") + options.name = cmp.Or(options.name, Name) return &provider{ options: options, @@ -101,7 +103,7 @@ func (g *provider) LanguageModel(modelID string) (ai.LanguageModel, error) { } return &languageModel{ modelID: modelID, - provider: fmt.Sprintf("%s.generative-ai", g.options.name), + provider: g.options.name, providerOptions: g.options, client: client, }, nil diff --git a/google/provider_options.go b/google/provider_options.go index 703c277a8..945bff847 100644 --- a/google/provider_options.go +++ b/google/provider_options.go @@ -1,7 +1,5 @@ package google -const Name = "google" - type ThinkingConfig struct { ThinkingBudget *int64 `json:"thinking_budget"` IncludeThoughts *bool `json:"include_thoughts"` diff --git a/openai/openai.go b/openai/openai.go index 4451aba20..e7d51767a 100644 --- a/openai/openai.go +++ b/openai/openai.go @@ -21,8 +21,8 @@ import ( ) const ( - ProviderName = "openai" - DefaultURL = "https://api.openai.com/v1" + Name = "openai" + DefaultURL = "https://api.openai.com/v1" ) type provider struct { @@ -50,7 +50,7 @@ func New(opts ...Option) ai.Provider { } providerOptions.baseURL = cmp.Or(providerOptions.baseURL, DefaultURL) - providerOptions.name = cmp.Or(providerOptions.name, ProviderName) + providerOptions.name = cmp.Or(providerOptions.name, Name) if providerOptions.organization != "" { providerOptions.headers["OpenAi-Organization"] = providerOptions.organization @@ -124,7 +124,7 @@ func (o *provider) LanguageModel(modelID string) (ai.LanguageModel, error) { return languageModel{ modelID: modelID, - provider: fmt.Sprintf("%s.chat", o.options.name), + provider: o.options.name, options: o.options, client: openai.NewClient(openaiClientOptions...), }, nil diff --git a/openai/provider_options.go b/openai/provider_options.go index af3d86fdf..617604883 100644 --- a/openai/provider_options.go +++ b/openai/provider_options.go @@ -5,8 +5,6 @@ import ( "github.com/openai/openai-go/v2" ) -const Name = "openai" - type ReasoningEffort string const ( From b513c4436cc99b8595bc6b71c05fe076adc06bc2 Mon Sep 17 00:00:00 2001 From: Kujtim Hoxha Date: Wed, 17 Sep 2025 14:13:45 +0200 Subject: [PATCH 4/6] test: add more specific reasoning tests --- providertests/provider_test.go | 63 +++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/providertests/provider_test.go b/providertests/provider_test.go index c788f20bb..a43b155ec 100644 --- a/providertests/provider_test.go +++ b/providertests/provider_test.go @@ -2,6 +2,7 @@ package providertests import ( "context" + "fmt" "strconv" "strings" "testing" @@ -10,8 +11,8 @@ import ( "github.com/charmbracelet/fantasy/anthropic" "github.com/charmbracelet/fantasy/google" "github.com/charmbracelet/fantasy/openai" - "github.com/stretchr/testify/require" _ "github.com/joho/godotenv/autoload" + "github.com/stretchr/testify/require" ) func TestSimple(t *testing.T) { @@ -127,6 +128,8 @@ func TestThinking(t *testing.T) { want2 := "40" got := result.Response.Content.Text() require.True(t, strings.Contains(got, want1) && strings.Contains(got, want2), "unexpected response: got %q, want %q %q", got, want1, want2) + + testThinkingSteps(t, languageModel.Provider(), result.Steps) }) } } @@ -181,10 +184,68 @@ func TestThinkingStreaming(t *testing.T) { want2 := "40" got := result.Response.Content.Text() require.True(t, strings.Contains(got, want1) && strings.Contains(got, want2), "unexpected response: got %q, want %q %q", got, want1, want2) + + testThinkingSteps(t, languageModel.Provider(), result.Steps) }) } } +func testThinkingSteps(t *testing.T, providerName string, steps []ai.StepResult) { + if providerName == anthropic.Name { + reasoningContentCount := 0 + signaturesCount := 0 + // Test if we got the signature + for _, step := range steps { + for _, msg := range step.Messages { + for _, content := range msg.Content { + if content.GetType() == ai.ContentTypeReasoning { + reasoningContentCount += 1 + reasoningContent, ok := ai.AsContentType[ai.ReasoningPart](content) + if !ok { + continue + } + if len(reasoningContent.ProviderOptions) == 0 { + continue + } + + anthropicReasoningMetadata, ok := reasoningContent.ProviderOptions[anthropic.Name] + if !ok { + continue + } + if reasoningContent.Text != "" { + if typed, ok := anthropicReasoningMetadata.(*anthropic.ReasoningOptionMetadata); ok { + require.NotEmpty(t, typed.Signature) + signaturesCount += 1 + } + } + } + } + } + } + require.Greater(t, reasoningContentCount, 0) + require.Greater(t, signaturesCount, 0) + require.Equal(t, reasoningContentCount, signaturesCount) + } else if providerName == google.Name { + reasoningContentCount := 0 + // Test if we got the signature + for _, step := range steps { + for _, msg := range step.Messages { + for _, content := range msg.Content { + if content.GetType() == ai.ContentTypeReasoning { + reasoningContentCount += 1 + reasoningContent, ok := ai.AsContentType[ai.ReasoningContent](content) + if !ok { + continue + } + fmt.Println(reasoningContent.Text) + } + } + } + } + require.Greater(t, reasoningContentCount, 0) + } +} + func TestStream(t *testing.T) { for _, pair := range languageModelBuilders { t.Run(pair.name, func(t *testing.T) { From 15452825111d70e8378a0eaed3382b3d98a387a8 Mon Sep 17 00:00:00 2001 From: Kujtim Hoxha Date: Wed, 17 Sep 2025 14:16:44 +0200 Subject: [PATCH 5/6] chore: remove some annoying spell checks --- cspell.json | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/cspell.json b/cspell.json index 4436732c9..dad6dc4e9 100644 --- a/cspell.json +++ b/cspell.json @@ -1,9 +1 @@ -{ - "language": "en", - "version": "0.2", - "flagWords": [], - "words": [ - "mapstructure", - "mapstructure" - ] -} +{"language":"en","words":["mapstructure","mapstructure","charmbracelet","providertests","joho","godotenv","stretchr"],"version":"0.2","flagWords":[]} \ No newline at end of file From be3c1fae2690b33e58dfee46f82d2bb728546514 Mon Sep 17 00:00:00 2001 From: kujtimiihoxha Date: Wed, 17 Sep 2025 17:19:31 +0200 Subject: [PATCH 6/6] chore: reorganize thinking tests --- providertests/provider_test.go | 57 --------------------------- providertests/thinking_test.go | 70 ++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 57 deletions(-) create mode 100644 providertests/thinking_test.go diff --git a/providertests/provider_test.go b/providertests/provider_test.go index a43b155ec..acc30e78d 100644 --- a/providertests/provider_test.go +++ b/providertests/provider_test.go @@ -2,7 +2,6 @@ package providertests import ( "context" - "fmt" "strconv" "strings" "testing" @@ -190,62 +189,6 @@ func TestThinkingStreaming(t *testing.T) { } } -func testThinkingSteps(t *testing.T, providerName string, steps []ai.StepResult) { - if providerName == anthropic.Name { - reasoningContentCount := 0 - signaturesCount := 0 - // Test if we got the signature - for _, step := range steps { - for _, msg := range step.Messages { - for _, content := range msg.Content { - if content.GetType() == ai.ContentTypeReasoning { - reasoningContentCount += 1 - reasoningContent, ok := ai.AsContentType[ai.ReasoningPart](content) - if !ok { - continue - } - if len(reasoningContent.ProviderOptions) == 0 { - continue - } - - anthropicReasoningMetadata, ok := reasoningContent.ProviderOptions[anthropic.Name] - if !ok { - continue - } - if reasoningContent.Text != "" { - if typed, ok := anthropicReasoningMetadata.(*anthropic.ReasoningOptionMetadata); ok { - require.NotEmpty(t, typed.Signature) - signaturesCount += 1 - } - } - } - } - } - } - require.Greater(t, reasoningContentCount, 0) - require.Greater(t, signaturesCount, 0) - require.Equal(t, reasoningContentCount, signaturesCount) - } else if providerName == google.Name { - reasoningContentCount := 0 - // Test if we got the signature - for _, step := range steps { - for _, msg := range step.Messages { - for _, content := range msg.Content { - if content.GetType() == ai.ContentTypeReasoning { - reasoningContentCount += 1 - reasoningContent, ok := ai.AsContentType[ai.ReasoningContent](content) - if !ok { - continue - } - fmt.Println(reasoningContent.Text) - } - } - } - } - require.Greater(t, reasoningContentCount, 0) - } -} - func TestStream(t *testing.T) { for _, pair := range languageModelBuilders { t.Run(pair.name, func(t *testing.T) { diff --git a/providertests/thinking_test.go b/providertests/thinking_test.go new file mode 100644 index 000000000..e4a9363be --- /dev/null +++ b/providertests/thinking_test.go @@ -0,0 +1,70 @@ +package providertests + +import ( + "testing" + + "github.com/charmbracelet/fantasy/ai" + "github.com/charmbracelet/fantasy/anthropic" + "github.com/charmbracelet/fantasy/google" + "github.com/stretchr/testify/require" +) + +func testThinkingSteps(t *testing.T, providerName string, steps []ai.StepResult) { + switch providerName { + case anthropic.Name: + testAnthropicThinking(t, steps) + case google.Name: + testGoogleThinking(t, steps) + } +} + +func testGoogleThinking(t *testing.T, steps []ai.StepResult) { + reasoningContentCount := 0 + // Test if we got the signature + for _, step := range steps { + for _, msg := range step.Messages { + for _, content := range msg.Content { + if content.GetType() == ai.ContentTypeReasoning { + reasoningContentCount += 1 + } + } + } + } + require.Greater(t, reasoningContentCount, 0) +} + +func testAnthropicThinking(t *testing.T, steps []ai.StepResult) { + reasoningContentCount := 0 + signaturesCount := 0 + // Test if we got the signature + for _, step := range steps { + for _, msg := range step.Messages { + for _, content := range msg.Content { + if content.GetType() == ai.ContentTypeReasoning { + reasoningContentCount += 1 + reasoningContent, ok := ai.AsContentType[ai.ReasoningPart](content) + if !ok { + continue + } + if len(reasoningContent.ProviderOptions) == 0 { + continue + } + + anthropicReasoningMetadata, ok := reasoningContent.ProviderOptions[anthropic.Name] + if !ok { + continue + } + if reasoningContent.Text != "" { + if typed, ok := anthropicReasoningMetadata.(*anthropic.ReasoningOptionMetadata); ok { + require.NotEmpty(t, typed.Signature) + signaturesCount += 1 + } + } + } + } + } + } + require.Greater(t, reasoningContentCount, 0) + require.Greater(t, signaturesCount, 0) + require.Equal(t, reasoningContentCount, signaturesCount) +}