From e74fe08014943369989b081a2b0a12a92405659d Mon Sep 17 00:00:00 2001 From: Andrey Nering Date: Wed, 3 Sep 2025 18:00:00 -0300 Subject: [PATCH 1/8] test: setup go-vcr to start recording real providers interactions --- .gitattributes | 1 + go.mod | 3 + go.sum | 6 ++ providertests/.env.sample | 2 + providertests/builders_test.go | 47 ++++++++++++ providertests/provider_test.go | 39 ++++++++++ providertests/recorder_test.go | 74 +++++++++++++++++++ .../TestSimple/anthropic-claude-sonnet.yaml | 33 +++++++++ .../testdata/TestSimple/openai-gpt-4o.yaml | 33 +++++++++ 9 files changed, 238 insertions(+) create mode 100644 .gitattributes create mode 100644 providertests/.env.sample create mode 100644 providertests/builders_test.go create mode 100644 providertests/provider_test.go create mode 100644 providertests/recorder_test.go create mode 100644 providertests/testdata/TestSimple/anthropic-claude-sonnet.yaml create mode 100644 providertests/testdata/TestSimple/openai-gpt-4o.yaml diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..46d706f3c --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +providertests/testdata/**/*.yaml linguist-generated=true diff --git a/go.mod b/go.mod index 1353ace5d..90f60c757 100644 --- a/go.mod +++ b/go.mod @@ -7,12 +7,15 @@ require ( github.com/charmbracelet/x/json v0.2.0 github.com/go-viper/mapstructure/v2 v2.4.0 github.com/google/uuid v1.6.0 + github.com/joho/godotenv v1.5.1 github.com/openai/openai-go/v2 v2.3.0 github.com/stretchr/testify v1.11.1 + gopkg.in/dnaeon/go-vcr.v4 v4.0.5 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/goccy/go-yaml v1.18.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.1.1 // indirect diff --git a/go.sum b/go.sum index f0842950e..93ed55706 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/openai/openai-go/v2 v2.3.0 h1:y9U+V1tlHjvvb/5XIswuySqnG5EnKBFAbMxgBvTHXvg= github.com/openai/openai-go/v2 v2.3.0/go.mod h1:sIUkR+Cu/PMUVkSKhkk742PRURkQOCFhiwJ7eRSBqmk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -26,5 +30,7 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/dnaeon/go-vcr.v4 v4.0.5 h1:I0hpTIvD5rII+8LgYGrHMA2d4SQPoL6u7ZvJakWKsiA= +gopkg.in/dnaeon/go-vcr.v4 v4.0.5/go.mod h1:dRos81TkW9C1WJt6tTaE+uV2Lo8qJT3AG2b35+CB/nQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/providertests/.env.sample b/providertests/.env.sample new file mode 100644 index 000000000..69b599454 --- /dev/null +++ b/providertests/.env.sample @@ -0,0 +1,2 @@ +ANTHROPIC_API_KEY= +OPENAI_API_KEY= diff --git a/providertests/builders_test.go b/providertests/builders_test.go new file mode 100644 index 000000000..3d5e080ce --- /dev/null +++ b/providertests/builders_test.go @@ -0,0 +1,47 @@ +package providertests + +import ( + "net/http" + "os" + + "github.com/charmbracelet/ai/ai" + "github.com/charmbracelet/ai/anthropic" + "github.com/charmbracelet/ai/openai" + "gopkg.in/dnaeon/go-vcr.v4/pkg/recorder" +) + +type builderFunc func(r *recorder.Recorder) (ai.LanguageModel, error) + +type builderPair struct { + name string + builder builderFunc +} + +var languageModelBuilders = []builderPair{ + {"openai-gpt-4o", builderOpenaiGpt4o}, + {"anthropic-claude-sonnet", builderAnthropicClaudeSonnet4}, +} + +func builderOpenaiGpt4o(r *recorder.Recorder) (ai.LanguageModel, error) { + provider := openai.New( + openai.WithAPIKey(os.Getenv("OPENAI_API_KEY")), + openai.WithHTTPClient(&http.Client{Transport: r}), + ) + model, err := provider.LanguageModel("gpt-4o") + if err != nil { + return nil, err + } + return model, nil +} + +func builderAnthropicClaudeSonnet4(r *recorder.Recorder) (ai.LanguageModel, error) { + provider := anthropic.New( + anthropic.WithAPIKey(os.Getenv("ANTHROPIC_API_KEY")), + anthropic.WithHTTPClient(&http.Client{Transport: r}), + ) + model, err := provider.LanguageModel("claude-sonnet-4-20250514") + if err != nil { + return nil, err + } + return model, nil +} diff --git a/providertests/provider_test.go b/providertests/provider_test.go new file mode 100644 index 000000000..0f77692df --- /dev/null +++ b/providertests/provider_test.go @@ -0,0 +1,39 @@ +package providertests + +import ( + "strings" + "testing" + + "github.com/charmbracelet/ai/ai" + _ "github.com/joho/godotenv/autoload" +) + +func TestSimple(t *testing.T) { + for _, pair := range languageModelBuilders { + t.Run(pair.name, func(t *testing.T) { + r := newRecorder(t) + + languageModel, err := pair.builder(r) + if err != nil { + t.Fatalf("failed to build language model: %v", err) + } + + agent := ai.NewAgent( + languageModel, + ai.WithSystemPrompt("You are a helpful assistant"), + ) + result, err := agent.Generate(t.Context(), ai.AgentCall{ + Prompt: "Say hi in Portuguese", + }) + if err != nil { + t.Fatalf("failed to generate: %v", err) + } + + want := "Olá" + got := result.Response.Content.Text() + if !strings.Contains(got, want) { + t.Fatalf("unexpected response: got %q, want %q", got, want) + } + }) + } +} diff --git a/providertests/recorder_test.go b/providertests/recorder_test.go new file mode 100644 index 000000000..0ff64571e --- /dev/null +++ b/providertests/recorder_test.go @@ -0,0 +1,74 @@ +package providertests + +import ( + "bytes" + "io" + "net/http" + "path/filepath" + "strings" + "testing" + + "gopkg.in/dnaeon/go-vcr.v4/pkg/cassette" + "gopkg.in/dnaeon/go-vcr.v4/pkg/recorder" +) + +func newRecorder(t *testing.T) *recorder.Recorder { + cassetteName := filepath.Join("testdata", t.Name()) + + r, err := recorder.New( + cassetteName, + recorder.WithMode(recorder.ModeRecordOnce), + recorder.WithMatcher(customMatcher(t)), + recorder.WithHook(hookRemoveHeaders, recorder.AfterCaptureHook), + ) + if err != nil { + t.Fatalf("recorder: failed to create recorder: %v", err) + } + + t.Cleanup(func() { + if err := r.Stop(); err != nil { + t.Errorf("recorder: failed to stop recorder: %v", err) + } + }) + + return r +} + +func customMatcher(t *testing.T) recorder.MatcherFunc { + return func(r *http.Request, i cassette.Request) bool { + if r.Body == nil || r.Body == http.NoBody { + return cassette.DefaultMatcher(r, i) + } + + var reqBody []byte + var err error + reqBody, err = io.ReadAll(r.Body) + if err != nil { + t.Fatalf("recorder: failed to read request body") + } + r.Body.Close() + r.Body = io.NopCloser(bytes.NewBuffer(reqBody)) + + return r.Method == i.Method && r.URL.String() == i.URL && string(reqBody) == i.Body + } +} + +var headersToKeep = map[string]struct{}{ + "accept": {}, + "content-type": {}, + "user-agent": {}, +} + +func hookRemoveHeaders(i *cassette.Interaction) error { + for k := range i.Request.Headers { + if _, ok := headersToKeep[strings.ToLower(k)]; !ok { + delete(i.Request.Headers, k) + } + } + for k := range i.Response.Headers { + if _, ok := headersToKeep[strings.ToLower(k)]; !ok { + delete(i.Response.Headers, k) + } + } + return nil +} diff --git a/providertests/testdata/TestSimple/anthropic-claude-sonnet.yaml b/providertests/testdata/TestSimple/anthropic-claude-sonnet.yaml new file mode 100644 index 000000000..00c497012 --- /dev/null +++ b/providertests/testdata/TestSimple/anthropic-claude-sonnet.yaml @@ -0,0 +1,33 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 205 + host: "" + body: "{\"max_tokens\":4096,\"messages\":[{\"content\":[{\"text\":\"Say hi in Portuguese\",\"type\":\"text\"}],\"role\":\"user\"}],\"model\":\"claude-sonnet-4-20250514\",\"system\":[{\"text\":\"You are a helpful assistant\",\"type\":\"text\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - Anthropic/Go 1.10.0 + url: https://api.anthropic.com/v1/messages + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\"id\":\"msg_014AQFTJZeZ1KNGT5y9TSMSs\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-20250514\",\"content\":[{\"type\":\"text\",\"text\":\"Oi! or Olá!\\n\\nBoth are common ways to say \\\"hi\\\" in Portuguese. \\\"Oi\\\" is more casual and commonly used in Brazilian Portuguese, while \\\"Olá\\\" is a bit more formal and used in both Brazilian and European Portuguese.\"}],\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"usage\":{\"input_tokens\":16,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":60,\"service_tier\":\"standard\"}}" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 2.412166125s diff --git a/providertests/testdata/TestSimple/openai-gpt-4o.yaml b/providertests/testdata/TestSimple/openai-gpt-4o.yaml new file mode 100644 index 000000000..72b7135ec --- /dev/null +++ b/providertests/testdata/TestSimple/openai-gpt-4o.yaml @@ -0,0 +1,33 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 138 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"Say hi in Portuguese\",\"role\":\"user\"}],\"model\":\"gpt-4o\"}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"id\": \"chatcmpl-CBolfurp0H2jFXSwJHVGbLWOYHhbM\",\n \"object\": \"chat.completion\",\n \"created\": 1756932795,\n \"model\": \"gpt-4o-2024-08-06\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"Olá! Como posso ajudar você hoje?\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 20,\n \"completion_tokens\": 8,\n \"total_tokens\": 28,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_f33640a400\"\n}\n" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 3.363218s From 838319c620f250787458171ad906ed823bdc4f46 Mon Sep 17 00:00:00 2001 From: Andrey Nering Date: Thu, 4 Sep 2025 11:45:39 -0300 Subject: [PATCH 2/8] chore: disable go-vcr's internal "sleep" to make tests much faster --- providertests/recorder_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/providertests/recorder_test.go b/providertests/recorder_test.go index 0ff64571e..f1710e7cd 100644 --- a/providertests/recorder_test.go +++ b/providertests/recorder_test.go @@ -19,6 +19,7 @@ func newRecorder(t *testing.T) *recorder.Recorder { cassetteName, recorder.WithMode(recorder.ModeRecordOnce), recorder.WithMatcher(customMatcher(t)), + recorder.WithSkipRequestLatency(true), // disable sleep to simulate response time, makes tests faster recorder.WithHook(hookRemoveHeaders, recorder.AfterCaptureHook), ) if err != nil { From 381a037c2bad9d1ad4fdc75b48499f727a84e5a9 Mon Sep 17 00:00:00 2001 From: Andrey Nering Date: Thu, 4 Sep 2025 11:51:22 -0300 Subject: [PATCH 3/8] test: add test with tool --- providertests/provider_test.go | 45 +++++++++++++ .../TestTool/anthropic-claude-sonnet.yaml | 63 +++++++++++++++++++ .../testdata/TestTool/openai-gpt-4o.yaml | 63 +++++++++++++++++++ 3 files changed, 171 insertions(+) create mode 100644 providertests/testdata/TestTool/anthropic-claude-sonnet.yaml create mode 100644 providertests/testdata/TestTool/openai-gpt-4o.yaml diff --git a/providertests/provider_test.go b/providertests/provider_test.go index 0f77692df..81cb0cb52 100644 --- a/providertests/provider_test.go +++ b/providertests/provider_test.go @@ -1,6 +1,7 @@ package providertests import ( + "context" "strings" "testing" @@ -37,3 +38,47 @@ func TestSimple(t *testing.T) { }) } } + +func TestTool(t *testing.T) { + for _, pair := range languageModelBuilders { + t.Run(pair.name, func(t *testing.T) { + r := newRecorder(t) + + languageModel, err := pair.builder(r) + if err != nil { + t.Fatalf("failed to build language model: %v", err) + } + + type WeatherInput struct { + Location string `json:"location" description:"the city"` + } + + weatherTool := ai.NewAgentTool( + "weather", + "Get weather information for a location", + func(ctx context.Context, input WeatherInput, _ ai.ToolCall) (ai.ToolResponse, error) { + return ai.NewTextResponse("40 C"), nil + }, + ) + + agent := ai.NewAgent( + languageModel, + ai.WithSystemPrompt("You are a helpful assistant"), + ai.WithTools(weatherTool), + ) + result, err := agent.Generate(t.Context(), ai.AgentCall{ + Prompt: "What's the weather in Florence?", + }) + if err != nil { + t.Fatalf("failed to generate: %v", err) + } + + want1 := "Florence" + want2 := "40" + got := result.Response.Content.Text() + if !strings.Contains(got, want1) || !strings.Contains(got, want2) { + t.Fatalf("unexpected response: got %q, want %q %q", got, want1, want2) + } + }) + } +} diff --git a/providertests/testdata/TestTool/anthropic-claude-sonnet.yaml b/providertests/testdata/TestTool/anthropic-claude-sonnet.yaml new file mode 100644 index 000000000..7babf85fe --- /dev/null +++ b/providertests/testdata/TestTool/anthropic-claude-sonnet.yaml @@ -0,0 +1,63 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 490 + host: "" + body: "{\"max_tokens\":4096,\"messages\":[{\"content\":[{\"text\":\"What's the weather in Florence?\",\"type\":\"text\"}],\"role\":\"user\"}],\"model\":\"claude-sonnet-4-20250514\",\"system\":[{\"text\":\"You are a helpful assistant\",\"type\":\"text\"}],\"tool_choice\":{\"disable_parallel_tool_use\":false,\"type\":\"auto\"},\"tools\":[{\"input_schema\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"},\"name\":\"weather\",\"description\":\"Get weather information for a location\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - Anthropic/Go 1.10.0 + url: https://api.anthropic.com/v1/messages + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\"id\":\"msg_01EnEXYvvJsZqCkmy6JSJbmZ\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-20250514\",\"content\":[{\"type\":\"text\",\"text\":\"I'll get the weather information for Florence for you.\"},{\"type\":\"tool_use\",\"id\":\"toolu_01DuHpbePNYnbErjgBf5iHsF\",\"name\":\"weather\",\"input\":{\"location\":\"Florence\"}}],\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"usage\":{\"input_tokens\":392,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":63,\"service_tier\":\"standard\"}}" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 2.213104625s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 850 + host: "" + body: "{\"max_tokens\":4096,\"messages\":[{\"content\":[{\"text\":\"What's the weather in Florence?\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"text\":\"I'll get the weather information for Florence for you.\",\"type\":\"text\"},{\"id\":\"toolu_01DuHpbePNYnbErjgBf5iHsF\",\"input\":{\"location\":\"Florence\"},\"name\":\"weather\",\"type\":\"tool_use\"}],\"role\":\"assistant\"},{\"content\":[{\"tool_use_id\":\"toolu_01DuHpbePNYnbErjgBf5iHsF\",\"content\":[{\"text\":\"40 C\",\"type\":\"text\"}],\"type\":\"tool_result\"}],\"role\":\"user\"}],\"model\":\"claude-sonnet-4-20250514\",\"system\":[{\"text\":\"You are a helpful assistant\",\"type\":\"text\"}],\"tool_choice\":{\"disable_parallel_tool_use\":false,\"type\":\"auto\"},\"tools\":[{\"input_schema\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"},\"name\":\"weather\",\"description\":\"Get weather information for a location\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - Anthropic/Go 1.10.0 + url: https://api.anthropic.com/v1/messages + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\"id\":\"msg_01HyNDRGB4Vs42jfUPPcBaak\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-20250514\",\"content\":[{\"type\":\"text\",\"text\":\"The current temperature in Florence is 40°C (104°F). That's quite hot! Make sure to stay hydrated and seek shade or air conditioning if you're in the area.\"}],\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"usage\":{\"input_tokens\":470,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":42,\"service_tier\":\"standard\"}}" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 2.306400542s diff --git a/providertests/testdata/TestTool/openai-gpt-4o.yaml b/providertests/testdata/TestTool/openai-gpt-4o.yaml new file mode 100644 index 000000000..3a2a9ad71 --- /dev/null +++ b/providertests/testdata/TestTool/openai-gpt-4o.yaml @@ -0,0 +1,63 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 424 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence?\",\"role\":\"user\"}],\"model\":\"gpt-4o\",\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"id\": \"chatcmpl-CC5MhNb2zdVtCwHG9w33J2Ht2fRLk\",\n \"object\": \"chat.completion\",\n \"created\": 1756996595,\n \"model\": \"gpt-4o-2024-08-06\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_VDWPAXXYtOgpIp7VtHaQDlFz\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"weather\",\n \"arguments\": \"{\\\"location\\\":\\\"Florence\\\"}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 59,\n \"completion_tokens\": 14,\n \"total_tokens\": 73,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_f33640a400\"\n}\n" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 2.399816792s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 669 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence?\",\"role\":\"user\"},{\"tool_calls\":[{\"id\":\"call_VDWPAXXYtOgpIp7VtHaQDlFz\",\"function\":{\"arguments\":\"{\\\"location\\\":\\\"Florence\\\"}\",\"name\":\"weather\"},\"type\":\"function\"}],\"role\":\"assistant\"},{\"content\":\"40 C\",\"tool_call_id\":\"call_VDWPAXXYtOgpIp7VtHaQDlFz\",\"role\":\"tool\"}],\"model\":\"gpt-4o\",\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"id\": \"chatcmpl-CC5MmCn0b4nV6e0m4ZGC7tX1t0JhL\",\n \"object\": \"chat.completion\",\n \"created\": 1756996600,\n \"model\": \"gpt-4o-2024-08-06\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The current temperature in Florence is 40°C.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 82,\n \"completion_tokens\": 11,\n \"total_tokens\": 93,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_f33640a400\"\n}\n" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 2.093890917s From 5209835defbf854b03da284c19086fe3c13f5c55 Mon Sep 17 00:00:00 2001 From: Andrey Nering Date: Thu, 4 Sep 2025 13:52:23 -0300 Subject: [PATCH 4/8] test: add tests for stream with and without tool calls --- providertests/provider_test.go | 138 ++++++++++++++++++ .../TestStream/anthropic-claude-sonnet.yaml | 32 ++++ .../testdata/TestStream/openai-gpt-4o.yaml | 32 ++++ .../anthropic-claude-sonnet.yaml | 61 ++++++++ .../TestStreamWithTools/openai-gpt-4o.yaml | 61 ++++++++ 5 files changed, 324 insertions(+) create mode 100644 providertests/testdata/TestStream/anthropic-claude-sonnet.yaml create mode 100644 providertests/testdata/TestStream/openai-gpt-4o.yaml create mode 100644 providertests/testdata/TestStreamWithTools/anthropic-claude-sonnet.yaml create mode 100644 providertests/testdata/TestStreamWithTools/openai-gpt-4o.yaml diff --git a/providertests/provider_test.go b/providertests/provider_test.go index 81cb0cb52..fa30a55e7 100644 --- a/providertests/provider_test.go +++ b/providertests/provider_test.go @@ -2,6 +2,7 @@ package providertests import ( "context" + "strconv" "strings" "testing" @@ -82,3 +83,140 @@ func TestTool(t *testing.T) { }) } } + +func TestStream(t *testing.T) { + for _, pair := range languageModelBuilders { + t.Run(pair.name, func(t *testing.T) { + r := newRecorder(t) + + languageModel, err := pair.builder(r) + if err != nil { + t.Fatalf("failed to build language model: %v", err) + } + + agent := ai.NewAgent( + languageModel, + ai.WithSystemPrompt("You are a helpful assistant"), + ) + + var collectedText strings.Builder + textDeltaCount := 0 + stepCount := 0 + + streamCall := ai.AgentStreamCall{ + Prompt: "Count from 1 to 3 in Spanish", + OnTextDelta: func(id, text string) error { + textDeltaCount++ + collectedText.WriteString(text) + return nil + }, + OnStepFinish: func(step ai.StepResult) error { + stepCount++ + return nil + }, + } + + result, err := agent.Stream(t.Context(), streamCall) + if err != nil { + t.Fatalf("failed to stream: %v", err) + } + + finalText := result.Response.Content.Text() + if finalText == "" { + t.Fatal("expected non-empty response") + } + + if !strings.Contains(strings.ToLower(finalText), "uno") || + !strings.Contains(strings.ToLower(finalText), "dos") || + !strings.Contains(strings.ToLower(finalText), "tres") { + t.Fatalf("unexpected response: %q", finalText) + } + + if textDeltaCount == 0 { + t.Fatal("expected at least one text delta callback") + } + + if stepCount == 0 { + t.Fatal("expected at least one step finish callback") + } + + if collectedText.String() == "" { + t.Fatal("expected collected text from deltas to be non-empty") + } + }) + } +} + +func TestStreamWithTools(t *testing.T) { + for _, pair := range languageModelBuilders { + t.Run(pair.name, func(t *testing.T) { + r := newRecorder(t) + + languageModel, err := pair.builder(r) + if err != nil { + t.Fatalf("failed to build language model: %v", err) + } + + type CalculatorInput struct { + A int `json:"a" description:"first number"` + B int `json:"b" description:"second number"` + } + + calculatorTool := ai.NewAgentTool( + "add", + "Add two numbers", + func(ctx context.Context, input CalculatorInput, _ ai.ToolCall) (ai.ToolResponse, error) { + result := input.A + input.B + return ai.NewTextResponse(strings.TrimSpace(strconv.Itoa(result))), nil + }, + ) + + agent := ai.NewAgent( + languageModel, + ai.WithSystemPrompt("You are a helpful assistant. Use the add tool to perform calculations."), + ai.WithTools(calculatorTool), + ) + + toolCallCount := 0 + toolResultCount := 0 + var collectedText strings.Builder + + streamCall := ai.AgentStreamCall{ + Prompt: "What is 15 + 27?", + OnTextDelta: func(id, text string) error { + collectedText.WriteString(text) + return nil + }, + OnToolCall: func(toolCall ai.ToolCallContent) error { + toolCallCount++ + if toolCall.ToolName != "add" { + t.Errorf("unexpected tool name: %s", toolCall.ToolName) + } + return nil + }, + OnToolResult: func(result ai.ToolResultContent) error { + toolResultCount++ + return nil + }, + } + + result, err := agent.Stream(t.Context(), streamCall) + if err != nil { + t.Fatalf("failed to stream: %v", err) + } + + finalText := result.Response.Content.Text() + if !strings.Contains(finalText, "42") { + t.Fatalf("expected response to contain '42', got: %q", finalText) + } + + if toolCallCount == 0 { + t.Fatal("expected at least one tool call") + } + + if toolResultCount == 0 { + t.Fatal("expected at least one tool result") + } + }) + } +} diff --git a/providertests/testdata/TestStream/anthropic-claude-sonnet.yaml b/providertests/testdata/TestStream/anthropic-claude-sonnet.yaml new file mode 100644 index 000000000..a5ece02f5 --- /dev/null +++ b/providertests/testdata/TestStream/anthropic-claude-sonnet.yaml @@ -0,0 +1,32 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 227 + host: "" + body: "{\"max_tokens\":4096,\"messages\":[{\"content\":[{\"text\":\"Count from 1 to 3 in Spanish\",\"type\":\"text\"}],\"role\":\"user\"}],\"model\":\"claude-sonnet-4-20250514\",\"system\":[{\"text\":\"You are a helpful assistant\",\"type\":\"text\"}],\"stream\":true}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - Anthropic/Go 1.10.0 + url: https://api.anthropic.com/v1/messages + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_01LC9keN4A55kbF4ZSERTbqh\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-20250514\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":23,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"1 - uno\"} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"\\n2 - dos \\n3\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" - tres\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null},\"usage\":{\"input_tokens\":23,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":19} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n" + headers: + Content-Type: + - text/event-stream; charset=utf-8 + status: 200 OK + code: 200 + duration: 924.937333ms diff --git a/providertests/testdata/TestStream/openai-gpt-4o.yaml b/providertests/testdata/TestStream/openai-gpt-4o.yaml new file mode 100644 index 000000000..ac47d4482 --- /dev/null +++ b/providertests/testdata/TestStream/openai-gpt-4o.yaml @@ -0,0 +1,32 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 200 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"Count from 1 to 3 in Spanish\",\"role\":\"user\"}],\"model\":\"gpt-4o\",\"stream_options\":{\"include_usage\":true},\"stream\":true}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "data: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"iussH0LBg1XxJN\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Sure\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"Ut9uBKsz3oTx\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"!\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"7GBuOOfjVIPLjDY\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" In\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"d9GWSSttWv8lN\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" Spanish\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"ptr1Mq2J\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"KrKRxanTn8EuWAe\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" counting\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"KzaUquV\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" from\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"FkX07hTx3TF\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"2ZMZIj2oiXvLcxn\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"1\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"UX1rge5Ob1J0kOI\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" to\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"BAGM5tj5WkNjW\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"qQ0ETIlKihAOAE7\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"3\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"Gl1TPKRizlWwiwI\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" is\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"SF4BgD7Pw39gI\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\":\\n\\n\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"B40X3SCV1tv\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"1\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"ZUpUw67SCpRAtht\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"3bNT0rvK44BGeqi\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" Uno\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"L6bruBJ5A7fA\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\\n\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"tPHymjAuf4HtlD\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"2\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"C4lbztDMhus6xvv\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"JBrRK1Ly6feOSPa\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" Dos\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"zMXBzcxHyuX6\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\\n\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"6YPK2aWh7M9ACK\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"3\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"V8BqeMEkcu7GvU5\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"nfRygsLG3J6zfnG\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" Tres\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"23HCbgMjAso\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":null,\"obfuscation\":\"q71xDuvFhX\"}\n\ndata: {\"id\":\"chatcmpl-CC7KTblMWEfVKNHjLNnFwvi0m8DFE\",\"object\":\"chat.completion.chunk\",\"created\":1757004145,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[],\"usage\":{\"prompt_tokens\":25,\"completion_tokens\":25,\"total_tokens\":50,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"EBBsHlGQYAdi5Z6\"}\n\ndata: [DONE]\n\n" + headers: + Content-Type: + - text/event-stream; charset=utf-8 + status: 200 OK + code: 200 + duration: 1.619950917s diff --git a/providertests/testdata/TestStreamWithTools/anthropic-claude-sonnet.yaml b/providertests/testdata/TestStreamWithTools/anthropic-claude-sonnet.yaml new file mode 100644 index 000000000..d67c96cb5 --- /dev/null +++ b/providertests/testdata/TestStreamWithTools/anthropic-claude-sonnet.yaml @@ -0,0 +1,61 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 553 + host: "" + body: "{\"max_tokens\":4096,\"messages\":[{\"content\":[{\"text\":\"What is 15 + 27?\",\"type\":\"text\"}],\"role\":\"user\"}],\"model\":\"claude-sonnet-4-20250514\",\"system\":[{\"text\":\"You are a helpful assistant. Use the add tool to perform calculations.\",\"type\":\"text\"}],\"tool_choice\":{\"disable_parallel_tool_use\":false,\"type\":\"auto\"},\"tools\":[{\"input_schema\":{\"properties\":{\"a\":{\"description\":\"first number\",\"type\":\"integer\"},\"b\":{\"description\":\"second number\",\"type\":\"integer\"}},\"required\":[\"a\",\"b\"],\"type\":\"object\"},\"name\":\"add\",\"description\":\"Add two numbers\"}],\"stream\":true}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - Anthropic/Go 1.10.0 + url: https://api.anthropic.com/v1/messages + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_01KtoJaXMqmtVtLnjtX5cPUT\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-20250514\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":420,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":8,\"service_tier\":\"standard\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"I'll help you calculate 15 \"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"+ 27 using the add function.\"}}\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_015ewucD8KduCw15A5kPth7R\",\"name\":\"add\",\"input\":{}} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"a\\\":\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\" 15\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\", \\\"b\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\": \"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"27}\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":1 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null},\"usage\":{\"input_tokens\":420,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":86}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n" + headers: + Content-Type: + - text/event-stream; charset=utf-8 + status: 200 OK + code: 200 + duration: 1.174656584s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 900 + host: "" + body: "{\"max_tokens\":4096,\"messages\":[{\"content\":[{\"text\":\"What is 15 + 27?\",\"type\":\"text\"}],\"role\":\"user\"},{\"content\":[{\"text\":\"I'll help you calculate 15 + 27 using the add function.\",\"type\":\"text\"},{\"id\":\"toolu_015ewucD8KduCw15A5kPth7R\",\"input\":{\"a\":15,\"b\":27},\"name\":\"add\",\"type\":\"tool_use\"}],\"role\":\"assistant\"},{\"content\":[{\"tool_use_id\":\"toolu_015ewucD8KduCw15A5kPth7R\",\"content\":[{\"text\":\"42\",\"type\":\"text\"}],\"type\":\"tool_result\"}],\"role\":\"user\"}],\"model\":\"claude-sonnet-4-20250514\",\"system\":[{\"text\":\"You are a helpful assistant. Use the add tool to perform calculations.\",\"type\":\"text\"}],\"tool_choice\":{\"disable_parallel_tool_use\":false,\"type\":\"auto\"},\"tools\":[{\"input_schema\":{\"properties\":{\"a\":{\"description\":\"first number\",\"type\":\"integer\"},\"b\":{\"description\":\"second number\",\"type\":\"integer\"}},\"required\":[\"a\",\"b\"],\"type\":\"object\"},\"name\":\"add\",\"description\":\"Add two numbers\"}],\"stream\":true}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - Anthropic/Go 1.10.0 + url: https://api.anthropic.com/v1/messages + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_01DNqLDbjh3SijUzzFvKZJYX\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-20250514\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":519,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"15\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" + 27 = 42\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null},\"usage\":{\"input_tokens\":519,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":12} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n" + headers: + Content-Type: + - text/event-stream; charset=utf-8 + status: 200 OK + code: 200 + duration: 1.181155542s diff --git a/providertests/testdata/TestStreamWithTools/openai-gpt-4o.yaml b/providertests/testdata/TestStreamWithTools/openai-gpt-4o.yaml new file mode 100644 index 000000000..045f3e599 --- /dev/null +++ b/providertests/testdata/TestStreamWithTools/openai-gpt-4o.yaml @@ -0,0 +1,61 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 527 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant. Use the add tool to perform calculations.\",\"role\":\"system\"},{\"content\":\"What is 15 + 27?\",\"role\":\"user\"}],\"model\":\"gpt-4o\",\"stream_options\":{\"include_usage\":true},\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"add\",\"strict\":false,\"description\":\"Add two numbers\",\"parameters\":{\"properties\":{\"a\":{\"description\":\"first number\",\"type\":\"integer\"},\"b\":{\"description\":\"second number\",\"type\":\"integer\"}},\"required\":[\"a\",\"b\"],\"type\":\"object\"}},\"type\":\"function\"}],\"stream\":true}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "data: {\"id\":\"chatcmpl-CC7KWfg68JP1ntmxrXmMcjeGwt2yf\",\"object\":\"chat.completion.chunk\",\"created\":1757004148,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"index\":0,\"id\":\"call_32EKgZToXJGCcoOsKGeKDBMG\",\"type\":\"function\",\"function\":{\"name\":\"add\",\"arguments\":\"\"}}],\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"G\"}\n\ndata: {\"id\":\"chatcmpl-CC7KWfg68JP1ntmxrXmMcjeGwt2yf\",\"object\":\"chat.completion.chunk\",\"created\":1757004148,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"o83\"}\n\ndata: {\"id\":\"chatcmpl-CC7KWfg68JP1ntmxrXmMcjeGwt2yf\",\"object\":\"chat.completion.chunk\",\"created\":1757004148,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"a\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"Sqazb\"}\n\ndata: {\"id\":\"chatcmpl-CC7KWfg68JP1ntmxrXmMcjeGwt2yf\",\"object\":\"chat.completion.chunk\",\"created\":1757004148,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\":\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"ytC\"}\n\ndata: {\"id\":\"chatcmpl-CC7KWfg68JP1ntmxrXmMcjeGwt2yf\",\"object\":\"chat.completion.chunk\",\"created\":1757004148,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"15\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"Jt63\"}\n\ndata: {\"id\":\"chatcmpl-CC7KWfg68JP1ntmxrXmMcjeGwt2yf\",\"object\":\"chat.completion.chunk\",\"created\":1757004148,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\",\\\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"UEV\"}\n\ndata: {\"id\":\"chatcmpl-CC7KWfg68JP1ntmxrXmMcjeGwt2yf\",\"object\":\"chat.completion.chunk\",\"created\":1757004148,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"b\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"lGxRz\"}\n\ndata: {\"id\":\"chatcmpl-CC7KWfg68JP1ntmxrXmMcjeGwt2yf\",\"object\":\"chat.completion.chunk\",\"created\":1757004148,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\":\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"72d\"}\n\ndata: {\"id\":\"chatcmpl-CC7KWfg68JP1ntmxrXmMcjeGwt2yf\",\"object\":\"chat.completion.chunk\",\"created\":1757004148,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"27\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"D0VY\"}\n\ndata: {\"id\":\"chatcmpl-CC7KWfg68JP1ntmxrXmMcjeGwt2yf\",\"object\":\"chat.completion.chunk\",\"created\":1757004148,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"}\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"VyN2Q\"}\n\ndata: {\"id\":\"chatcmpl-CC7KWfg68JP1ntmxrXmMcjeGwt2yf\",\"object\":\"chat.completion.chunk\",\"created\":1757004148,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"tool_calls\"}],\"usage\":null,\"obfuscation\":\"lUEs\"}\n\ndata: {\"id\":\"chatcmpl-CC7KWfg68JP1ntmxrXmMcjeGwt2yf\",\"object\":\"chat.completion.chunk\",\"created\":1757004148,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[],\"usage\":{\"prompt_tokens\":73,\"completion_tokens\":17,\"total_tokens\":90,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"u6D1A99gV8BIvOR\"}\n\ndata: [DONE]\n\n" + headers: + Content-Type: + - text/event-stream; charset=utf-8 + status: 200 OK + code: 200 + duration: 1.275977s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 758 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant. Use the add tool to perform calculations.\",\"role\":\"system\"},{\"content\":\"What is 15 + 27?\",\"role\":\"user\"},{\"tool_calls\":[{\"id\":\"call_32EKgZToXJGCcoOsKGeKDBMG\",\"function\":{\"arguments\":\"{\\\"a\\\":15,\\\"b\\\":27}\",\"name\":\"add\"},\"type\":\"function\"}],\"role\":\"assistant\"},{\"content\":\"42\",\"tool_call_id\":\"call_32EKgZToXJGCcoOsKGeKDBMG\",\"role\":\"tool\"}],\"model\":\"gpt-4o\",\"stream_options\":{\"include_usage\":true},\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"add\",\"strict\":false,\"description\":\"Add two numbers\",\"parameters\":{\"properties\":{\"a\":{\"description\":\"first number\",\"type\":\"integer\"},\"b\":{\"description\":\"second number\",\"type\":\"integer\"}},\"required\":[\"a\",\"b\"],\"type\":\"object\"}},\"type\":\"function\"}],\"stream\":true}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "data: {\"id\":\"chatcmpl-CC7KYLaQy1yokH0JcpuncBILcIrSc\",\"object\":\"chat.completion.chunk\",\"created\":1757004150,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"lHQy6EEWHTpR9e\"}\n\ndata: {\"id\":\"chatcmpl-CC7KYLaQy1yokH0JcpuncBILcIrSc\",\"object\":\"chat.completion.chunk\",\"created\":1757004150,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"15\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"4l3v3yNWlz6ezS\"}\n\ndata: {\"id\":\"chatcmpl-CC7KYLaQy1yokH0JcpuncBILcIrSc\",\"object\":\"chat.completion.chunk\",\"created\":1757004150,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" +\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"jBTpHdjr9xDqYh\"}\n\ndata: {\"id\":\"chatcmpl-CC7KYLaQy1yokH0JcpuncBILcIrSc\",\"object\":\"chat.completion.chunk\",\"created\":1757004150,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"10oJvn7BeIA3iry\"}\n\ndata: {\"id\":\"chatcmpl-CC7KYLaQy1yokH0JcpuncBILcIrSc\",\"object\":\"chat.completion.chunk\",\"created\":1757004150,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"27\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"NFNSh8QNVa9Fl5\"}\n\ndata: {\"id\":\"chatcmpl-CC7KYLaQy1yokH0JcpuncBILcIrSc\",\"object\":\"chat.completion.chunk\",\"created\":1757004150,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" is\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"ayv5ohnA1tCWD\"}\n\ndata: {\"id\":\"chatcmpl-CC7KYLaQy1yokH0JcpuncBILcIrSc\",\"object\":\"chat.completion.chunk\",\"created\":1757004150,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"B81ro7bFywRKLgj\"}\n\ndata: {\"id\":\"chatcmpl-CC7KYLaQy1yokH0JcpuncBILcIrSc\",\"object\":\"chat.completion.chunk\",\"created\":1757004150,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"42\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"lDVhH4mGOWHn0L\"}\n\ndata: {\"id\":\"chatcmpl-CC7KYLaQy1yokH0JcpuncBILcIrSc\",\"object\":\"chat.completion.chunk\",\"created\":1757004150,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"CUsaGMVGQyyqVbK\"}\n\ndata: {\"id\":\"chatcmpl-CC7KYLaQy1yokH0JcpuncBILcIrSc\",\"object\":\"chat.completion.chunk\",\"created\":1757004150,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":null,\"obfuscation\":\"1lVNWlWnms\"}\n\ndata: {\"id\":\"chatcmpl-CC7KYLaQy1yokH0JcpuncBILcIrSc\",\"object\":\"chat.completion.chunk\",\"created\":1757004150,\"model\":\"gpt-4o-2024-08-06\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_f33640a400\",\"choices\":[],\"usage\":{\"prompt_tokens\":98,\"completion_tokens\":9,\"total_tokens\":107,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"xStpLtBU1J2gDbA\"}\n\ndata: [DONE]\n\n" + headers: + Content-Type: + - text/event-stream; charset=utf-8 + status: 200 OK + code: 200 + duration: 1.835219083s From 34d3f564d6dad02e715ce97fa5edb54538c91ba4 Mon Sep 17 00:00:00 2001 From: Andrey Nering Date: Thu, 4 Sep 2025 14:24:22 -0300 Subject: [PATCH 5/8] test: make test less intermittent --- providertests/provider_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/providertests/provider_test.go b/providertests/provider_test.go index fa30a55e7..8ffcfb247 100644 --- a/providertests/provider_test.go +++ b/providertests/provider_test.go @@ -31,10 +31,11 @@ func TestSimple(t *testing.T) { t.Fatalf("failed to generate: %v", err) } - want := "Olá" + option1 := "Oi" + option2 := "Olá" got := result.Response.Content.Text() - if !strings.Contains(got, want) { - t.Fatalf("unexpected response: got %q, want %q", got, want) + if !strings.Contains(got, option1) && !strings.Contains(got, option2) { + t.Fatalf("unexpected response: got %q, want %q or %q", got, option1, option2) } }) } From cc45b0fd0baef1f725c403b5c66e6b29d5bcf9d6 Mon Sep 17 00:00:00 2001 From: Andrey Nering Date: Thu, 4 Sep 2025 14:33:10 -0300 Subject: [PATCH 6/8] test: add openai gpt o4 mini to the test matrix --- providertests/builders_test.go | 13 ++++ .../TestSimple/openai-gpt-4o-mini.yaml | 33 ++++++++++ .../TestStream/openai-gpt-4o-mini.yaml | 32 ++++++++++ .../openai-gpt-4o-mini.yaml | 61 ++++++++++++++++++ .../testdata/TestTool/openai-gpt-4o-mini.yaml | 63 +++++++++++++++++++ 5 files changed, 202 insertions(+) create mode 100644 providertests/testdata/TestSimple/openai-gpt-4o-mini.yaml create mode 100644 providertests/testdata/TestStream/openai-gpt-4o-mini.yaml create mode 100644 providertests/testdata/TestStreamWithTools/openai-gpt-4o-mini.yaml create mode 100644 providertests/testdata/TestTool/openai-gpt-4o-mini.yaml diff --git a/providertests/builders_test.go b/providertests/builders_test.go index 3d5e080ce..72d122de3 100644 --- a/providertests/builders_test.go +++ b/providertests/builders_test.go @@ -19,6 +19,7 @@ type builderPair struct { var languageModelBuilders = []builderPair{ {"openai-gpt-4o", builderOpenaiGpt4o}, + {"openai-gpt-4o-mini", builderOpenaiGpt4oMini}, {"anthropic-claude-sonnet", builderAnthropicClaudeSonnet4}, } @@ -34,6 +35,18 @@ func builderOpenaiGpt4o(r *recorder.Recorder) (ai.LanguageModel, error) { return model, nil } +func builderOpenaiGpt4oMini(r *recorder.Recorder) (ai.LanguageModel, error) { + provider := openai.New( + openai.WithAPIKey(os.Getenv("OPENAI_API_KEY")), + openai.WithHTTPClient(&http.Client{Transport: r}), + ) + model, err := provider.LanguageModel("gpt-4o-mini") + if err != nil { + return nil, err + } + return model, nil +} + func builderAnthropicClaudeSonnet4(r *recorder.Recorder) (ai.LanguageModel, error) { provider := anthropic.New( anthropic.WithAPIKey(os.Getenv("ANTHROPIC_API_KEY")), diff --git a/providertests/testdata/TestSimple/openai-gpt-4o-mini.yaml b/providertests/testdata/TestSimple/openai-gpt-4o-mini.yaml new file mode 100644 index 000000000..662adf566 --- /dev/null +++ b/providertests/testdata/TestSimple/openai-gpt-4o-mini.yaml @@ -0,0 +1,33 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 143 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"Say hi in Portuguese\",\"role\":\"user\"}],\"model\":\"gpt-4o-mini\"}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"id\": \"chatcmpl-CC86Cz7oEjbaC9INACjv4rZgUqkui\",\n \"object\": \"chat.completion\",\n \"created\": 1757007104,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"Olá!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 20,\n \"completion_tokens\": 2,\n \"total_tokens\": 22,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_e665f7564b\"\n}\n" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 1.377235042s diff --git a/providertests/testdata/TestStream/openai-gpt-4o-mini.yaml b/providertests/testdata/TestStream/openai-gpt-4o-mini.yaml new file mode 100644 index 000000000..f033bf5dd --- /dev/null +++ b/providertests/testdata/TestStream/openai-gpt-4o-mini.yaml @@ -0,0 +1,32 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 205 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"Count from 1 to 3 in Spanish\",\"role\":\"user\"}],\"model\":\"gpt-4o-mini\",\"stream_options\":{\"include_usage\":true},\"stream\":true}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "data: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"MdM0iUdCi\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Sure\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"0WYvyGg\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"!\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"O4x0jEzy77\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" Here\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"hFhgoZ\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" is\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"21niu3eg\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" the\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"Q1U79RV\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" counting\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"n3\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" from\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"m8aW0r\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"SoH0gBsrYI\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"1\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"RDpov9RJ21\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" to\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"Pd02xohG\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"YEupeYkvgT\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"3\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"xPQZkPJmmD\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" in\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"x8LjLQvm\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" Spanish\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"HXj\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\":\\n\\n\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"6GrBPJ\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"1\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"dmyuaXYXjr\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"XXstNKfoS7\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" Uno\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"Z38XQzw\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\\n\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"L5vhFF9y7\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"2\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"Q2P9ThvZgh\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"WIIywHPwle\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" Dos\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"c3CsgGA\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\\n\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"CNPc8eacZ\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"3\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"eFqdAD3vb8\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"pc8kDYYOmv\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" Tres\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"NjLFvL\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":null,\"obfuscation\":\"BMBVR\"}\n\ndata: {\"id\":\"chatcmpl-CC86GJzAnNH4KeIucQvSjcw9ApvP1\",\"object\":\"chat.completion.chunk\",\"created\":1757007108,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_e665f7564b\",\"choices\":[],\"usage\":{\"prompt_tokens\":25,\"completion_tokens\":26,\"total_tokens\":51,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"WnRGyJ1PLM\"}\n\ndata: [DONE]\n\n" + headers: + Content-Type: + - text/event-stream; charset=utf-8 + status: 200 OK + code: 200 + duration: 1.171848125s diff --git a/providertests/testdata/TestStreamWithTools/openai-gpt-4o-mini.yaml b/providertests/testdata/TestStreamWithTools/openai-gpt-4o-mini.yaml new file mode 100644 index 000000000..3699cbc24 --- /dev/null +++ b/providertests/testdata/TestStreamWithTools/openai-gpt-4o-mini.yaml @@ -0,0 +1,61 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 532 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant. Use the add tool to perform calculations.\",\"role\":\"system\"},{\"content\":\"What is 15 + 27?\",\"role\":\"user\"}],\"model\":\"gpt-4o-mini\",\"stream_options\":{\"include_usage\":true},\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"add\",\"strict\":false,\"description\":\"Add two numbers\",\"parameters\":{\"properties\":{\"a\":{\"description\":\"first number\",\"type\":\"integer\"},\"b\":{\"description\":\"second number\",\"type\":\"integer\"}},\"required\":[\"a\",\"b\"],\"type\":\"object\"}},\"type\":\"function\"}],\"stream\":true}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "data: {\"id\":\"chatcmpl-CC86HU3sn6BghCHd9nGYvSrbG8Nq6\",\"object\":\"chat.completion.chunk\",\"created\":1757007109,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"index\":0,\"id\":\"call_SexxPUavmIXkdPj9YCY86rAv\",\"type\":\"function\",\"function\":{\"name\":\"add\",\"arguments\":\"\"}}],\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"Bi5lh4c7wm9V\"}\n\ndata: {\"id\":\"chatcmpl-CC86HU3sn6BghCHd9nGYvSrbG8Nq6\",\"object\":\"chat.completion.chunk\",\"created\":1757007109,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"9slbQBld7uDfGx\"}\n\ndata: {\"id\":\"chatcmpl-CC86HU3sn6BghCHd9nGYvSrbG8Nq6\",\"object\":\"chat.completion.chunk\",\"created\":1757007109,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"a\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"\"}\n\ndata: {\"id\":\"chatcmpl-CC86HU3sn6BghCHd9nGYvSrbG8Nq6\",\"object\":\"chat.completion.chunk\",\"created\":1757007109,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\":\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"In7gVcr4gFp8AQ\"}\n\ndata: {\"id\":\"chatcmpl-CC86HU3sn6BghCHd9nGYvSrbG8Nq6\",\"object\":\"chat.completion.chunk\",\"created\":1757007109,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"15\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"E8J66QfDmefl0se\"}\n\ndata: {\"id\":\"chatcmpl-CC86HU3sn6BghCHd9nGYvSrbG8Nq6\",\"object\":\"chat.completion.chunk\",\"created\":1757007109,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\",\\\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"OQRle4Kd1hRgyU\"}\n\ndata: {\"id\":\"chatcmpl-CC86HU3sn6BghCHd9nGYvSrbG8Nq6\",\"object\":\"chat.completion.chunk\",\"created\":1757007109,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"b\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"\"}\n\ndata: {\"id\":\"chatcmpl-CC86HU3sn6BghCHd9nGYvSrbG8Nq6\",\"object\":\"chat.completion.chunk\",\"created\":1757007109,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\":\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"8Nz9jINTs9JpkR\"}\n\ndata: {\"id\":\"chatcmpl-CC86HU3sn6BghCHd9nGYvSrbG8Nq6\",\"object\":\"chat.completion.chunk\",\"created\":1757007109,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"27\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"1Qv0gJNAeyn9KCl\"}\n\ndata: {\"id\":\"chatcmpl-CC86HU3sn6BghCHd9nGYvSrbG8Nq6\",\"object\":\"chat.completion.chunk\",\"created\":1757007109,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"}\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"\"}\n\ndata: {\"id\":\"chatcmpl-CC86HU3sn6BghCHd9nGYvSrbG8Nq6\",\"object\":\"chat.completion.chunk\",\"created\":1757007109,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"tool_calls\"}],\"usage\":null,\"obfuscation\":\"NViWjR1EBYC8EUs\"}\n\ndata: {\"id\":\"chatcmpl-CC86HU3sn6BghCHd9nGYvSrbG8Nq6\",\"object\":\"chat.completion.chunk\",\"created\":1757007109,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[],\"usage\":{\"prompt_tokens\":73,\"completion_tokens\":17,\"total_tokens\":90,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"Twq5TqPXwD\"}\n\ndata: [DONE]\n\n" + headers: + Content-Type: + - text/event-stream; charset=utf-8 + status: 200 OK + code: 200 + duration: 852.636542ms +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 763 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant. Use the add tool to perform calculations.\",\"role\":\"system\"},{\"content\":\"What is 15 + 27?\",\"role\":\"user\"},{\"tool_calls\":[{\"id\":\"call_SexxPUavmIXkdPj9YCY86rAv\",\"function\":{\"arguments\":\"{\\\"a\\\":15,\\\"b\\\":27}\",\"name\":\"add\"},\"type\":\"function\"}],\"role\":\"assistant\"},{\"content\":\"42\",\"tool_call_id\":\"call_SexxPUavmIXkdPj9YCY86rAv\",\"role\":\"tool\"}],\"model\":\"gpt-4o-mini\",\"stream_options\":{\"include_usage\":true},\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"add\",\"strict\":false,\"description\":\"Add two numbers\",\"parameters\":{\"properties\":{\"a\":{\"description\":\"first number\",\"type\":\"integer\"},\"b\":{\"description\":\"second number\",\"type\":\"integer\"}},\"required\":[\"a\",\"b\"],\"type\":\"object\"}},\"type\":\"function\"}],\"stream\":true}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + body: "data: {\"id\":\"chatcmpl-CC86I0sQZB3BScMjD1Vlq9r4LNG7K\",\"object\":\"chat.completion.chunk\",\"created\":1757007110,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"6dNsxDRYv\"}\n\ndata: {\"id\":\"chatcmpl-CC86I0sQZB3BScMjD1Vlq9r4LNG7K\",\"object\":\"chat.completion.chunk\",\"created\":1757007110,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"15\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"eeOggkItX\"}\n\ndata: {\"id\":\"chatcmpl-CC86I0sQZB3BScMjD1Vlq9r4LNG7K\",\"object\":\"chat.completion.chunk\",\"created\":1757007110,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" +\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"eTh8CvrQp\"}\n\ndata: {\"id\":\"chatcmpl-CC86I0sQZB3BScMjD1Vlq9r4LNG7K\",\"object\":\"chat.completion.chunk\",\"created\":1757007110,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"Y4Xvam5bns\"}\n\ndata: {\"id\":\"chatcmpl-CC86I0sQZB3BScMjD1Vlq9r4LNG7K\",\"object\":\"chat.completion.chunk\",\"created\":1757007110,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"27\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"taAs18INA\"}\n\ndata: {\"id\":\"chatcmpl-CC86I0sQZB3BScMjD1Vlq9r4LNG7K\",\"object\":\"chat.completion.chunk\",\"created\":1757007110,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" equals\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"QqWb\"}\n\ndata: {\"id\":\"chatcmpl-CC86I0sQZB3BScMjD1Vlq9r4LNG7K\",\"object\":\"chat.completion.chunk\",\"created\":1757007110,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"UKNosA2zT5\"}\n\ndata: {\"id\":\"chatcmpl-CC86I0sQZB3BScMjD1Vlq9r4LNG7K\",\"object\":\"chat.completion.chunk\",\"created\":1757007110,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"42\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"2eSH3S32n\"}\n\ndata: {\"id\":\"chatcmpl-CC86I0sQZB3BScMjD1Vlq9r4LNG7K\",\"object\":\"chat.completion.chunk\",\"created\":1757007110,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"qeXjwKLKsA\"}\n\ndata: {\"id\":\"chatcmpl-CC86I0sQZB3BScMjD1Vlq9r4LNG7K\",\"object\":\"chat.completion.chunk\",\"created\":1757007110,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":null,\"obfuscation\":\"grITR\"}\n\ndata: {\"id\":\"chatcmpl-CC86I0sQZB3BScMjD1Vlq9r4LNG7K\",\"object\":\"chat.completion.chunk\",\"created\":1757007110,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_8bda4d3a2c\",\"choices\":[],\"usage\":{\"prompt_tokens\":98,\"completion_tokens\":9,\"total_tokens\":107,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"8R73BwOCQi\"}\n\ndata: [DONE]\n\n" + headers: + Content-Type: + - text/event-stream; charset=utf-8 + status: 200 OK + code: 200 + duration: 709.664625ms diff --git a/providertests/testdata/TestTool/openai-gpt-4o-mini.yaml b/providertests/testdata/TestTool/openai-gpt-4o-mini.yaml new file mode 100644 index 000000000..0e0310b93 --- /dev/null +++ b/providertests/testdata/TestTool/openai-gpt-4o-mini.yaml @@ -0,0 +1,63 @@ +--- +version: 2 +interactions: +- id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 429 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence?\",\"role\":\"user\"}],\"model\":\"gpt-4o-mini\",\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"id\": \"chatcmpl-CC86DuEym05aNFxLlzEELyWrWahas\",\n \"object\": \"chat.completion\",\n \"created\": 1757007105,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_P1Q6WfBaX0ZdGvLJC4TacMGY\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"weather\",\n \"arguments\": \"{\\\"location\\\":\\\"Florence\\\"}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 59,\n \"completion_tokens\": 14,\n \"total_tokens\": 73,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_8bda4d3a2c\"\n}\n" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 1.524888209s +- id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 674 + host: "" + body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What's the weather in Florence?\",\"role\":\"user\"},{\"tool_calls\":[{\"id\":\"call_P1Q6WfBaX0ZdGvLJC4TacMGY\",\"function\":{\"arguments\":\"{\\\"location\\\":\\\"Florence\\\"}\",\"name\":\"weather\"},\"type\":\"function\"}],\"role\":\"assistant\"},{\"content\":\"40 C\",\"tool_call_id\":\"call_P1Q6WfBaX0ZdGvLJC4TacMGY\",\"role\":\"tool\"}],\"model\":\"gpt-4o-mini\",\"tool_choice\":\"auto\",\"tools\":[{\"function\":{\"name\":\"weather\",\"strict\":false,\"description\":\"Get weather information for a location\",\"parameters\":{\"properties\":{\"location\":{\"description\":\"the city\",\"type\":\"string\"}},\"required\":[\"location\"],\"type\":\"object\"}},\"type\":\"function\"}]}" + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAI/Go 2.3.0 + url: https://api.openai.com/v1/chat/completions + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + content_length: -1 + uncompressed: true + body: "{\n \"id\": \"chatcmpl-CC86FlWw0t13MgwmVeuzu5irpjyS2\",\n \"object\": \"chat.completion\",\n \"created\": 1757007107,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The weather in Florence is currently 40°C.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 82,\n \"completion_tokens\": 11,\n \"total_tokens\": 93,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_8bda4d3a2c\"\n}\n" + headers: + Content-Type: + - application/json + status: 200 OK + code: 200 + duration: 1.421106917s From d914fa9ce2b83fcb0124df651c93829235cb0630 Mon Sep 17 00:00:00 2001 From: Andrey Nering Date: Thu, 4 Sep 2025 14:33:31 -0300 Subject: [PATCH 7/8] chore(.gitattributes): do not show diffs for cassettes --- .gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 46d706f3c..62404f9d5 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1 @@ -providertests/testdata/**/*.yaml linguist-generated=true +providertests/testdata/**/*.yaml -diff linguist-generated=true From 6a8dbdf6f071781d5214272fcfcfe9365655fdd0 Mon Sep 17 00:00:00 2001 From: Andrey Nering Date: Thu, 4 Sep 2025 14:34:21 -0300 Subject: [PATCH 8/8] refactor: simplify code a little bit --- providertests/builders_test.go | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/providertests/builders_test.go b/providertests/builders_test.go index 72d122de3..ef5032918 100644 --- a/providertests/builders_test.go +++ b/providertests/builders_test.go @@ -28,11 +28,7 @@ func builderOpenaiGpt4o(r *recorder.Recorder) (ai.LanguageModel, error) { openai.WithAPIKey(os.Getenv("OPENAI_API_KEY")), openai.WithHTTPClient(&http.Client{Transport: r}), ) - model, err := provider.LanguageModel("gpt-4o") - if err != nil { - return nil, err - } - return model, nil + return provider.LanguageModel("gpt-4o") } func builderOpenaiGpt4oMini(r *recorder.Recorder) (ai.LanguageModel, error) { @@ -40,11 +36,7 @@ func builderOpenaiGpt4oMini(r *recorder.Recorder) (ai.LanguageModel, error) { openai.WithAPIKey(os.Getenv("OPENAI_API_KEY")), openai.WithHTTPClient(&http.Client{Transport: r}), ) - model, err := provider.LanguageModel("gpt-4o-mini") - if err != nil { - return nil, err - } - return model, nil + return provider.LanguageModel("gpt-4o-mini") } func builderAnthropicClaudeSonnet4(r *recorder.Recorder) (ai.LanguageModel, error) { @@ -52,9 +44,5 @@ func builderAnthropicClaudeSonnet4(r *recorder.Recorder) (ai.LanguageModel, erro anthropic.WithAPIKey(os.Getenv("ANTHROPIC_API_KEY")), anthropic.WithHTTPClient(&http.Client{Transport: r}), ) - model, err := provider.LanguageModel("claude-sonnet-4-20250514") - if err != nil { - return nil, err - } - return model, nil + return provider.LanguageModel("claude-sonnet-4-20250514") }