test: setup go-vcr to start recording real providers interactions

Author: andreynering

.gitattributes

@@ -0,0 +1 @@
+providertests/testdata/* linguist-generated=true

go.mod

@@ -7,12 +7,15 @@ require (
 	github.com/charmbracelet/x/json v0.2.0
 	github.com/go-viper/mapstructure/v2 v2.4.0
 	github.com/google/uuid v1.6.0
+	github.com/joho/godotenv v1.5.1
 	github.com/openai/openai-go/v2 v2.3.0
 	github.com/stretchr/testify v1.11.1
+	gopkg.in/dnaeon/go-vcr.v4 v4.0.5
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/goccy/go-yaml v1.18.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/tidwall/gjson v1.18.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect

go.sum

@@ -6,8 +6,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
 github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
+github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/openai/openai-go/v2 v2.3.0 h1:y9U+V1tlHjvvb/5XIswuySqnG5EnKBFAbMxgBvTHXvg=
 github.com/openai/openai-go/v2 v2.3.0/go.mod h1:sIUkR+Cu/PMUVkSKhkk742PRURkQOCFhiwJ7eRSBqmk=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -26,5 +30,7 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/dnaeon/go-vcr.v4 v4.0.5 h1:I0hpTIvD5rII+8LgYGrHMA2d4SQPoL6u7ZvJakWKsiA=
+gopkg.in/dnaeon/go-vcr.v4 v4.0.5/go.mod h1:dRos81TkW9C1WJt6tTaE+uV2Lo8qJT3AG2b35+CB/nQ=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

providertests/.env.sample

@@ -0,0 +1,2 @@
+ANTHROPIC_API_KEY=
+OPENAI_API_KEY=

providertests/builders_test.go

@@ -0,0 +1,47 @@
+package providertests
+
+import (
+	"net/http"
+	"os"
+
+	"github.com/charmbracelet/ai/ai"
+	"github.com/charmbracelet/ai/anthropic"
+	"github.com/charmbracelet/ai/openai"
+	"gopkg.in/dnaeon/go-vcr.v4/pkg/recorder"
+)
+
+type builderFunc func(r *recorder.Recorder) (ai.LanguageModel, error)
+
+type builderPair struct {
+	name    string
+	builder builderFunc
+}
+
+var languageModelBuilders = []builderPair{
+	{"openai-gpt-4o", builderOpenaiGpt4o},
+	{"anthropic-claude-sonnet", builderAnthropicClaudeSonnet4},
+}
+
+func builderOpenaiGpt4o(r *recorder.Recorder) (ai.LanguageModel, error) {
+	provider := openai.New(
+		openai.WithAPIKey(os.Getenv("OPENAI_API_KEY")),
+		openai.WithHTTPClient(&http.Client{Transport: r}),
+	)
+	model, err := provider.LanguageModel("gpt-4o")
+	if err != nil {
+		return nil, err
+	}
+	return model, nil
+}
+
+func builderAnthropicClaudeSonnet4(r *recorder.Recorder) (ai.LanguageModel, error) {
+	provider := anthropic.New(
+		anthropic.WithAPIKey(os.Getenv("ANTHROPIC_API_KEY")),
+		anthropic.WithHTTPClient(&http.Client{Transport: r}),
+	)
+	model, err := provider.LanguageModel("claude-sonnet-4-20250514")
+	if err != nil {
+		return nil, err
+	}
+	return model, nil
+}

providertests/provider_test.go

@@ -0,0 +1,39 @@
+package providertests
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/charmbracelet/ai/ai"
+	_ "github.com/joho/godotenv/autoload"
+)
+
+func TestSimple(t *testing.T) {
+	for _, pair := range languageModelBuilders {
+		t.Run(pair.name, func(t *testing.T) {
+			r := newRecorder(t)
+
+			languageModel, err := pair.builder(r)
+			if err != nil {
+				t.Fatalf("failed to build language model: %v", err)
+			}
+
+			agent := ai.NewAgent(
+				languageModel,
+				ai.WithSystemPrompt("You are a helpful assistant"),
+			)
+			result, err := agent.Generate(t.Context(), ai.AgentCall{
+				Prompt: "Say hi in Portuguese",
+			})
+			if err != nil {
+				t.Fatalf("failed to generate: %v", err)
+			}
+
+			want := "Olá"
+			got := result.Response.Content.Text()
+			if !strings.Contains(got, want) {
+				t.Fatalf("unexpected response: got %q, want %q", got, want)
+			}
+		})
+	}
+}

providertests/recorder_test.go

@@ -0,0 +1,74 @@
+package providertests
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"gopkg.in/dnaeon/go-vcr.v4/pkg/cassette"
+	"gopkg.in/dnaeon/go-vcr.v4/pkg/recorder"
+)
+
+func newRecorder(t *testing.T) *recorder.Recorder {
+	cassetteName := filepath.Join("testdata", t.Name())
+
+	r, err := recorder.New(
+		cassetteName,
+		recorder.WithMode(recorder.ModeRecordOnce),
+		recorder.WithMatcher(customMatcher(t)),
+		recorder.WithHook(hookRemoveHeaders, recorder.AfterCaptureHook),
+	)
+	if err != nil {
+		t.Fatalf("recorder: failed to create recorder: %v", err)
+	}
+
+	t.Cleanup(func() {
+		if err := r.Stop(); err != nil {
+			t.Errorf("recorder: failed to stop recorder: %v", err)
+		}
+	})
+
+	return r
+}
+
+func customMatcher(t *testing.T) recorder.MatcherFunc {
+	return func(r *http.Request, i cassette.Request) bool {
+		if r.Body == nil || r.Body == http.NoBody {
+			return cassette.DefaultMatcher(r, i)
+		}
+
+		var reqBody []byte
+		var err error
+		reqBody, err = io.ReadAll(r.Body)
+		if err != nil {
+			t.Fatalf("recorder: failed to read request body")
+		}
+		r.Body.Close()
+		r.Body = io.NopCloser(bytes.NewBuffer(reqBody))
+
+		return r.Method == i.Method && r.URL.String() == i.URL && string(reqBody) == i.Body
+	}
+}
+
+var headersToKeep = map[string]struct{}{
+	"accept":       {},
+	"content-type": {},
+	"user-agent":   {},
+}
+
+func hookRemoveHeaders(i *cassette.Interaction) error {
+	for k := range i.Request.Headers {
+		if _, ok := headersToKeep[strings.ToLower(k)]; !ok {
+			delete(i.Request.Headers, k)
+		}
+	}
+	for k := range i.Response.Headers {
+		if _, ok := headersToKeep[strings.ToLower(k)]; !ok {
+			delete(i.Response.Headers, k)
+		}
+	}
+	return nil
+}

providertests/testdata/TestSimple/anthropic-claude-sonnet.yaml

@@ -0,0 +1,33 @@
+---
+version: 2
+interactions:
+- id: 0
+  request:
+    proto: HTTP/1.1
+    proto_major: 1
+    proto_minor: 1
+    content_length: 205
+    host: ""
+    body: "{\"max_tokens\":4096,\"messages\":[{\"content\":[{\"text\":\"Say hi in Portuguese\",\"type\":\"text\"}],\"role\":\"user\"}],\"model\":\"claude-sonnet-4-20250514\",\"system\":[{\"text\":\"You are a helpful assistant\",\"type\":\"text\"}]}"
+    headers:
+      Accept:
+      - application/json
+      Content-Type:
+      - application/json
+      User-Agent:
+      - Anthropic/Go 1.10.0
+    url: https://api.anthropic.com/v1/messages
+    method: POST
+  response:
+    proto: HTTP/2.0
+    proto_major: 2
+    proto_minor: 0
+    content_length: -1
+    uncompressed: true
+    body: "{\"id\":\"msg_014AQFTJZeZ1KNGT5y9TSMSs\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-20250514\",\"content\":[{\"type\":\"text\",\"text\":\"Oi! or Olá!\\n\\nBoth are common ways to say \\\"hi\\\" in Portuguese. \\\"Oi\\\" is more casual and commonly used in Brazilian Portuguese, while \\\"Olá\\\" is a bit more formal and used in both Brazilian and European Portuguese.\"}],\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"usage\":{\"input_tokens\":16,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":60,\"service_tier\":\"standard\"}}"
+    headers:
+      Content-Type:
+      - application/json
+    status: 200 OK
+    code: 200
+    duration: 2.412166125s

providertests/testdata/TestSimple/openai-gpt-4o.yaml

@@ -0,0 +1,33 @@
+---
+version: 2
+interactions:
+- id: 0
+  request:
+    proto: HTTP/1.1
+    proto_major: 1
+    proto_minor: 1
+    content_length: 138
+    host: ""
+    body: "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"Say hi in Portuguese\",\"role\":\"user\"}],\"model\":\"gpt-4o\"}"
+    headers:
+      Accept:
+      - application/json
+      Content-Type:
+      - application/json
+      User-Agent:
+      - OpenAI/Go 2.3.0
+    url: https://api.openai.com/v1/chat/completions
+    method: POST
+  response:
+    proto: HTTP/2.0
+    proto_major: 2
+    proto_minor: 0
+    content_length: -1
+    uncompressed: true
+    body: "{\n  \"id\": \"chatcmpl-CBolfurp0H2jFXSwJHVGbLWOYHhbM\",\n  \"object\": \"chat.completion\",\n  \"created\": 1756932795,\n  \"model\": \"gpt-4o-2024-08-06\",\n  \"choices\": [\n    {\n      \"index\": 0,\n      \"message\": {\n        \"role\": \"assistant\",\n        \"content\": \"Olá! Como posso ajudar você hoje?\",\n        \"refusal\": null,\n        \"annotations\": []\n      },\n      \"logprobs\": null,\n      \"finish_reason\": \"stop\"\n    }\n  ],\n  \"usage\": {\n    \"prompt_tokens\": 20,\n    \"completion_tokens\": 8,\n    \"total_tokens\": 28,\n    \"prompt_tokens_details\": {\n      \"cached_tokens\": 0,\n      \"audio_tokens\": 0\n    },\n    \"completion_tokens_details\": {\n      \"reasoning_tokens\": 0,\n      \"audio_tokens\": 0,\n      \"accepted_prediction_tokens\": 0,\n      \"rejected_prediction_tokens\": 0\n    }\n  },\n  \"service_tier\": \"default\",\n  \"system_fingerprint\": \"fp_f33640a400\"\n}\n"
+    headers:
+      Content-Type:
+      - application/json
+    status: 200 OK
+    code: 200
+    duration: 3.363218s