Merge pull request #20 from WqyJh/feat-transcribe

WqyJh · web-flow · commit 83067fc75fd6 · 2025-06-15T18:12:12.000+08:00
feat: add tests &amp; examples for transcription
diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go-version: ["1.19", "1.20", "1.22", "1.23"]
+        go-version: ["1.20", "1.22", "1.23", "1.24"]
 
     steps:
       - uses: actions/checkout@v4
diff --git a/api.go b/api.go
@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 )
 
@@ -29,6 +30,63 @@ type CreateSessionResponse struct {
 	ClientSecret ClientSecret `json:"client_secret"`
 }
 
+// CreateTranscriptionSessionRequest is the request for creating a transcription session.
+type CreateTranscriptionSessionRequest struct {
+	// The set of items to include in the transcription.
+	Include []string `json:"include,omitempty"`
+	// The format of input audio. Options are "pcm16", "g711_ulaw", or "g711_alaw".
+	InputAudioFormat AudioFormat `json:"input_audio_format,omitempty"`
+	// Configuration for input audio noise reduction.
+	InputAudioNoiseReduction *InputAudioNoiseReduction `json:"input_audio_noise_reduction,omitempty"`
+	// Configuration for input audio transcription.
+	InputAudioTranscription *InputAudioTranscription `json:"input_audio_transcription,omitempty"`
+
+	// Attention: Keep this field empty! It's shocking that this field is documented but not supported.
+	// You may get error of "Unknown parameter: 'modalities'." if this field is not empty.
+	// Issue reported: https://community.openai.com/t/unknown-parameter-modalities-when-creating-transcriptionsessions/1150141/6
+	// Docs: https://platform.openai.com/docs/api-reference/realtime-sessions/create-transcription#realtime-sessions-create-transcription-modalities
+	// The set of modalities the model can respond with. To disable audio, set this to ["text"].
+	Modalities []Modality `json:"modalities,omitempty"`
+
+	// Configuration for turn detection.
+	TurnDetection *ClientTurnDetection `json:"turn_detection,omitempty"`
+}
+
+// CreateTranscriptionSessionResponse is the response from creating a transcription session.
+type CreateTranscriptionSessionResponse struct {
+	// The unique ID of the session.
+	ID string `json:"id"`
+	// The object type, must be "realtime.transcription_session".
+	Object string `json:"object"`
+	// The format of input audio.
+	InputAudioFormat AudioFormat `json:"input_audio_format,omitempty"`
+	// Configuration of the transcription model.
+	InputAudioTranscription *InputAudioTranscription `json:"input_audio_transcription,omitempty"`
+	// The set of modalities.
+	Modalities []Modality `json:"modalities,omitempty"`
+	// Configuration for turn detection.
+	TurnDetection *ServerTurnDetection `json:"turn_detection,omitempty"`
+	// Ephemeral key returned by the API.
+	ClientSecret ClientSecret `json:"client_secret"`
+}
+
+type OpenAIError struct {
+	StatusCode int    `json:"-"`
+	Message    string `json:"message"`
+	Type       string `json:"type"`
+	Param      string `json:"param"`
+	Code       any    `json:"code"`
+}
+
+type ErrorResponse struct { //nolint:errname // this is a http error response
+	StatusCode  int `json:"-"`
+	OpenAIError `json:"error"`
+}
+
+func (e *ErrorResponse) Error() string {
+	return e.OpenAIError.Message
+}
+
 type httpOption struct {
 	client  *http.Client
 	headers http.Header
@@ -83,12 +141,23 @@ func HTTPDo[Q any, R any](ctx context.Context, url string, req *Q, opts ...HTTPO
 	}
 	defer response.Body.Close()
 
+	data, err = io.ReadAll(response.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+
 	if response.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("http status code: %d", response.StatusCode)
+		var errResp ErrorResponse
+		err = json.Unmarshal(data, &errResp)
+		if err != nil {
+			return nil, fmt.Errorf("http status code: %d, error: %s", response.StatusCode, string(data))
+		}
+		errResp.StatusCode = response.StatusCode
+		return nil, &errResp
 	}
 
 	var resp R
-	err = json.NewDecoder(response.Body).Decode(&resp)
+	err = json.Unmarshal(data, &resp)
 	if err != nil {
 		return nil, fmt.Errorf("failed to decode response: %w", err)
 	}
diff --git a/api_integration_test.go b/api_integration_test.go
@@ -28,5 +28,51 @@ func TestCreateSession(t *testing.T) {
 	require.NoError(t, err)
 	require.NotEmpty(t, session.ClientSecret.Value)
 	require.NotZero(t, session.ClientSecret.ExpiresAt)
+	require.Equal(t, openairt.GPT4oRealtimePreview20241217, session.Model)
+	require.Equal(t, "You are a friendly assistant.", session.Instructions)
 	t.Logf("session: %+v", session)
 }
+
+func TestCreateTranscriptionSession(t *testing.T) {
+	key := os.Getenv("OPENAI_API_KEY")
+	if key == "" {
+		t.Skip("OPENAI_API_KEY is not set")
+	}
+	client := openairt.NewClient(key)
+	session, err := client.CreateTranscriptionSession(context.Background(), &openairt.CreateTranscriptionSessionRequest{
+		InputAudioFormat: openairt.AudioFormatPcm16,
+		InputAudioTranscription: &openairt.InputAudioTranscription{
+			Model:    openairt.GPT4oTranscribe,
+			Language: "en",
+		},
+		InputAudioNoiseReduction: &openairt.InputAudioNoiseReduction{
+			Type: openairt.NearFieldNoiseReduction,
+		},
+		// Attention: Keep this field empty! It's shocking that this field is documented but not supported.
+		// Modalities: []openairt.Modality{
+		// 	openairt.ModalityText,
+		// },
+		TurnDetection: &openairt.ClientTurnDetection{
+			Type: openairt.ClientTurnDetectionTypeServerVad,
+			TurnDetectionParams: openairt.TurnDetectionParams{
+				Threshold:         0.6,
+				PrefixPaddingMs:   300,
+				SilenceDurationMs: 500,
+			},
+		},
+		Include: []string{},
+	})
+	require.NoError(t, err)
+	require.NotEmpty(t, session.ClientSecret.Value)
+	require.NotZero(t, session.ClientSecret.ExpiresAt)
+	require.Equal(t, "realtime.transcription_session", session.Object)
+	require.Equal(t, openairt.AudioFormatPcm16, session.InputAudioFormat)
+	require.Equal(t, openairt.GPT4oTranscribe, session.InputAudioTranscription.Model)
+	require.Equal(t, "en", session.InputAudioTranscription.Language)
+	require.Equal(t, openairt.ServerTurnDetectionTypeServerVad, session.TurnDetection.Type)
+	require.InEpsilon(t, 0.6, session.TurnDetection.Threshold, 0.0001)
+	require.Equal(t, 300, session.TurnDetection.PrefixPaddingMs)
+	require.Equal(t, 500, session.TurnDetection.SilenceDurationMs)
+	require.Empty(t, session.Modalities)
+	t.Logf("transcription session: %+v", session)
+}
diff --git a/api_test.go b/api_test.go
@@ -95,3 +95,109 @@ func TestCreateSessionResponse(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, expected, actual)
 }
+
+func TestCreateTranscriptionSessionRequest(t *testing.T) {
+	data := `{
+    "input_audio_format": "pcm16",
+    "input_audio_transcription": {
+        "model": "gpt-4o-transcribe",
+        "language": "en",
+        "prompt": "This is a test transcription"
+    },
+    "input_audio_noise_reduction": {
+        "type": "near_field"
+    },
+    "modalities": ["text"],
+    "turn_detection": {
+        "type": "server_vad",
+        "threshold": 0.6,
+        "prefix_padding_ms": 300,
+        "silence_duration_ms": 500
+    },
+    "include": ["item.input_audio_transcription.logprobs"]
+}`
+	expected := openairt.CreateTranscriptionSessionRequest{
+		InputAudioFormat: openairt.AudioFormatPcm16,
+		InputAudioTranscription: &openairt.InputAudioTranscription{
+			Model:    "gpt-4o-transcribe",
+			Language: "en",
+			Prompt:   "This is a test transcription",
+		},
+		InputAudioNoiseReduction: &openairt.InputAudioNoiseReduction{
+			Type: "near_field",
+		},
+		Modalities: []openairt.Modality{
+			openairt.ModalityText,
+		},
+		TurnDetection: &openairt.ClientTurnDetection{
+			Type: openairt.ClientTurnDetectionTypeServerVad,
+			TurnDetectionParams: openairt.TurnDetectionParams{
+				Threshold:         0.6,
+				PrefixPaddingMs:   300,
+				SilenceDurationMs: 500,
+			},
+		},
+		Include: []string{"item.input_audio_transcription.logprobs"},
+	}
+
+	var actual openairt.CreateTranscriptionSessionRequest
+	err := json.Unmarshal([]byte(data), &actual)
+	require.NoError(t, err)
+	require.Equal(t, expected, actual)
+
+	actualBytes, err := json.Marshal(actual)
+	require.NoError(t, err)
+	jsontools.RequireJSONEq(t, data, string(actualBytes))
+}
+
+func TestCreateTranscriptionSessionResponse(t *testing.T) {
+	data := `{
+    "id": "trans_123456",
+    "object": "realtime.transcription_session",
+    "input_audio_format": "pcm16",
+    "input_audio_transcription": {
+        "model": "gpt-4o-transcribe",
+        "language": "en"
+    },
+    "modalities": ["text"],
+    "turn_detection": {
+        "type": "server_vad",
+        "threshold": 0.6,
+        "prefix_padding_ms": 300,
+        "silence_duration_ms": 500
+    },
+    "client_secret": {
+        "value": "ek_trans_abc123",
+        "expires_at": 1234567890
+    }
+}`
+	expected := openairt.CreateTranscriptionSessionResponse{
+		ID:               "trans_123456",
+		Object:           "realtime.transcription_session",
+		InputAudioFormat: openairt.AudioFormatPcm16,
+		InputAudioTranscription: &openairt.InputAudioTranscription{
+			Model:    "gpt-4o-transcribe",
+			Language: "en",
+		},
+		Modalities: []openairt.Modality{
+			openairt.ModalityText,
+		},
+		TurnDetection: &openairt.ServerTurnDetection{
+			Type: openairt.ServerTurnDetectionTypeServerVad,
+			TurnDetectionParams: openairt.TurnDetectionParams{
+				Threshold:         0.6,
+				PrefixPaddingMs:   300,
+				SilenceDurationMs: 500,
+			},
+		},
+		ClientSecret: openairt.ClientSecret{
+			Value:     "ek_trans_abc123",
+			ExpiresAt: 1234567890,
+		},
+	}
+
+	var actual openairt.CreateTranscriptionSessionResponse
+	err := json.Unmarshal([]byte(data), &actual)
+	require.NoError(t, err)
+	require.Equal(t, expected, actual)
+}
diff --git a/client.go b/client.go
@@ -2,7 +2,7 @@ package openairt
 
 import (
 	"context"
-	"fmt"
+	"errors"
 	"net/http"
 	"net/url"
 )
@@ -15,6 +15,13 @@ const (
 	GPT4oMiniRealtimePreview20241217 = "gpt-4o-mini-realtime-preview-2024-12-17"
 )
 
+// Transcription models.
+const (
+	GPT4oTranscribe     = "gpt-4o-transcribe"
+	GPT4oMiniTranscribe = "gpt-4o-mini-transcribe"
+	Whisper1            = "whisper-1"
+)
+
 // Client is OpenAI Realtime API client.
 type Client struct {
 	config ClientConfig
@@ -74,7 +81,7 @@ func WithModel(model string) ConnectOption {
 	}
 }
 
-// Set transcription intent instead of model
+// Set transcription intent instead of model.
 func WithIntent() ConnectOption {
 	return func(opts *connectOption) {
 		opts.intent = "transcription"
@@ -113,10 +120,10 @@ func (c *Client) Connect(ctx context.Context, opts ...ConnectOption) (*Conn, err
 
 	// get url by model
 	var url string
-	if connectOpts.intent == "" {
+	if connectOpts.intent == "" { //nolint:gocritic // if conditions would be determined in order
 		url = c.getURL(connectOpts.model)
 	} else if c.config.APIType != APITypeOpenAI {
-		return nil, fmt.Errorf("Azure API type with intent set not implemented");
+		return nil, errors.New("intent not supported for Azure API type")
 	} else {
 		url = c.config.BaseURL + "?" + "intent=" + connectOpts.intent
 	}
@@ -152,3 +159,15 @@ func (c *Client) CreateSession(ctx context.Context, req *CreateSessionRequest) (
 		WithHeaders(c.getAPIHeaders()),
 	)
 }
+
+// CreateTranscriptionSession creates a new transcription session.
+func (c *Client) CreateTranscriptionSession(ctx context.Context, req *CreateTranscriptionSessionRequest) (*CreateTranscriptionSessionResponse, error) {
+	return HTTPDo[CreateTranscriptionSessionRequest, CreateTranscriptionSessionResponse](
+		ctx,
+		c.config.APIBaseURL+"/realtime/transcription_sessions",
+		req,
+		WithClient(c.config.HTTPClient),
+		WithMethod(http.MethodPost),
+		WithHeaders(c.getAPIHeaders()),
+	)
+}
diff --git a/client_event.go b/client_event.go
@@ -52,6 +52,8 @@ type ClientSession struct {
 	Temperature *float32 `json:"temperature,omitempty"`
 	// Maximum number of output tokens for a single assistant response, inclusive of tool calls. Provide an integer between 1 and 4096 to limit output tokens, or "inf" for the maximum available tokens for a given model. Defaults to "inf".
 	MaxOutputTokens IntOrInf `json:"max_response_output_tokens,omitempty"`
+	// Configuration for input audio noise reduction. This can be set to null to turn off. Noise reduction filters audio added to the input audio buffer before it is sent to VAD and the model. Filtering the audio can improve VAD and turn detection accuracy (reducing false positives) and model performance by improving perception of the input audio.
+	InputAudioNoiseReduction *InputAudioNoiseReduction `json:"input_audio_noise_reduction,omitempty"`
 }
 
 // SessionUpdateEvent is the event for session update.
diff --git a/examples/go.mod b/examples/go.mod
@@ -19,4 +19,4 @@ require (
 	golang.org/x/sys v0.25.0 // indirect
 )
 
-// replace github.com/WqyJh/go-openai-realtime => ../
+replace github.com/WqyJh/go-openai-realtime => ../
diff --git a/examples/transcription/main.go b/examples/transcription/main.go
diff --git a/server_event.go b/server_event.go
diff --git a/types.go b/types.go

Original file line number	Diff line number	Diff line change
`@@ -52,6 +52,8 @@ type ClientSession struct {`
`52`	`52`	Temperature *float32 `json:"temperature,omitempty"`
`53`	`53`	`// Maximum number of output tokens for a single assistant response, inclusive of tool calls. Provide an integer between 1 and 4096 to limit output tokens, or "inf" for the maximum available tokens for a given model. Defaults to "inf".`
`54`	`54`	MaxOutputTokens IntOrInf `json:"max_response_output_tokens,omitempty"`
	`55`	`+ // Configuration for input audio noise reduction. This can be set to null to turn off. Noise reduction filters audio added to the input audio buffer before it is sent to VAD and the model. Filtering the audio can improve VAD and turn detection accuracy (reducing false positives) and model performance by improving perception of the input audio.`
	`56`	+ InputAudioNoiseReduction *InputAudioNoiseReduction `json:"input_audio_noise_reduction,omitempty"`
`55`	`57`	`}`
`56`	`58`
`57`	`59`	`// SessionUpdateEvent is the event for session update.`
Original file line number	Diff line number	Diff line change
`@@ -19,4 +19,4 @@ require (`
`19`	`19`	`golang.org/x/sys v0.25.0 // indirect`
`20`	`20`	`)`
`21`	`21`
`22`		`-// replace github.com/WqyJh/go-openai-realtime => ../`
	`22`	`+replace github.com/WqyJh/go-openai-realtime => ../`