Skip to content

Commit 753bda0

Browse files
zijiren233zijiren233
authored
fix: gemini vertexai audio data need convert http to base64 (#608)
1 parent 678040a commit 753bda0

7 files changed

Lines changed: 222 additions & 9 deletions

File tree

‎core/.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@ core
33
core.exe
44
/public/dist/*
55
!*.gitkeep
6-
.env.local
6+
**/*.local

‎core/relay/adaptor/doubao/main_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -823,7 +823,7 @@ func TestAdaptorConvertRequestVideoGenerationMapsOpenAIFields(t *testing.T) {
823823
"n_seconds": 5,
824824
"input_reference": "https://example.com/reference.png",
825825
"video_url": "https://example.com/reference.mp4",
826-
"input_audio": {"data": "AAAA", "format": "wav"},
826+
"input_audio": {"data": "https://example.com/audio.wav", "format": "wav"},
827827
"generate_audio": true,
828828
"watermark": false
829829
}`),
@@ -867,7 +867,7 @@ func TestAdaptorConvertRequestVideoGenerationMapsOpenAIFields(t *testing.T) {
867867
assertDoubaoVideoContent(t, content[0], "text", "", "Animate a calm ocean")
868868
assertDoubaoVideoContent(t, content[1], "image_url", "https://example.com/reference.png", "")
869869
assertDoubaoVideoContent(t, content[2], "video_url", "https://example.com/reference.mp4", "")
870-
assertDoubaoVideoContent(t, content[3], "audio_url", "data:audio/wav;base64,AAAA", "")
870+
assertDoubaoVideoContent(t, content[3], "audio_url", "https://example.com/audio.wav", "")
871871

872872
usageContext := doubaoVideoRequestUsageContext(m)
873873
if usageContext.InputVideo == nil || !*usageContext.InputVideo ||

‎core/relay/adaptor/doubao/video.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,10 @@ func (audio *doubaoOpenAIInputAudio) DoubaoURL() *doubaoVideoURLContent {
126126
return &doubaoVideoURLContent{URL: data}
127127
}
128128

129+
if strings.HasPrefix(data, "http://") || strings.HasPrefix(data, "https://") {
130+
return &doubaoVideoURLContent{URL: data}
131+
}
132+
129133
format := strings.TrimSpace(strings.ToLower(audio.Format))
130134
if format == "" {
131135
format = "wav"

‎core/relay/adaptor/gemini/openai.go‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,22 @@ func autoImageURLToBase64Disabled(meta *meta.Meta, cfg Config) bool {
154154
return cfg.DisableAutoImageURLToBase64
155155
}
156156

157+
func autoAudioURLToBase64Disabled(meta *meta.Meta, cfg Config) bool {
158+
if meta != nil && meta.Channel.Type == model.ChannelTypeVertexAI {
159+
return false
160+
}
161+
162+
return cfg.DisableAutoAudioURLToBase64
163+
}
164+
165+
func autoVideoURLToBase64Disabled(meta *meta.Meta, cfg Config) bool {
166+
if meta != nil && meta.Channel.Type == model.ChannelTypeVertexAI {
167+
return false
168+
}
169+
170+
return cfg.DisableAutoVideoURLToBase64
171+
}
172+
157173
type CountTokensResponse struct {
158174
Error *relaymodel.GeminiError `json:"error,omitempty"`
159175
TotalTokens int `json:"totalTokens"`
@@ -445,6 +461,11 @@ func buildMessageParts(
445461
func buildGeminiMediaPart(data, uri, format, mediaType string) *relaymodel.GeminiPart {
446462
part := &relaymodel.GeminiPart{}
447463

464+
if uri == "" && isHTTPURL(data) {
465+
uri = data
466+
data = ""
467+
}
468+
448469
if data != "" {
449470
if mimeType, base64Data, ok := parseMediaDataURL(data, mediaType); ok {
450471
part.InlineData = &relaymodel.GeminiInlineData{
@@ -566,6 +587,10 @@ func firstNonEmpty(values ...string) string {
566587
return ""
567588
}
568589

590+
func isHTTPURL(rawURL string) bool {
591+
return strings.HasPrefix(rawURL, "http://") || strings.HasPrefix(rawURL, "https://")
592+
}
593+
569594
func parseToolCallArguments(arguments string) map[string]any {
570595
if arguments == "" {
571596
return make(map[string]any)
@@ -1045,12 +1070,14 @@ func convertRequest(
10451070
meta.Set("stream", textRequest.Stream)
10461071

10471072
disableAutoImageURLToBase64 := autoImageURLToBase64Disabled(meta, adaptorConfig)
1073+
disableAutoAudioURLToBase64 := autoAudioURLToBase64Disabled(meta, adaptorConfig)
1074+
disableAutoVideoURLToBase64 := autoVideoURLToBase64Disabled(meta, adaptorConfig)
10481075

10491076
systemContent, contents, imageTasks, audioTasks, videoTasks := buildContents(
10501077
textRequest,
10511078
!disableAutoImageURLToBase64,
1052-
!adaptorConfig.DisableAutoAudioURLToBase64,
1053-
!adaptorConfig.DisableAutoVideoURLToBase64,
1079+
!disableAutoAudioURLToBase64,
1080+
!disableAutoVideoURLToBase64,
10541081
)
10551082

10561083
// Process image tasks concurrently

‎core/relay/adaptor/gemini/openai_test.go‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1011,6 +1011,79 @@ func TestConvertRequestAutoConvertsAudioAndVideoURLs(t *testing.T) {
10111011
assert.Nil(t, geminiReq.Contents[0].Parts[1].FileData)
10121012
}
10131013

1014+
func TestConvertRequestAutoConvertsAudioHTTPDataURL(t *testing.T) {
1015+
t.Parallel()
1016+
1017+
audioData := []byte("audio bytes")
1018+
1019+
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1020+
assert.Equal(t, "/audio.wav", r.URL.Path)
1021+
w.Header().Set("Content-Type", "audio/wav")
1022+
_, _ = w.Write(audioData)
1023+
}))
1024+
defer ts.Close()
1025+
1026+
channel := &model.Channel{
1027+
Type: model.ChannelTypeGoogleGemini,
1028+
}
1029+
meta := meta.NewMeta(
1030+
channel,
1031+
mode.ChatCompletions,
1032+
"gemini-2.5-flash",
1033+
model.ModelConfig{},
1034+
)
1035+
1036+
openAIReq := map[string]any{
1037+
"model": "gemini-2.5-flash",
1038+
"messages": []map[string]any{
1039+
{
1040+
"role": "user",
1041+
"content": []map[string]any{
1042+
{
1043+
"type": "input_audio",
1044+
"input_audio": map[string]any{
1045+
"data": ts.URL + "/audio.wav",
1046+
"format": "wav",
1047+
},
1048+
},
1049+
},
1050+
},
1051+
},
1052+
}
1053+
1054+
jsonData, err := sonic.Marshal(openAIReq)
1055+
assert.NoError(t, err)
1056+
1057+
req, err := http.NewRequestWithContext(
1058+
t.Context(),
1059+
http.MethodPost,
1060+
"http://localhost/v1/chat/completions",
1061+
bytes.NewBuffer(jsonData),
1062+
)
1063+
assert.NoError(t, err)
1064+
1065+
result, err := gemini.ConvertRequest(meta, req)
1066+
assert.NoError(t, err)
1067+
1068+
bodyBytes, err := io.ReadAll(result.Body)
1069+
assert.NoError(t, err)
1070+
1071+
var geminiReq relaymodel.GeminiChatRequest
1072+
1073+
err = json.Unmarshal(bodyBytes, &geminiReq)
1074+
assert.NoError(t, err)
1075+
assert.Len(t, geminiReq.Contents, 1)
1076+
assert.Len(t, geminiReq.Contents[0].Parts, 1)
1077+
assert.NotNil(t, geminiReq.Contents[0].Parts[0].InlineData)
1078+
assert.Equal(t, "audio/wav", geminiReq.Contents[0].Parts[0].InlineData.MimeType)
1079+
assert.Equal(
1080+
t,
1081+
base64.StdEncoding.EncodeToString(audioData),
1082+
geminiReq.Contents[0].Parts[0].InlineData.Data,
1083+
)
1084+
assert.Nil(t, geminiReq.Contents[0].Parts[0].FileData)
1085+
}
1086+
10141087
func TestConvertRequestCanDisableAudioAndVideoURLAutoBase64(t *testing.T) {
10151088
t.Parallel()
10161089

@@ -1092,6 +1165,105 @@ func TestConvertRequestCanDisableAudioAndVideoURLAutoBase64(t *testing.T) {
10921165
)
10931166
}
10941167

1168+
func TestConvertRequestVertexAIAlwaysConvertsAudioAndVideoURLs(t *testing.T) {
1169+
t.Parallel()
1170+
1171+
audioData := []byte("audio bytes")
1172+
videoData := []byte("video bytes")
1173+
1174+
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1175+
switch r.URL.Path {
1176+
case "/audio.wav":
1177+
w.Header().Set("Content-Type", "audio/wav")
1178+
_, _ = w.Write(audioData)
1179+
case "/video.mp4":
1180+
w.Header().Set("Content-Type", "video/mp4")
1181+
_, _ = w.Write(videoData)
1182+
default:
1183+
http.NotFound(w, r)
1184+
}
1185+
}))
1186+
defer ts.Close()
1187+
1188+
channel := &model.Channel{
1189+
Type: model.ChannelTypeVertexAI,
1190+
Configs: model.ChannelConfigs{
1191+
"disable_auto_audio_url_to_base64": true,
1192+
"disable_auto_video_url_to_base64": true,
1193+
},
1194+
}
1195+
meta := meta.NewMeta(
1196+
channel,
1197+
mode.ChatCompletions,
1198+
"gemini-2.5-flash",
1199+
model.ModelConfig{},
1200+
)
1201+
1202+
openAIReq := map[string]any{
1203+
"model": "gemini-2.5-flash",
1204+
"messages": []map[string]any{
1205+
{
1206+
"role": "user",
1207+
"content": []map[string]any{
1208+
{
1209+
"type": "input_audio",
1210+
"input_audio": map[string]any{
1211+
"data": ts.URL + "/audio.wav",
1212+
"format": "wav",
1213+
},
1214+
},
1215+
{
1216+
"type": "video_url",
1217+
"video_url": map[string]any{
1218+
"url": ts.URL + "/video.mp4",
1219+
},
1220+
},
1221+
},
1222+
},
1223+
},
1224+
}
1225+
1226+
jsonData, err := sonic.Marshal(openAIReq)
1227+
assert.NoError(t, err)
1228+
1229+
req, err := http.NewRequestWithContext(
1230+
t.Context(),
1231+
http.MethodPost,
1232+
"http://localhost/v1/chat/completions",
1233+
bytes.NewBuffer(jsonData),
1234+
)
1235+
assert.NoError(t, err)
1236+
1237+
result, err := gemini.ConvertRequest(meta, req)
1238+
assert.NoError(t, err)
1239+
1240+
bodyBytes, err := io.ReadAll(result.Body)
1241+
assert.NoError(t, err)
1242+
1243+
var geminiReq relaymodel.GeminiChatRequest
1244+
1245+
err = json.Unmarshal(bodyBytes, &geminiReq)
1246+
assert.NoError(t, err)
1247+
assert.Len(t, geminiReq.Contents, 1)
1248+
assert.Len(t, geminiReq.Contents[0].Parts, 2)
1249+
assert.NotNil(t, geminiReq.Contents[0].Parts[0].InlineData)
1250+
assert.Equal(t, "audio/wav", geminiReq.Contents[0].Parts[0].InlineData.MimeType)
1251+
assert.Equal(
1252+
t,
1253+
base64.StdEncoding.EncodeToString(audioData),
1254+
geminiReq.Contents[0].Parts[0].InlineData.Data,
1255+
)
1256+
assert.Nil(t, geminiReq.Contents[0].Parts[0].FileData)
1257+
assert.NotNil(t, geminiReq.Contents[0].Parts[1].InlineData)
1258+
assert.Equal(t, "video/mp4", geminiReq.Contents[0].Parts[1].InlineData.MimeType)
1259+
assert.Equal(
1260+
t,
1261+
base64.StdEncoding.EncodeToString(videoData),
1262+
geminiReq.Contents[0].Parts[1].InlineData.Data,
1263+
)
1264+
assert.Nil(t, geminiReq.Contents[0].Parts[1].FileData)
1265+
}
1266+
10951267
func TestProcessMediaTasksKeepsFileDataWhenConversionFails(t *testing.T) {
10961268
t.Parallel()
10971269

‎core/relay/adaptor/siliconflow/chat.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ func openAIInputAudioURL(item *ast.Node) (string, bool, error) {
168168
return "", false, nil
169169
}
170170

171-
if strings.HasPrefix(data, "data:audio/") {
171+
if isOpenAIInputAudioURL(data) || strings.HasPrefix(data, "data:audio/") {
172172
return data, true, nil
173173
}
174174

@@ -208,6 +208,10 @@ func openAIInputAudioDataURL(inputAudio *relaymodel.InputAudio) string {
208208
return data
209209
}
210210

211+
if isOpenAIInputAudioURL(data) {
212+
return data
213+
}
214+
211215
format := strings.TrimPrefix(strings.TrimSpace(strings.ToLower(inputAudio.Format)), ".")
212216
if format == "" {
213217
format = "wav"
@@ -216,6 +220,10 @@ func openAIInputAudioDataURL(inputAudio *relaymodel.InputAudio) string {
216220
return "data:audio/" + format + ";base64," + data
217221
}
218222

223+
func isOpenAIInputAudioURL(value string) bool {
224+
return strings.HasPrefix(value, "http://") || strings.HasPrefix(value, "https://")
225+
}
226+
219227
func newSiliconFlowAudioURLContent(audioURL string) ast.Node {
220228
return ast.NewObject([]ast.Pair{
221229
ast.NewPair("type", ast.NewString("audio_url")),

‎core/relay/adaptor/siliconflow/chat_test.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ func TestConvertRequestChatPatchesInputAudioToAudioURL(t *testing.T) {
3030
{"type":"text","text":"Transcribe this audio."},
3131
{"type":"input_audio","input_audio":{"data":"QUJD","format":"wav"}},
3232
{"type":"input_audio","input_audio":{"url":"https://example.com/audio.mp3"}},
33+
{"type":"input_audio","input_audio":{"data":"https://example.com/audio.wav","format":"wav"}},
3334
{
3435
"type":"video_url",
3536
"video_url":{
@@ -66,14 +67,15 @@ func TestConvertRequestChatPatchesInputAudioToAudioURL(t *testing.T) {
6667
}
6768

6869
content, ok := message["content"].([]any)
69-
if !ok || len(content) != 4 {
70-
t.Fatalf("expected four content items, got %#v", message["content"])
70+
if !ok || len(content) != 5 {
71+
t.Fatalf("expected five content items, got %#v", message["content"])
7172
}
7273

7374
assertSiliconFlowTextContent(t, content[0], "Transcribe this audio.")
7475
assertSiliconFlowAudioURL(t, content[1], "data:audio/wav;base64,QUJD")
7576
assertSiliconFlowAudioURL(t, content[2], "https://example.com/audio.mp3")
76-
assertSiliconFlowVideoURL(t, content[3])
77+
assertSiliconFlowAudioURL(t, content[3], "https://example.com/audio.wav")
78+
assertSiliconFlowVideoURL(t, content[4])
7779

7880
streamOptions, ok := got["stream_options"].(map[string]any)
7981
if !ok || streamOptions["include_usage"] != true {

0 commit comments

Comments
 (0)