Skip to content

feat(ai-proxy/gemini): dynamically adjust custom upstream_url for Vertex AI streaming mode #14684

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
+12 −2
Open

feat(ai-proxy/gemini): dynamically adjust custom upstream_url for Vertex AI streaming mode #14684

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1869779
feat(ai-proxy/gemini): dynamically adjust custom upstream_url for Ver…
ldadu-QTL Jul 10, 2025
bb71c35
Merge branch 'master' into master
ledadu Jul 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions kong/llm/drivers/gemini.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ local function handle_stream_event(event_t, model_info, route_type)
}

return cjson.encode(new_event), nil, metadata

elseif is_tool_content(event) then
local metadata = {}
metadata.finish_reason = finish_reason
Expand Down Expand Up @@ -332,7 +332,7 @@ local function to_gemini_chat_openai(request_table, model_info, route_type)
if #v.content > 0 then -- check it has ome kind of array element
for j, part in ipairs(v.content) do
local this_part, err = openai_part_to_gemini_part(part)

if not this_part then
if not err then
err = "message at position " .. i .. ", part at position " .. j .. " does not match expected OpenAI format"
Expand Down Expand Up @@ -536,6 +536,11 @@ function _M.subrequest(body, conf, http_opts, return_res_table, identity_interfa
or "generateContent"
local f_url = conf.model.options and conf.model.options.upstream_url

-- Update the operation in the custom upstream_url to match the current mode (streaming or non-streaming)
if f_url then
f_url = f_url:gsub(":(streamGenerateContent)$", ":" .. operation)
:gsub(":(generateContent)$", ":" .. operation)

if not f_url then -- upstream_url override is not set
-- check if this is "public" or "vertex" gemini deployment
if conf.model.options
Expand Down Expand Up @@ -640,6 +645,11 @@ function _M.configure_request(conf, identity_interface)
or "generateContent"
local f_url = model.options and model.options.upstream_url

--
if f_url then
f_url = f_url:gsub(":(streamGenerateContent)$", ":" .. operation)
:gsub(":(generateContent)$", ":" .. operation)

if not f_url then -- upstream_url override is not set
-- check if this is "public" or "vertex" gemini deployment
if model.options
Expand Down