From 986a3a345f225b64b5463fb36ff5b766651d7ed6 Mon Sep 17 00:00:00 2001 From: abhishekkrthakur <1183441+abhishekkrthakur@users.noreply.github.com> Date: Mon, 20 Jul 2026 15:47:40 +0200 Subject: [PATCH 1/2] Support reasoning_effort in the OpenAI client Reasoning models count thinking tokens against max_completion_tokens and some default to unbounded dynamic thinking, so without this option there is no way to bound latency or keep thinking from consuming the completion budget through OpenAI-compatible endpoints (observed with Gemini via Vertex AI: 5-9s per call, or truncated JSON when capped by maxTokens). Adds InferenceParameters.OPTION_REASONING_EFFORT and a reasoningEffort config on llm-client, mapped to the chat completions reasoning_effort field. Values pass through as-is since providers accept different sets (OpenAI: minimal/low/medium/high; Google's compat API also takes none). Co-Authored-By: Claude Fable 5 --- .../java/ai/vespa/llm/clients/OpenAI.java | 13 +++++++-- .../configdefinitions/llm-client.def | 7 ++++- .../java/ai/vespa/llm/clients/OpenAITest.java | 27 +++++++++++++++++++ vespajlib/abi-spec.json | 3 ++- .../ai/vespa/llm/InferenceParameters.java | 1 + 5 files changed, 47 insertions(+), 4 deletions(-) diff --git a/model-integration/src/main/java/ai/vespa/llm/clients/OpenAI.java b/model-integration/src/main/java/ai/vespa/llm/clients/OpenAI.java index 6b7bdca6c3a1..5711ac9b761d 100644 --- a/model-integration/src/main/java/ai/vespa/llm/clients/OpenAI.java +++ b/model-integration/src/main/java/ai/vespa/llm/clients/OpenAI.java @@ -18,6 +18,7 @@ import com.openai.client.OpenAIClientAsync; import com.openai.core.JsonValue; import com.openai.models.ResponseFormatJsonSchema; +import com.openai.models.ReasoningEffort; import com.fasterxml.jackson.core.type.TypeReference; import java.util.HashMap; @@ -70,9 +71,14 @@ public OpenAI(LlmClientConfig config, Secrets secretStore) { if (config.maxTokens() >= 0) { configOptions.put(InferenceParameters.OPTION_MAX_TOKENS, String.valueOf(config.maxTokens())); } + + if (!config.reasoningEffort().isBlank()) { + configOptions.put(InferenceParameters.OPTION_REASONING_EFFORT, config.reasoningEffort()); + } } - private InferenceParameters prepareParameters(InferenceParameters parameters) { + // Package-private for testing + InferenceParameters prepareParameters(InferenceParameters parameters) { setApiKey(parameters); setEndpoint(parameters); return parameters.withDefaultOptions(configOptions::get); @@ -183,7 +189,8 @@ public CompletableFuture completeAsync( return future; } - private ChatCompletionCreateParams getChatCompletionCreateParams(InferenceParameters parameters, Prompt prompt) { + // Package-private for testing + ChatCompletionCreateParams getChatCompletionCreateParams(InferenceParameters parameters, Prompt prompt) { ChatCompletionCreateParams.Builder builder = ChatCompletionCreateParams.builder() .model(ChatModel.of(parameters.get(InferenceParameters.OPTION_MODEL).map(Object::toString).orElse(DEFAULT_MODEL))) .addUserMessage(prompt.toString()); @@ -194,6 +201,8 @@ private ChatCompletionCreateParams getChatCompletionCreateParams(InferenceParame parameters.getInt(InferenceParameters.OPTION_N_PREDICT).ifPresent(builder::n); parameters.getDouble(InferenceParameters.OPTION_FREQUENCY_PENALTY).ifPresent(builder::frequencyPenalty); parameters.getDouble(InferenceParameters.OPTION_PRESENCE_PENALTY).ifPresent(builder::presencePenalty); + parameters.get(InferenceParameters.OPTION_REASONING_EFFORT) + .ifPresent(effort -> builder.reasoningEffort(ReasoningEffort.of(effort))); // Add JSON schema if specified addResponseFormat(parameters, builder); diff --git a/model-integration/src/main/resources/configdefinitions/llm-client.def b/model-integration/src/main/resources/configdefinitions/llm-client.def index 69c24a687dc0..31423ca5d21f 100755 --- a/model-integration/src/main/resources/configdefinitions/llm-client.def +++ b/model-integration/src/main/resources/configdefinitions/llm-client.def @@ -20,4 +20,9 @@ temperature double default=-1 # The maximum number of tokens that can be generated in the completion. # -1 means use default for the API. -maxTokens int default=-1 \ No newline at end of file +maxTokens int default=-1 +# Constrains effort on reasoning for reasoning models, e.g. "low", "medium", "high". +# Accepted values depend on the provider (OpenAI also accepts "minimal"; Google's +# OpenAI-compatible API accepts "none"). Passed through to the API as-is. +# Empty means use default for the API. +reasoningEffort string default="" diff --git a/model-integration/src/test/java/ai/vespa/llm/clients/OpenAITest.java b/model-integration/src/test/java/ai/vespa/llm/clients/OpenAITest.java index fc4b25b16313..2d6367ee196d 100644 --- a/model-integration/src/test/java/ai/vespa/llm/clients/OpenAITest.java +++ b/model-integration/src/test/java/ai/vespa/llm/clients/OpenAITest.java @@ -11,6 +11,7 @@ import org.junit.jupiter.api.Test; import com.openai.errors.UnauthorizedException; import com.openai.errors.OpenAIIoException; +import com.openai.models.ReasoningEffort; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -69,6 +70,32 @@ public void testCompleteAsync() { assertNumTokens(text.toString(), 3, 10); } + @Test + public void testReasoningEffortOption() { + var prompt = StringPrompt.from("hello"); + + // Not set: not sent to the API + var openai = new OpenAI(new LlmClientConfig.Builder().apiKeySecretName("openai").build(), new MockSecrets()); + var params = openai.getChatCompletionCreateParams(new InferenceParameters(Map.of()::get), prompt); + assertTrue(params.reasoningEffort().isEmpty()); + + // Set in component config + var openaiWithConfig = new OpenAI( + new LlmClientConfig.Builder().apiKeySecretName("openai").reasoningEffort("low").build(), + new MockSecrets()); + params = openaiWithConfig.getChatCompletionCreateParams( + openaiWithConfig.prepareParameters(new InferenceParameters(Map.of()::get)), prompt); + assertEquals(ReasoningEffort.LOW, params.reasoningEffort().orElseThrow()); + + // Per-request option overrides config; non-enum values pass through + // as-is (providers accept different sets, e.g. "minimal" and "none"). + params = openaiWithConfig.getChatCompletionCreateParams( + openaiWithConfig.prepareParameters( + new InferenceParameters(Map.of(InferenceParameters.OPTION_REASONING_EFFORT, "none")::get)), + prompt); + assertEquals(ReasoningEffort.of("none"), params.reasoningEffort().orElseThrow()); + } + @Test public void testClientCaching() { // Create OpenAI instance diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index ba680513cfce..a692a6928d0d 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -4324,7 +4324,8 @@ "public static final java.lang.String OPTION_FREQUENCY_PENALTY", "public static final java.lang.String OPTION_PRESENCE_PENALTY", "public static final java.lang.String OPTION_SEED", - "public static final java.lang.String OPTION_JSON_SCHEMA" + "public static final java.lang.String OPTION_JSON_SCHEMA", + "public static final java.lang.String OPTION_REASONING_EFFORT" ] }, "ai.vespa.llm.LanguageModel" : { diff --git a/vespajlib/src/main/java/ai/vespa/llm/InferenceParameters.java b/vespajlib/src/main/java/ai/vespa/llm/InferenceParameters.java index 1dc10df9e5f2..fc490f6155f3 100755 --- a/vespajlib/src/main/java/ai/vespa/llm/InferenceParameters.java +++ b/vespajlib/src/main/java/ai/vespa/llm/InferenceParameters.java @@ -31,6 +31,7 @@ public class InferenceParameters { public static final String OPTION_PRESENCE_PENALTY = "presencepenalty"; public static final String OPTION_SEED = "seed"; public static final String OPTION_JSON_SCHEMA = "json_schema"; + public static final String OPTION_REASONING_EFFORT = "reasoningEffort"; private String apiKey; private String endpoint; From 91b699c5a856ea6391203a1e2ab9065646cf2f08 Mon Sep 17 00:00:00 2001 From: abhishekkrthakur <1183441+abhishekkrthakur@users.noreply.github.com> Date: Mon, 20 Jul 2026 15:56:52 +0200 Subject: [PATCH 2/2] Update model-integration abi spec for generated LlmClientConfig accessors Co-Authored-By: Claude Fable 5 --- model-integration/abi-spec.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/model-integration/abi-spec.json b/model-integration/abi-spec.json index ae3ca946f378..1f2b285b214e 100644 --- a/model-integration/abi-spec.json +++ b/model-integration/abi-spec.json @@ -36,6 +36,7 @@ "public ai.vespa.llm.clients.LlmClientConfig$Builder model(java.lang.String)", "public ai.vespa.llm.clients.LlmClientConfig$Builder temperature(double)", "public ai.vespa.llm.clients.LlmClientConfig$Builder maxTokens(int)", + "public ai.vespa.llm.clients.LlmClientConfig$Builder reasoningEffort(java.lang.String)", "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", "public final java.lang.String getDefMd5()", "public final java.lang.String getDefName()", @@ -78,7 +79,8 @@ "public java.lang.String endpoint()", "public java.lang.String model()", "public double temperature()", - "public int maxTokens()" + "public int maxTokens()", + "public java.lang.String reasoningEffort()" ], "fields" : [ "public static final java.lang.String CONFIG_DEF_MD5",