Merge pull request #37428 from vespa-engine/add-reasoning-effort-option

Support reasoning_effort in the OpenAI client

Author: abhishekkrthakur

model-integration/abi-spec.json

@@ -36,6 +36,7 @@
       "public ai.vespa.llm.clients.LlmClientConfig$Builder model(java.lang.String)",
       "public ai.vespa.llm.clients.LlmClientConfig$Builder temperature(double)",
       "public ai.vespa.llm.clients.LlmClientConfig$Builder maxTokens(int)",
+      "public ai.vespa.llm.clients.LlmClientConfig$Builder reasoningEffort(java.lang.String)",
       "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)",
       "public final java.lang.String getDefMd5()",
       "public final java.lang.String getDefName()",
@@ -78,7 +79,8 @@
       "public java.lang.String endpoint()",
       "public java.lang.String model()",
       "public double temperature()",
-      "public int maxTokens()"
+      "public int maxTokens()",
+      "public java.lang.String reasoningEffort()"
     ],
     "fields" : [
       "public static final java.lang.String CONFIG_DEF_MD5",

model-integration/src/main/java/ai/vespa/llm/clients/OpenAI.java

@@ -18,6 +18,7 @@
 import com.openai.client.OpenAIClientAsync;
 import com.openai.core.JsonValue;
 import com.openai.models.ResponseFormatJsonSchema;
+import com.openai.models.ReasoningEffort;
 import com.fasterxml.jackson.core.type.TypeReference;
 
 import java.util.HashMap;
@@ -70,9 +71,14 @@ public OpenAI(LlmClientConfig config, Secrets secretStore) {
         if (config.maxTokens() >= 0) {
             configOptions.put(InferenceParameters.OPTION_MAX_TOKENS, String.valueOf(config.maxTokens()));
         }
+
+        if (!config.reasoningEffort().isBlank()) {
+            configOptions.put(InferenceParameters.OPTION_REASONING_EFFORT, config.reasoningEffort());
+        }
     }
 
-    private InferenceParameters prepareParameters(InferenceParameters parameters) {
+    // Package-private for testing
+    InferenceParameters prepareParameters(InferenceParameters parameters) {
         setApiKey(parameters);
         setEndpoint(parameters);
         return parameters.withDefaultOptions(configOptions::get);
@@ -183,7 +189,8 @@ public CompletableFuture<Completion.FinishReason> completeAsync(
         return future;
     }
 
-    private ChatCompletionCreateParams getChatCompletionCreateParams(InferenceParameters parameters, Prompt prompt) {
+    // Package-private for testing
+    ChatCompletionCreateParams getChatCompletionCreateParams(InferenceParameters parameters, Prompt prompt) {
         ChatCompletionCreateParams.Builder builder = ChatCompletionCreateParams.builder()
             .model(ChatModel.of(parameters.get(InferenceParameters.OPTION_MODEL).map(Object::toString).orElse(DEFAULT_MODEL)))
             .addUserMessage(prompt.toString());
@@ -194,6 +201,8 @@ private ChatCompletionCreateParams getChatCompletionCreateParams(InferenceParame
         parameters.getInt(InferenceParameters.OPTION_N_PREDICT).ifPresent(builder::n);
         parameters.getDouble(InferenceParameters.OPTION_FREQUENCY_PENALTY).ifPresent(builder::frequencyPenalty);
         parameters.getDouble(InferenceParameters.OPTION_PRESENCE_PENALTY).ifPresent(builder::presencePenalty);
+        parameters.get(InferenceParameters.OPTION_REASONING_EFFORT)
+                .ifPresent(effort -> builder.reasoningEffort(ReasoningEffort.of(effort)));
         // Add JSON schema if specified
         addResponseFormat(parameters, builder);
 

model-integration/src/main/resources/configdefinitions/llm-client.def

@@ -20,4 +20,9 @@ temperature double default=-1
 
 # The maximum number of tokens that can be generated in the completion.
 # -1 means use default for the API.
-maxTokens int default=-1
+maxTokens int default=-1
+# Constrains effort on reasoning for reasoning models, e.g. "low", "medium", "high".
+# Accepted values depend on the provider (OpenAI also accepts "minimal"; Google's
+# OpenAI-compatible API accepts "none"). Passed through to the API as-is.
+# Empty means use default for the API.
+reasoningEffort string default=""

model-integration/src/test/java/ai/vespa/llm/clients/OpenAITest.java

@@ -11,6 +11,7 @@
 import org.junit.jupiter.api.Test;
 import com.openai.errors.UnauthorizedException;
 import com.openai.errors.OpenAIIoException;
+import com.openai.models.ReasoningEffort;
 
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -69,6 +70,32 @@ public void testCompleteAsync() {
         assertNumTokens(text.toString(), 3, 10);
     }
 
+    @Test
+    public void testReasoningEffortOption() {
+        var prompt = StringPrompt.from("hello");
+
+        // Not set: not sent to the API
+        var openai = new OpenAI(new LlmClientConfig.Builder().apiKeySecretName("openai").build(), new MockSecrets());
+        var params = openai.getChatCompletionCreateParams(new InferenceParameters(Map.<String, String>of()::get), prompt);
+        assertTrue(params.reasoningEffort().isEmpty());
+
+        // Set in component config
+        var openaiWithConfig = new OpenAI(
+                new LlmClientConfig.Builder().apiKeySecretName("openai").reasoningEffort("low").build(),
+                new MockSecrets());
+        params = openaiWithConfig.getChatCompletionCreateParams(
+                openaiWithConfig.prepareParameters(new InferenceParameters(Map.<String, String>of()::get)), prompt);
+        assertEquals(ReasoningEffort.LOW, params.reasoningEffort().orElseThrow());
+
+        // Per-request option overrides config; non-enum values pass through
+        // as-is (providers accept different sets, e.g. "minimal" and "none").
+        params = openaiWithConfig.getChatCompletionCreateParams(
+                openaiWithConfig.prepareParameters(
+                        new InferenceParameters(Map.of(InferenceParameters.OPTION_REASONING_EFFORT, "none")::get)),
+                prompt);
+        assertEquals(ReasoningEffort.of("none"), params.reasoningEffort().orElseThrow());
+    }
+
     @Test
     public void testClientCaching() {
         // Create OpenAI instance

vespajlib/abi-spec.json

@@ -4324,7 +4324,8 @@
       "public static final java.lang.String OPTION_FREQUENCY_PENALTY",
       "public static final java.lang.String OPTION_PRESENCE_PENALTY",
       "public static final java.lang.String OPTION_SEED",
-      "public static final java.lang.String OPTION_JSON_SCHEMA"
+      "public static final java.lang.String OPTION_JSON_SCHEMA",
+      "public static final java.lang.String OPTION_REASONING_EFFORT"
     ]
   },
   "ai.vespa.llm.LanguageModel" : {

vespajlib/src/main/java/ai/vespa/llm/InferenceParameters.java

@@ -31,6 +31,7 @@ public class InferenceParameters {
     public static final String OPTION_PRESENCE_PENALTY = "presencepenalty";
     public static final String OPTION_SEED = "seed";
     public static final String OPTION_JSON_SCHEMA = "json_schema";
+    public static final String OPTION_REASONING_EFFORT = "reasoningEffort";
 
     private String apiKey;
     private String endpoint;