SUT Options defined at test level (#897)

* class method sut_options

* trim test wrapper

* put back test

* sut_options is an instance method

* First try: SUTOptions is separate from TestItem

* finish refactor

* Remove another sut_options from testitem

* Fix plugins

Author: bkorycki

demo_plugin/modelgauge/suts/demo_01_yes_no_sut.py

@@ -1,6 +1,6 @@
 from modelgauge.prompt import ChatPrompt, TextPrompt
 from modelgauge.prompt_formatting import format_chat
-from modelgauge.sut import PromptResponseSUT, SUTResponse
+from modelgauge.sut import PromptResponseSUT, SUTOptions, SUTResponse
 from modelgauge.sut_capabilities import AcceptsChatPrompt, AcceptsTextPrompt
 from modelgauge.sut_decorator import modelgauge_sut
 from modelgauge.sut_registry import SUTS
@@ -24,10 +24,10 @@ class DemoYesNoResponse(BaseModel):
 class DemoYesNoSUT(PromptResponseSUT[DemoYesNoRequest, DemoYesNoResponse]):
     """This SUT demonstrates the bare minimum behavior of a SUT: Use the input Prompt to determine the response."""
 
-    def translate_text_prompt(self, prompt: TextPrompt) -> DemoYesNoRequest:
+    def translate_text_prompt(self, prompt: TextPrompt, options: SUTOptions) -> DemoYesNoRequest:
         return DemoYesNoRequest(text=prompt.text)
 
-    def translate_chat_prompt(self, prompt: ChatPrompt) -> DemoYesNoRequest:
+    def translate_chat_prompt(self, prompt: ChatPrompt, options: SUTOptions) -> DemoYesNoRequest:
         return DemoYesNoRequest(text=format_chat(prompt))
 
     def evaluate(self, request: DemoYesNoRequest) -> DemoYesNoResponse:

demo_plugin/modelgauge/suts/demo_02_secrets_and_options_sut.py

@@ -1,7 +1,7 @@
 import random
-from modelgauge.prompt import ChatPrompt, SUTOptions, TextPrompt
+from modelgauge.prompt import ChatPrompt, TextPrompt
 from modelgauge.secret_values import InjectSecret, RequiredSecret, SecretDescription
-from modelgauge.sut import PromptResponseSUT, SUTResponse
+from modelgauge.sut import PromptResponseSUT, SUTOptions, SUTResponse
 from modelgauge.sut_capabilities import AcceptsChatPrompt, AcceptsTextPrompt
 from modelgauge.sut_decorator import modelgauge_sut
 from modelgauge.sut_registry import SUTS
@@ -46,12 +46,12 @@ def __init__(self, uid: str, api_key: DemoApiKey):
     def _load_client(self) -> "RandomWordsClient":
         return RandomWordsClient(api_key=self.api_key)
 
-    def translate_text_prompt(self, prompt: TextPrompt) -> DemoRandomWordsRequest:
-        return self._translate(prompt.text, prompt.options)
+    def translate_text_prompt(self, prompt: TextPrompt, options: SUTOptions) -> DemoRandomWordsRequest:
+        return self._translate(prompt.text, options)
 
-    def translate_chat_prompt(self, prompt: ChatPrompt) -> DemoRandomWordsRequest:
+    def translate_chat_prompt(self, prompt: ChatPrompt, options: SUTOptions) -> DemoRandomWordsRequest:
         # All we care about are the words in the chat history, not who said them.
-        return self._translate(_words_in_chat(prompt), prompt.options)
+        return self._translate(_words_in_chat(prompt), options)
 
     def _translate(self, text, options: SUTOptions) -> DemoRandomWordsRequest:
         return DemoRandomWordsRequest(

demo_plugin/modelgauge/suts/demo_03_sut_with_args.py

@@ -1,5 +1,5 @@
 from modelgauge.prompt import ChatPrompt, TextPrompt
-from modelgauge.sut import PromptResponseSUT, SUTResponse
+from modelgauge.sut import PromptResponseSUT, SUTOptions, SUTResponse
 from modelgauge.sut_capabilities import AcceptsChatPrompt, AcceptsTextPrompt
 from modelgauge.sut_decorator import modelgauge_sut
 from modelgauge.sut_registry import SUTS
@@ -26,10 +26,10 @@ def __init__(self, uid: str, response_text: str):
         super().__init__(uid)
         self.response_text = response_text
 
-    def translate_text_prompt(self, prompt: TextPrompt) -> DemoConstantRequest:
+    def translate_text_prompt(self, prompt: TextPrompt, options: SUTOptions) -> DemoConstantRequest:
         return DemoConstantRequest(configured_response=self.response_text)
 
-    def translate_chat_prompt(self, prompt: ChatPrompt) -> DemoConstantRequest:
+    def translate_chat_prompt(self, prompt: ChatPrompt, options: SUTOptions) -> DemoConstantRequest:
         return DemoConstantRequest(configured_response=self.response_text)
 
     def evaluate(self, request: DemoConstantRequest) -> DemoConstantResponse:

plugins/amazon/modelgauge/suts/aws_bedrock_client.py

@@ -10,7 +10,7 @@
 from modelgauge.prompt import TextPrompt
 from modelgauge.retry_decorator import retry
 from modelgauge.secret_values import InjectSecret, RequiredSecret, SecretDescription
-from modelgauge.sut import PromptResponseSUT, SUTResponse
+from modelgauge.sut import PromptResponseSUT, SUTOptions, SUTResponse
 from modelgauge.sut_capabilities import AcceptsTextPrompt
 from modelgauge.sut_decorator import modelgauge_sut
 from modelgauge.sut_registry import SUTS
@@ -122,12 +122,12 @@ def _load_client(self):
             aws_secret_access_key=self.secret_access_key,
         )
 
-    def translate_text_prompt(self, prompt: TextPrompt) -> BedrockRequest:
+    def translate_text_prompt(self, prompt: TextPrompt, options: SUTOptions) -> BedrockRequest:
         inference_config = BedrockRequest.InferenceConfig(
-            maxTokens=prompt.options.max_tokens,
-            temperature=prompt.options.temperature,
-            topP=prompt.options.top_p,
-            stopSequences=prompt.options.stop_sequences,
+            maxTokens=options.max_tokens,
+            temperature=options.temperature,
+            topP=options.top_p,
+            stopSequences=options.stop_sequences,
         )
 
         return BedrockRequest(

plugins/amazon/tests/test_aws_bedrock_client.py

@@ -1,8 +1,8 @@
 import pytest
 from unittest.mock import patch
 
-from modelgauge.prompt import SUTOptions, TextPrompt
-from modelgauge.sut import SUTResponse
+from modelgauge.prompt import TextPrompt
+from modelgauge.sut import SUTOptions, SUTResponse
 from modelgauge.typed_data import is_typeable
 
 from modelgauge.suts.aws_bedrock_client import (
@@ -44,8 +44,8 @@ def _make_response(response_text):
 
 def test_translate_text_prompt(fake_sut):
     default_options = SUTOptions()
-    prompt = TextPrompt(text="some-text", options=default_options)
-    request = fake_sut.translate_text_prompt(prompt)
+    prompt = TextPrompt(text="some-text")
+    request = fake_sut.translate_text_prompt(prompt, default_options)
 
     assert isinstance(request, BedrockRequest)
     assert request.modelId == FAKE_MODEL_ID

plugins/anthropic/modelgauge/suts/anthropic_api.py

@@ -11,7 +11,7 @@
 from modelgauge.general import APIException
 from modelgauge.prompt import ChatRole, TextPrompt
 from modelgauge.secret_values import InjectSecret, RequiredSecret, SecretDescription
-from modelgauge.sut import PromptResponseSUT, SUTResponse
+from modelgauge.sut import PromptResponseSUT, SUTOptions, SUTResponse
 from modelgauge.sut_capabilities import AcceptsTextPrompt
 from modelgauge.sut_decorator import modelgauge_sut
 from modelgauge.sut_registry import SUTS
@@ -54,16 +54,16 @@ def _load_client(self) -> Anthropic:
             max_retries=7,
         )
 
-    def translate_text_prompt(self, prompt: TextPrompt) -> AnthropicRequest:
+    def translate_text_prompt(self, prompt: TextPrompt, options: SUTOptions) -> AnthropicRequest:
         messages = [OpenAIChatMessage(content=prompt.text, role=_ROLE_MAP[ChatRole.user])]
         return AnthropicRequest(
             model=self.model,
             messages=messages,
-            max_tokens=prompt.options.max_tokens,
-            stop_sequences=prompt.options.stop_sequences,
-            temperature=prompt.options.temperature,
-            top_k=prompt.options.top_k_per_token,
-            top_p=prompt.options.top_p,
+            max_tokens=options.max_tokens,
+            stop_sequences=options.stop_sequences,
+            temperature=options.temperature,
+            top_k=options.top_k_per_token,
+            top_p=options.top_p,
         )
 
     def evaluate(self, request: AnthropicRequest) -> AnthropicMessage:

plugins/anthropic/tests/test_anthropic_api.py

@@ -3,8 +3,8 @@
 from unittest.mock import patch
 
 from modelgauge.general import APIException
-from modelgauge.prompt import SUTOptions, TextPrompt
-from modelgauge.sut import SUTResponse
+from modelgauge.prompt import TextPrompt
+from modelgauge.sut import SUTOptions, SUTResponse
 
 from modelgauge.suts.anthropic_api import AnthropicRequest, AnthropicApiKey, AnthropicSUT
 from modelgauge.suts.openai_client import OpenAIChatMessage
@@ -24,7 +24,7 @@ def simple_anthropic_request():
 def test_anthropic_api_translate_request_default_sut_options(fake_sut):
     prompt = TextPrompt(text="some-text")
 
-    request = fake_sut.translate_text_prompt(prompt)
+    request = fake_sut.translate_text_prompt(prompt, SUTOptions())
 
     assert isinstance(request, AnthropicRequest)
     assert request.model == "fake-model"
@@ -47,9 +47,9 @@ def test_anthropic_api_translate_request_non_default_sut_options(fake_sut):
         stop_sequences=["stop"],
         top_p=0.5,
     )
-    prompt = TextPrompt(text="some-text", options=options)
+    prompt = TextPrompt(text="some-text")
 
-    request = fake_sut.translate_text_prompt(prompt)
+    request = fake_sut.translate_text_prompt(prompt, options)
 
     assert request.max_tokens == 200
     assert request.temperature == 0.5

plugins/azure/modelgauge/suts/azure_client.py

@@ -8,7 +8,7 @@
 from modelgauge.general import APIException
 from modelgauge.prompt import TextPrompt
 from modelgauge.secret_values import InjectSecret, RequiredSecret, SecretDescription
-from modelgauge.sut import PromptResponseSUT, SUTResponse
+from modelgauge.sut import PromptResponseSUT, SUTOptions, SUTResponse
 from modelgauge.sut_capabilities import AcceptsTextPrompt
 from modelgauge.sut_decorator import modelgauge_sut
 from modelgauge.sut_registry import SUTS
@@ -105,16 +105,16 @@ def __init__(self, uid: str, endpoint_url: str, api_key: AzureApiKey):
         self.endpoint_url = endpoint_url
         self.api_key = api_key.value
 
-    def translate_text_prompt(self, prompt: TextPrompt) -> AzureChatRequest:
+    def translate_text_prompt(self, prompt: TextPrompt, options: SUTOptions) -> AzureChatRequest:
         messages = [AzureChatRequest.Message(content=prompt.text, role="user")]
         return AzureChatRequest(
             messages=messages,
-            max_tokens=prompt.options.max_tokens,
-            stop=prompt.options.stop_sequences,
-            temperature=prompt.options.temperature,
-            top_p=prompt.options.top_p,
-            frequency_penalty=prompt.options.frequency_penalty,
-            presence_penalty=prompt.options.presence_penalty,
+            max_tokens=options.max_tokens,
+            stop=options.stop_sequences,
+            temperature=options.temperature,
+            top_p=options.top_p,
+            frequency_penalty=options.frequency_penalty,
+            presence_penalty=options.presence_penalty,
         )
 
     def evaluate(self, request: AzureChatRequest) -> AzureChatResponse:

plugins/google/modelgauge/suts/google_genai_client.py

@@ -14,7 +14,7 @@
 from modelgauge.prompt import TextPrompt
 from modelgauge.retry_decorator import retry
 from modelgauge.secret_values import InjectSecret, RequiredSecret, SecretDescription
-from modelgauge.sut import REFUSAL_RESPONSE, PromptResponseSUT, SUTResponse  # usort: skip
+from modelgauge.sut import REFUSAL_RESPONSE, PromptResponseSUT, SUTOptions, SUTResponse  # usort: skip
 from modelgauge.sut_capabilities import AcceptsTextPrompt
 from modelgauge.sut_decorator import modelgauge_sut
 from modelgauge.sut_registry import SUTS
@@ -101,15 +101,15 @@ def safety_settings(self) -> Optional[Dict[HarmCategory, HarmBlockThreshold]]:
     def _load_client(self) -> genai.GenerativeModel:
         return genai.GenerativeModel(self.model_name)
 
-    def translate_text_prompt(self, prompt: TextPrompt) -> GoogleGenAiRequest:
+    def translate_text_prompt(self, prompt: TextPrompt, options: SUTOptions) -> GoogleGenAiRequest:
         generation_config = GoogleGenAiConfig(
-            stop_sequences=prompt.options.stop_sequences,
-            max_output_tokens=prompt.options.max_tokens,
-            temperature=prompt.options.temperature,
-            top_p=prompt.options.top_p,
-            top_k=prompt.options.top_k_per_token,
-            presence_penalty=prompt.options.presence_penalty,
-            frequency_penalty=prompt.options.frequency_penalty,
+            stop_sequences=options.stop_sequences,
+            max_output_tokens=options.max_tokens,
+            temperature=options.temperature,
+            top_p=options.top_p,
+            top_k=options.top_k_per_token,
+            presence_penalty=options.presence_penalty,
+            frequency_penalty=options.frequency_penalty,
         )
         return GoogleGenAiRequest(
             contents=prompt.text, generation_config=generation_config, safety_settings=self.safety_settings

plugins/google/tests/test_google_genai_client.py

@@ -6,8 +6,8 @@
 from google.generativeai.types import HarmCategory, HarmBlockThreshold, generation_types  # type: ignore
 
 from modelgauge.general import APIException
-from modelgauge.prompt import SUTOptions, TextPrompt
-from modelgauge.sut import REFUSAL_RESPONSE, SUTResponse
+from modelgauge.prompt import TextPrompt
+from modelgauge.sut import REFUSAL_RESPONSE, SUTOptions, SUTResponse
 from modelgauge.suts.google_genai_client import (  # type: ignore
     GEMINI_HARM_CATEGORIES,
     GoogleAiApiKey,
@@ -108,7 +108,7 @@ def mock_model(mock_model_patch, fake_raw_response):
 
 def test_google_genai_translate_request_default_options(google_default_sut):
     prompt = TextPrompt(text="some-text")
-    request = google_default_sut.translate_text_prompt(prompt)
+    request = google_default_sut.translate_text_prompt(prompt, SUTOptions())
     assert request == GoogleGenAiRequest(
         contents="some-text",
         generation_config=GoogleGenAiConfig(
@@ -129,7 +129,7 @@ def test_google_genai_translate_request_default_options_disabled_safety(google_d
     for harm in GEMINI_HARM_CATEGORIES:
         safety_settings[harm] = HarmBlockThreshold.BLOCK_NONE
 
-    request = google_disabled_safety_sut.translate_text_prompt(prompt)
+    request = google_disabled_safety_sut.translate_text_prompt(prompt, SUTOptions())
 
     assert request == GoogleGenAiRequest(
         contents="some-text",
@@ -147,13 +147,11 @@ def test_google_genai_translate_request_default_options_disabled_safety(google_d
 
 
 def test_google_genai_translate_request_generation_options(google_default_sut):
-    prompt = TextPrompt(
-        text="some-text",
-        options=SUTOptions(
-            stop_sequences=["stop"], max_tokens=200, temperature=0.5, top_k_per_token=5, frequency_penalty=0.5
-        ),
+    prompt = TextPrompt(text="some-text")
+    options = SUTOptions(
+        stop_sequences=["stop"], max_tokens=200, temperature=0.5, top_k_per_token=5, frequency_penalty=0.5
     )
-    request = google_default_sut.translate_text_prompt(prompt)
+    request = google_default_sut.translate_text_prompt(prompt, options)
     assert request == GoogleGenAiRequest(
         contents="some-text",
         generation_config=GoogleGenAiConfig(

plugins/huggingface/modelgauge/suts/huggingface_api.py

@@ -6,7 +6,7 @@
 from modelgauge.auth.huggingface_inference_token import HuggingFaceInferenceToken
 from modelgauge.prompt import TextPrompt
 from modelgauge.secret_values import InjectSecret
-from modelgauge.sut import PromptResponseSUT, SUTResponse
+from modelgauge.sut import PromptResponseSUT, SUTOptions, SUTResponse
 from modelgauge.sut_capabilities import AcceptsTextPrompt
 from modelgauge.sut_decorator import modelgauge_sut
 from modelgauge.sut_registry import SUTS
@@ -37,12 +37,10 @@ def __init__(self, uid: str, api_url: str, token: HuggingFaceInferenceToken):
         self.token = token.value
         self.api_url = api_url
 
-    def translate_text_prompt(self, prompt: TextPrompt) -> HuggingFaceChatRequest:
+    def translate_text_prompt(self, prompt: TextPrompt, options: SUTOptions) -> HuggingFaceChatRequest:
         return HuggingFaceChatRequest(
             inputs=prompt.text,
-            parameters=HuggingFaceChatParams(
-                max_new_tokens=prompt.options.max_tokens, temperature=prompt.options.temperature
-            ),
+            parameters=HuggingFaceChatParams(max_new_tokens=options.max_tokens, temperature=options.temperature),
         )
 
     @tenacity.retry(stop=stop_after_attempt(7), wait=wait_random_exponential())

plugins/huggingface/modelgauge/suts/huggingface_chat_completion.py

@@ -8,7 +8,7 @@
 from modelgauge.auth.huggingface_inference_token import HuggingFaceInferenceToken
 from modelgauge.prompt import TextPrompt
 from modelgauge.secret_values import InjectSecret
-from modelgauge.sut import PromptResponseSUT, SUTResponse, TokenProbability, TopTokens
+from modelgauge.sut import PromptResponseSUT, SUTOptions, SUTResponse, TokenProbability, TopTokens
 from modelgauge.sut_capabilities import AcceptsTextPrompt, ProducesPerTokenLogProbabilities
 from modelgauge.sut_decorator import modelgauge_sut
 from modelgauge.sut_registry import SUTS
@@ -76,14 +76,14 @@ def _create_client(self):
 
         self.client = InferenceClient(base_url=endpoint.url, token=self.token.value)
 
-    def translate_text_prompt(self, prompt: TextPrompt) -> HuggingFaceChatCompletionRequest:
+    def translate_text_prompt(self, prompt: TextPrompt, options: SUTOptions) -> HuggingFaceChatCompletionRequest:
         logprobs = False
-        if prompt.options.top_logprobs is not None:
+        if options.top_logprobs is not None:
             logprobs = True
         return HuggingFaceChatCompletionRequest(
             messages=[ChatMessage(role="user", content=prompt.text)],
             logprobs=logprobs,
-            **prompt.options.model_dump(),
+            **options.model_dump(),
         )
 
     def evaluate(self, request: HuggingFaceChatCompletionRequest) -> HuggingFaceChatCompletionOutput:

plugins/huggingface/tests/test_huggingface_api.py

@@ -2,8 +2,8 @@
 from unittest.mock import ANY, patch
 
 from modelgauge.auth.huggingface_inference_token import HuggingFaceInferenceToken
-from modelgauge.prompt import SUTOptions, TextPrompt
-from modelgauge.sut import SUTResponse
+from modelgauge.prompt import TextPrompt
+from modelgauge.sut import SUTOptions, SUTResponse
 from modelgauge.suts.huggingface_api import (
     HuggingFaceChatParams,
     HuggingFaceChatRequest,
@@ -17,22 +17,16 @@ def fake_sut():
     return HuggingFaceSUT("fake_uid", "https://fake_url.com", HuggingFaceInferenceToken("fake_token"))
 
 
-def _make_prompt(text="some text prompt", sut_options=None):
-    if sut_options is None:
-        sut_options = SUTOptions()
-    return TextPrompt(text=text, options=sut_options)
-
-
 def _make_sut_request(text, **params):
     return HuggingFaceChatRequest(inputs=text, parameters=HuggingFaceChatParams(**params))
 
 
 def test_huggingface_api_translate_text_prompt_request(fake_sut):
     prompt_text = "some text prompt"
     sut_options = SUTOptions(max_tokens=5, temperature=1.0, random="should be ignored")
-    prompt = _make_prompt(prompt_text, sut_options)
+    prompt = TextPrompt(text=prompt_text)
 
-    request = fake_sut.translate_text_prompt(prompt)
+    request = fake_sut.translate_text_prompt(prompt, sut_options)
 
     assert isinstance(request, HuggingFaceChatRequest)
     assert request.inputs == prompt_text