Store provider credentials as SecretStr in model configs

README.md

@@ -145,14 +145,9 @@ MODEL_NAME = "meta-llama/Meta-Llama-3-8B-Instruct"
 BENCHMARKS = "gsm8k"
 
 evaluation_tracker = EvaluationTracker(output_dir="./results")
-pipeline_params = PipelineParameters(
-    launcher_type=ParallelismManager.NONE,
-    max_samples=2
-)
+pipeline_params = PipelineParameters(launcher_type=ParallelismManager.NONE, max_samples=2)
 
-model = AutoModelForCausalLM.from_pretrained(
-  MODEL_NAME, device_map="auto"
-)
+model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, device_map="auto")
 config = TransformersModelConfig(model_name=MODEL_NAME, batch_size=1)
 model = TransformersModel.from_model(model, config)
 

docs/source/offline-evaluation.md

@@ -18,12 +18,7 @@ from lighteval.tasks.requests import Doc
 
 
 def local_prompt(line: dict, task_name: str) -> Doc:
-    return Doc(
-        task_name=task_name,
-        query=line["question"],
-        choices=line["choices"],
-        gold_index=line["answer"]
-    )
+    return Doc(task_name=task_name, query=line["question"], choices=line["choices"], gold_index=line["answer"])
 
 
 local_data = Path(__file__).parent / "samples" / "faq.jsonl"

src/lighteval/logging/evaluation_tracker.py

@@ -211,7 +211,11 @@ def __init__(
     @property
     def results(self):
         config_general = asdict(self.general_config_logger)
-        config_general["model_config"] = config_general["model_config"].model_dump()
+        # This exclude set is defense in depth: SecretStr fields already serialize masked,
+        # but this ensures no future plain-str secret field leaks through model_dump().
+        config_general["model_config"] = config_general["model_config"].model_dump(
+            exclude={"api_key", "inference_server_auth"}
+        )
         results = {
             "config_general": config_general,
             "results": self.metrics_logger.metric_aggregated,

src/lighteval/metrics/utils/llm_as_judge.py

@@ -29,7 +29,7 @@
 from typing import Callable, Literal, Optional
 
 from huggingface_hub import AsyncInferenceClient, InferenceTimeoutError
-from pydantic import BaseModel
+from pydantic import BaseModel, SecretStr
 from requests.exceptions import HTTPError
 from tqdm import tqdm
 from tqdm.asyncio import tqdm_asyncio
@@ -74,6 +74,7 @@ class JudgeLM:
         judge_backend (Literal["litellm", "openai", "transformers", "tgi", "vllm", "inference-providers"]): The backend for the judge.
         url (str | None): The URL for the OpenAI API.
         api_key (str | None): The API key for the OpenAI API (either OpenAI or HF key).
+            Stored internally as a SecretStr so it is masked in logs, reprs, and serialized configs.
         max_tokens (int): The maximum number of tokens to generate. Defaults to 512.
         response_format (BaseModel | None): The format of the response from the API, used for the OpenAI and TGI backend.
         hf_provider (Literal["black-forest-labs", "cerebras", "cohere", "fal-ai", "fireworks-ai",
@@ -129,7 +130,7 @@ def __init__(
         self.process_judge_response = process_judge_response
 
         self.url = url
-        self.api_key = api_key
+        self.api_key = SecretStr(api_key) if api_key is not None else None
         self.backend = judge_backend
         self.hf_provider = hf_provider
         self.max_tokens = max_tokens
@@ -156,7 +157,8 @@ def __lazy_load_client(self):  # noqa: C901
                     from openai import OpenAI
 
                     self.client = OpenAI(
-                        api_key=self.api_key if self.url is None else None, base_url=self.url if self.url else None
+                        api_key=self.api_key.get_secret_value() if self.url is None and self.api_key else None,
+                        base_url=self.url if self.url else None,
                     )
                 return self.__call_api_parallel
 
@@ -195,7 +197,11 @@ def __lazy_load_client(self):  # noqa: C901
             case "inference-providers":
                 from huggingface_hub import AsyncInferenceClient
 
-                self.client = AsyncInferenceClient(token=self.api_key, base_url=self.url, provider=self.hf_provider)
+                self.client = AsyncInferenceClient(
+                    token=self.api_key.get_secret_value() if self.api_key else None,
+                    base_url=self.url,
+                    provider=self.hf_provider,
+                )
                 return self.__call_hf_inference_async
 
             case _:
@@ -340,7 +346,7 @@ def __call_api(prompt):
                     if max_new_tokens is not None:
                         kwargs["max_tokens"] = (max_new_tokens,)
                     if self.api_key is not None:
-                        kwargs["api_key"] = self.api_key
+                        kwargs["api_key"] = self.api_key.get_secret_value()
                     if self.url is not None:
                         kwargs["base_url"] = self.url
 

src/lighteval/models/endpoints/litellm_model.py

@@ -26,6 +26,7 @@
 from json import JSONDecodeError
 
 import requests
+from pydantic import SecretStr
 from tqdm import tqdm
 
 from lighteval.data import GenerativeTaskDataset
@@ -77,9 +78,10 @@ class LiteLLMModelConfig(ModelConfig):
         base_url (str | None):
             Custom base URL for the API. If None, uses provider's default URL.
             Useful for using custom endpoints or local deployments.
-        api_key (str | None):
+        api_key (SecretStr | None):
             API key for authentication. If None, reads from environment variables.
             Environment variable names are provider-specific (e.g., OPENAI_API_KEY).
+            Stored as a SecretStr so it is masked in logs, reprs, and serialized configs.
         concurrent_requests (int):
             Maximum number of concurrent API requests to execute in parallel.
             Higher values can improve throughput for batch processing but may hit rate limits
@@ -121,7 +123,7 @@ class LiteLLMModelConfig(ModelConfig):
     model_name: str
     provider: str | None = None
     base_url: str | None = None
-    api_key: str | None = None
+    api_key: SecretStr | None = None
     concurrent_requests: int = 10
     verbose: bool = False
     max_model_length: int | None = None
@@ -144,7 +146,7 @@ def __init__(self, config: LiteLLMModelConfig) -> None:
         self.model = config.model_name
         self.provider = config.provider or config.model_name.split("/")[0]
         self.base_url = config.base_url
-        self.api_key = config.api_key
+        self.api_key = config.api_key.get_secret_value() if config.api_key is not None else None
         self.generation_parameters = config.generation_parameters
         self.concurrent_requests = config.concurrent_requests
         self._max_length = config.max_model_length

src/lighteval/models/endpoints/tgi_model.py

@@ -26,6 +26,7 @@
 
 import requests
 from huggingface_hub import TextGenerationInputGenerateParameters, TextGenerationInputGrammarType, TextGenerationOutput
+from pydantic import SecretStr
 from transformers.models.auto.tokenization_auto import AutoTokenizer
 
 from lighteval.models.abstract_model import ModelConfig
@@ -65,8 +66,9 @@ class TGIModelConfig(ModelConfig):
         inference_server_address (str | None):
             Address of the TGI server. Format: "http://host:port" or "https://host:port".
             Example: "http://localhost:8080"
-        inference_server_auth (str | None):
+        inference_server_auth (SecretStr | None):
             Authentication token for the TGI server. If None, no authentication is used.
+            Stored as a SecretStr so it is masked in logs, reprs, and serialized configs.
         model_name (str | None):
             Optional model name override. If None, uses the model name from server info.
         generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters):
@@ -91,7 +93,7 @@ class TGIModelConfig(ModelConfig):
     """
 
     inference_server_address: str | None = None
-    inference_server_auth: str | None = None
+    inference_server_auth: SecretStr | None = None
     model_name: str | None
     model_info: dict | None = None
     batch_size: int = 1
@@ -104,7 +106,9 @@ class ModelClient(InferenceEndpointModel):
 
     def __init__(self, config: TGIModelConfig) -> None:
         headers = (
-            {} if config.inference_server_auth is None else {"Authorization": f"Bearer {config.inference_server_auth}"}
+            {}
+            if config.inference_server_auth is None
+            else {"Authorization": f"Bearer {config.inference_server_auth.get_secret_value()}"}
         )
 
         self.client = AsyncClient(config.inference_server_address, headers=headers, timeout=240)

tests/unit/logging/test_evaluation_tracker.py

@@ -299,13 +299,11 @@ def setUp(self):
             "model_name": "test/case",
             "provider": None,
             "base_url": None,
-            "api_key": None,
             "system_prompt": ref_system_prompt,
             "generation_parameters": ref_generation_parameters,
         }  # ruff: noqa: E501
         self.tgi_ref_config = {
             "inference_server_address": None,
-            "inference_server_auth": None,
             "model_name": "test/case",
             "model_info": None,
             "system_prompt": ref_system_prompt,
@@ -485,3 +483,41 @@ def test_model_config_property_with_different_model_configs(self):
                 for k, v in ref_config.items():
                     with self.subTest(model_config=model_config, model_property=k):
                         self.assertEqual(results["config_general"]["model_config"][k], v)
+
+    def test_litellm_api_key_never_leaks_to_results(self):
+        """A LiteLLM api_key must never appear in the results dict or its JSON/parquet serialization."""
+        from lighteval.logging.evaluation_tracker import EnhancedJSONEncoder
+        from lighteval.models.endpoints.litellm_model import LiteLLMModelConfig
+
+        secret = "sk-super-secret-should-not-leak"
+        model_config = LiteLLMModelConfig(model_name="test/case", api_key=secret)
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            evaluation_tracker = EvaluationTracker(output_dir=tmp_dir)
+            evaluation_tracker.general_config_logger.log_model_info(model_config=model_config)
+
+            results = evaluation_tracker.results
+
+            self.assertNotIn("api_key", results["config_general"]["model_config"])
+
+            serialized = json.dumps(results, cls=EnhancedJSONEncoder)
+            self.assertNotIn(secret, serialized)
+
+    def test_tgi_inference_server_auth_never_leaks_to_results(self):
+        """A TGI inference_server_auth token must never appear in the results dict or its JSON/parquet serialization."""
+        from lighteval.logging.evaluation_tracker import EnhancedJSONEncoder
+        from lighteval.models.endpoints.tgi_model import TGIModelConfig
+
+        secret = "tgi-bearer-token-should-not-leak"
+        model_config = TGIModelConfig(model_name="test/case", inference_server_auth=secret)
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            evaluation_tracker = EvaluationTracker(output_dir=tmp_dir)
+            evaluation_tracker.general_config_logger.log_model_info(model_config=model_config)
+
+            results = evaluation_tracker.results
+
+            self.assertNotIn("inference_server_auth", results["config_general"]["model_config"])
+
+            serialized = json.dumps(results, cls=EnhancedJSONEncoder)
+            self.assertNotIn(secret, serialized)