From b12ac5bacb1dba0639d539d2794640d4b965665d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=BCneyt=20=C3=96zt=C3=BCrk?= Date: Wed, 12 Aug 2026 14:46:25 +0300 Subject: [PATCH 1/2] fix(inspect): align GenerateConfig field names with current inspect-ai - rename CLI/eval_set kwarg 'frequence_penalty' -> 'frequency_penalty' (typo; inspect-ai field name) - rename 'log_probs' -> 'logprobs' (inspect-ai field name) - pass 'response_format' only when the installed inspect-ai still accepts it, warn otherwise (removed in >= 0.3.141) Fixes #1327: a fresh 'pip install lighteval' resolves inspect-ai to the latest release and 'lighteval eval' crashed at startup with a pydantic ValidationError. Verified on inspect-ai 0.3.257: the repro command from the issue now completes. --- src/lighteval/main_inspect.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/lighteval/main_inspect.py b/src/lighteval/main_inspect.py index 4aa5cbf87..12c274368 100644 --- a/src/lighteval/main_inspect.py +++ b/src/lighteval/main_inspect.py @@ -43,6 +43,25 @@ logger = logging.getLogger(__name__) +def _response_format_kwargs(response_format: str | None) -> dict: + """inspect-ai removed GenerateConfig.response_format (>= 0.3.141). + + Pass the flag through only when the installed inspect-ai accepts it, so a + fresh install does not crash at startup with a pydantic ValidationError. + """ + if response_format is None: + return {} + from inspect_ai.model import GenerateConfig + + if "response_format" in GenerateConfig.model_fields: + return {"response_format": response_format} + logger.warning( + "--response-format is not supported by the installed inspect-ai (%s); ignoring it.", + getattr(__import__("inspect_ai"), "__version__", "unknown"), + ) + return {} + + @task def get_inspect_ai_task( @@ -267,7 +286,7 @@ def eval( # noqa C901 rich_help_panel=HELP_PANEL_NAME_1, ), ] = None, - frequence_penalty: Annotated[ + frequency_penalty: Annotated[ float | None, Option( help="Number between -2.0 and 2.0, Penalizes tokens that appear in the text too frequently, reducing repetition.", @@ -309,7 +328,7 @@ def eval( # noqa C901 rich_help_panel=HELP_PANEL_NAME_1, ), ] = None, - log_probs: Annotated[ + logprobs: Annotated[ bool | None, Option( help="Returns log probabilities for each token in the generated text", rich_help_panel=HELP_PANEL_NAME_1 @@ -474,19 +493,19 @@ def eval( # noqa C901 temperature=temperature, top_p=top_p, top_k=top_k, - frequence_penalty=frequence_penalty, + frequency_penalty=frequency_penalty, presence_penalty=presence_penalty, seed=seed, stop_seqs=stop_seqs, num_choices=num_choices, best_of=best_of, - log_probs=log_probs, + logprobs=logprobs, top_logprobs=top_logprobs, cache_prompt=cache_prompt, reasoning_effort=reasoning_effort, reasoning_tokens=reasoning_tokens, reasoning_history=reasoning_history, - response_format=response_format, + **_response_format_kwargs(response_format), parallel_tool_calls=parallel_tool_calls, max_tool_output=max_tool_output, internal_tools=internal_tools, From b038d699019e643b9cf71b703eec55579f2eee14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=BCneyt=20=C3=96zt=C3=BCrk?= Date: Wed, 12 Aug 2026 14:50:52 +0300 Subject: [PATCH 2/2] style: ruff blank-line fixes --- src/lighteval/main_inspect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lighteval/main_inspect.py b/src/lighteval/main_inspect.py index 12c274368..b59bc4631 100644 --- a/src/lighteval/main_inspect.py +++ b/src/lighteval/main_inspect.py @@ -43,6 +43,7 @@ logger = logging.getLogger(__name__) + def _response_format_kwargs(response_format: str | None) -> dict: """inspect-ai removed GenerateConfig.response_format (>= 0.3.141). @@ -62,7 +63,6 @@ def _response_format_kwargs(response_format: str | None) -> dict: return {} - @task def get_inspect_ai_task( lighteval_task_config: LightevalTaskConfig,