Rework the imports to be more versatile

LysandreJik · LysandreJik · commit 59b14fbdd48a · 2025-09-08T14:45:39.000+02:00
diff --git a/docs/source/using-the-python-api.mdx b/docs/source/using-the-python-api.mdx
@@ -12,9 +12,9 @@ import lighteval
 from lighteval.logging.evaluation_tracker import EvaluationTracker
 from lighteval.models.vllm.vllm_model import VLLMModelConfig
 from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters
-from lighteval.utils.imports import is_accelerate_available
+from lighteval.utils.imports import is_package_available
 
-if is_accelerate_available():
+if is_package_available("accelerate"):
     from datetime import timedelta
     from accelerate import Accelerator, InitProcessGroupKwargs
     accelerator = Accelerator(kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(seconds=3000))])
diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py
@@ -43,13 +43,13 @@
     TaskConfigLogger,
     VersionsLogger,
 )
-from lighteval.utils.imports import NO_TENSORBOARDX_WARN_MSG, is_nanotron_available, is_tensorboardX_available
+from lighteval.utils.imports import is_package_available, not_installed_error_message
 from lighteval.utils.utils import obj_to_markdown
 
 
 logger = logging.getLogger(__name__)
 
-if is_nanotron_available():
+if is_package_available("nanotron"):
     from nanotron.config import GeneralArgs  # type: ignore
 
 try:
@@ -645,11 +645,11 @@ def recreate_metadata_card(self, repo_id: str) -> None:  # noqa: C901
     def push_to_tensorboard(  # noqa: C901
         self, results: dict[str, dict[str, float]], details: dict[str, DetailsLogger.CompiledDetail]
     ):
-        if not is_tensorboardX_available:
-            logger.warning(NO_TENSORBOARDX_WARN_MSG)
+        if not is_package_available("tensorboardX"):
+            logger.warning(not_installed_error_message("tensorboardX"))
             return
 
-        if not is_nanotron_available():
+        if not is_package_available("nanotron"):
             logger.warning("You cannot push results to tensorboard without having nanotron installed. Skipping")
             return
 
diff --git a/src/lighteval/logging/info_loggers.py b/src/lighteval/logging/info_loggers.py
@@ -35,13 +35,13 @@
 from lighteval.models.model_output import ModelResponse
 from lighteval.tasks.lighteval_task import LightevalTask, LightevalTaskConfig
 from lighteval.tasks.requests import Doc
-from lighteval.utils.imports import is_nanotron_available
+from lighteval.utils.imports import is_package_available
 
 
 logger = logging.getLogger(__name__)
 
 
-if is_nanotron_available():
+if is_package_available("nanotron"):
     pass
 
 
diff --git a/src/lighteval/main_nanotron.py b/src/lighteval/main_nanotron.py
@@ -48,9 +48,9 @@ def nanotron(
     """
     Evaluate models using nanotron as backend.
     """
-    from lighteval.utils.imports import NO_NANOTRON_ERROR_MSG, is_nanotron_available
+    from lighteval.utils.imports import NO_NANOTRON_ERROR_MSG, is_package_available
 
-    if not is_nanotron_available():
+    if not is_package_available("nanotron"):
         raise ImportError(NO_NANOTRON_ERROR_MSG)
 
     from nanotron.config import GeneralArgs, ModelArgs, TokenizerArgs, get_config_from_dict, get_config_from_file
diff --git a/src/lighteval/metrics/utils/linguistic_tokenizers.py b/src/lighteval/metrics/utils/linguistic_tokenizers.py
@@ -19,9 +19,8 @@
 
 from lighteval.utils.imports import (
     NO_SPACY_TOKENIZER_ERROR_MSG,
-    NO_STANZA_TOKENIZER_ERROR_MSG,
     can_load_spacy_tokenizer,
-    can_load_stanza_tokenizer,
+    raise_if_package_not_available,
 )
 from lighteval.utils.language import Language
 
@@ -140,8 +139,7 @@ def span_tokenize(self, text: str) -> list[tuple[int, int]]:
 class StanzaTokenizer(WordTokenizer):
     def __init__(self, stanza_language: str, **stanza_kwargs):
         super().__init__()
-        if not can_load_stanza_tokenizer():
-            raise ImportError(NO_STANZA_TOKENIZER_ERROR_MSG)
+        raise_if_package_not_available("stanza")
         self.stanza_language = stanza_language
         self.stanza_kwargs = stanza_kwargs
         self._tokenizer = None
diff --git a/src/lighteval/metrics/utils/llm_as_judge.py b/src/lighteval/metrics/utils/llm_as_judge.py
@@ -33,7 +33,7 @@
 from tqdm import tqdm
 from tqdm.asyncio import tqdm_asyncio
 
-from lighteval.utils.imports import is_litellm_available, is_openai_available, is_vllm_available
+from lighteval.utils.imports import raise_if_package_not_available
 from lighteval.utils.utils import as_list
 
 
@@ -132,8 +132,7 @@ def __lazy_load_client(self):  # noqa: C901
             # Both "openai" and "tgi" backends use the OpenAI-compatible API
             # They are handled separately to allow for backend-specific validation and setup
             case "openai" | "tgi":
-                if not is_openai_available():
-                    raise RuntimeError("OpenAI backend is not available.")
+                raise_if_package_not_available("openai")
                 if self.client is None:
                     from openai import OpenAI
 
@@ -143,13 +142,11 @@ def __lazy_load_client(self):  # noqa: C901
                 return self.__call_api_parallel
 
             case "litellm":
-                if not is_litellm_available():
-                    raise RuntimeError("litellm is not available.")
+                raise_if_package_not_available("litellm")
                 return self.__call_litellm
 
             case "vllm":
-                if not is_vllm_available():
-                    raise RuntimeError("vllm is not available.")
+                raise_if_package_not_available("vllm")
                 if self.pipe is None:
                     from vllm import LLM, SamplingParams
                     from vllm.transformers_utils.tokenizer import get_tokenizer
diff --git a/src/lighteval/models/endpoints/tgi_model.py b/src/lighteval/models/endpoints/tgi_model.py
@@ -32,10 +32,10 @@
 from lighteval.models.endpoints.endpoint_model import InferenceEndpointModel
 from lighteval.tasks.prompt_manager import PromptManager
 from lighteval.utils.cache_management import SampleCache
-from lighteval.utils.imports import NO_TGI_ERROR_MSG, is_tgi_available
+from lighteval.utils.imports import NO_TGI_ERROR_MSG, is_package_available
 
 
-if is_tgi_available():
+if is_package_available("tgi"):
     from text_generation import AsyncClient
 else:
     from unittest.mock import Mock
@@ -98,7 +98,7 @@ class ModelClient(InferenceEndpointModel):
     _DEFAULT_MAX_LENGTH: int = 4096
 
     def __init__(self, config: TGIModelConfig) -> None:
-        if not is_tgi_available():
+        if not is_package_available("tgi"):
             raise ImportError(NO_TGI_ERROR_MSG)
         headers = (
             {} if config.inference_server_auth is None else {"Authorization": f"Bearer {config.inference_server_auth}"}
diff --git a/src/lighteval/models/model_loader.py b/src/lighteval/models/model_loader.py
@@ -49,8 +49,8 @@
     NO_TGI_ERROR_MSG,
     NO_VLLM_ERROR_MSG,
     is_litellm_available,
+    is_package_available,
     is_sglang_available,
-    is_tgi_available,
     is_vllm_available,
 )
 
@@ -105,7 +105,7 @@ def load_model(  # noqa: C901
 
 
 def load_model_with_tgi(config: TGIModelConfig):
-    if not is_tgi_available():
+    if not is_package_available("tgi"):
         raise ImportError(NO_TGI_ERROR_MSG)
 
     logger.info(f"Load model from inference server: {config.inference_server_address}")
diff --git a/src/lighteval/models/nanotron/nanotron_model.py b/src/lighteval/models/nanotron/nanotron_model.py
@@ -50,7 +50,7 @@
     Doc,
 )
 from lighteval.utils.cache_management import SampleCache, cached
-from lighteval.utils.imports import is_nanotron_available
+from lighteval.utils.imports import is_package_available
 from lighteval.utils.parallelism import find_executable_batch_size
 from lighteval.utils.utils import as_list
 
@@ -62,7 +62,7 @@
 
 TokenSequence = Union[List[int], torch.LongTensor, torch.Tensor, BatchEncoding]
 
-if is_nanotron_available():
+if is_package_available("nanotron"):
     from nanotron import distributed as dist
     from nanotron import logging
     from nanotron.config import GeneralArgs, ModelArgs, TokenizerArgs
diff --git a/src/lighteval/models/transformers/adapter_model.py b/src/lighteval/models/transformers/adapter_model.py
@@ -30,12 +30,12 @@
 
 from lighteval.models.transformers.transformers_model import TransformersModel, TransformersModelConfig
 from lighteval.models.utils import _get_dtype
-from lighteval.utils.imports import NO_PEFT_ERROR_MSG, is_peft_available
+from lighteval.utils.imports import is_package_available, raise_if_package_not_available
 
 
 logger = logging.getLogger(__name__)
 
-if is_peft_available():
+if is_package_available("peft"):
     from peft import PeftModel
 
 
@@ -60,8 +60,7 @@ class AdapterModelConfig(TransformersModelConfig):
     base_model: str
 
     def model_post_init(self, __context):
-        if not is_peft_available():
-            raise ImportError(NO_PEFT_ERROR_MSG)
+        raise_if_package_not_available("peft")
 
 
 class AdapterModel(TransformersModel):
diff --git a/src/lighteval/models/transformers/transformers_model.py b/src/lighteval/models/transformers/transformers_model.py
@@ -55,7 +55,7 @@
 from lighteval.tasks.requests import Doc
 from lighteval.utils.cache_management import SampleCache, cached
 from lighteval.utils.imports import (
-    is_accelerate_available,
+    is_package_available,
 )
 from lighteval.utils.parallelism import find_executable_batch_size
 
@@ -222,7 +222,7 @@ def __init__(
 
         self.model_name = _simplify_name(config.model_name)
 
-        if is_accelerate_available():
+        if is_package_available("accelerate"):
             model_size, _ = calculate_maximum_sizes(self.model)
             model_size = convert_bytes(model_size)
         else:
@@ -285,7 +285,7 @@ def from_model(
         else:
             self._device = self.config.device
 
-        if is_accelerate_available():
+        if is_package_available("accelerate"):
             model_size, _ = calculate_maximum_sizes(self.model)
             model_size = convert_bytes(model_size)
         else:
@@ -326,7 +326,7 @@ def disable_tqdm(self) -> bool:
 
     def init_model_parallel(self, model_parallel: bool | None = None) -> Tuple[bool, Optional[dict], Optional[str]]:
         """Compute all the parameters related to model_parallel"""
-        if not is_accelerate_available():
+        if not is_package_available("accelerate"):
             return False, None, None
 
         self.num_local_processes = int(os.environ.get("LOCAL_WORLD_SIZE", 1))
diff --git a/src/lighteval/models/transformers/vlm_transformers_model.py b/src/lighteval/models/transformers/vlm_transformers_model.py
@@ -47,7 +47,7 @@
 from lighteval.tasks.requests import Doc
 from lighteval.utils.cache_management import SampleCache, cached
 from lighteval.utils.imports import (
-    is_accelerate_available,
+    is_package_available,
 )
 
 
@@ -204,7 +204,7 @@ def disable_tqdm(self) -> bool:
     # Copied from ./transformers_model.py
     def init_model_parallel(self, model_parallel: bool | None = None) -> Tuple[bool, Optional[dict], Optional[str]]:
         """Compute all the parameters related to model_parallel"""
-        if not is_accelerate_available():
+        if not is_package_available("accelerate"):
             return False, None, None
 
         self.num_local_processes = int(os.environ.get("LOCAL_WORLD_SIZE", 1))
diff --git a/src/lighteval/pipeline.py b/src/lighteval/pipeline.py
@@ -42,31 +42,19 @@
 from lighteval.tasks.lighteval_task import LightevalTask
 from lighteval.tasks.registry import Registry
 from lighteval.tasks.requests import SamplingMethod
-from lighteval.utils.imports import (
-    NO_ACCELERATE_ERROR_MSG,
-    NO_NANOTRON_ERROR_MSG,
-    NO_OPENAI_ERROR_MSG,
-    NO_SGLANG_ERROR_MSG,
-    NO_TGI_ERROR_MSG,
-    NO_VLLM_ERROR_MSG,
-    is_accelerate_available,
-    is_nanotron_available,
-    is_openai_available,
-    is_sglang_available,
-    is_tgi_available,
-    is_vllm_available,
-)
+from lighteval.utils.imports import is_package_available, raise_if_package_not_available
 from lighteval.utils.parallelism import test_all_gather
 from lighteval.utils.utils import make_results_table, remove_reasoning_tags
 
 
-if is_accelerate_available():
+if is_package_available("accelerate"):
     from accelerate import Accelerator, InitProcessGroupKwargs
 else:
     from unittest.mock import Mock
 
     Accelerator = InitProcessGroupKwargs = Mock()
-if is_nanotron_available():
+
+if is_package_available("nanotron"):
     from nanotron import distributed as dist
     from nanotron.parallel.context import ParallelContext
 
@@ -110,23 +98,17 @@ class PipelineParameters:
     def __post_init__(self):  # noqa C901
         # Import testing
         if self.launcher_type == ParallelismManager.ACCELERATE:
-            if not is_accelerate_available():
-                raise ImportError(NO_ACCELERATE_ERROR_MSG)
+            raise_if_package_not_available("accelerate")
         elif self.launcher_type == ParallelismManager.VLLM:
-            if not is_vllm_available():
-                raise ImportError(NO_VLLM_ERROR_MSG)
+            raise_if_package_not_available("vllm")
         elif self.launcher_type == ParallelismManager.SGLANG:
-            if not is_sglang_available():
-                raise ImportError(NO_SGLANG_ERROR_MSG)
+            raise_if_package_not_available("sglang")
         elif self.launcher_type == ParallelismManager.TGI:
-            if not is_tgi_available():
-                raise ImportError(NO_TGI_ERROR_MSG)
+            raise_if_package_not_available("tgi")
         elif self.launcher_type == ParallelismManager.NANOTRON:
-            if not is_nanotron_available():
-                raise ImportError(NO_NANOTRON_ERROR_MSG)
+            raise_if_package_not_available("nanotron")
         elif self.launcher_type == ParallelismManager.OPENAI:
-            if not is_openai_available():
-                raise ImportError(NO_OPENAI_ERROR_MSG)
+            raise_if_package_not_available("openai")
 
         # Convert reasoning tags to list if needed
         if not isinstance(self.reasoning_tags, list):
@@ -187,12 +169,12 @@ def __init__(
     def _init_parallelism_manager(self):
         accelerator, parallel_context = None, None
         if self.launcher_type == ParallelismManager.ACCELERATE:
-            if not is_accelerate_available():
+            if not is_package_available("accelerate"):
                 raise ValueError("You are trying to launch an accelerate model, but accelerate is not installed")
             accelerator = Accelerator(kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(seconds=3000))])
             test_all_gather(accelerator=accelerator)
         elif self.launcher_type == ParallelismManager.NANOTRON:
-            if not is_nanotron_available():
+            if not is_package_available("nanotron"):
                 raise ValueError("You are trying to launch a nanotron model, but nanotron is not installed")
             dist.initialize_torch_distributed()
             parallel_context = ParallelContext(
diff --git a/src/lighteval/tasks/extended/__init__.py b/src/lighteval/tasks/extended/__init__.py
@@ -20,19 +20,14 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-from lighteval.utils.imports import can_load_extended_tasks
 
+import lighteval.tasks.extended.hle.main as hle
+import lighteval.tasks.extended.ifeval.main as ifeval
+import lighteval.tasks.extended.lcb.main as lcb
+import lighteval.tasks.extended.mix_eval.main as mix_eval
+import lighteval.tasks.extended.mt_bench.main as mt_bench
+import lighteval.tasks.extended.olympiade_bench.main as olympiad_bench
+import lighteval.tasks.extended.tiny_benchmarks.main as tiny_benchmarks
 
-if can_load_extended_tasks():
-    import lighteval.tasks.extended.hle.main as hle
-    import lighteval.tasks.extended.ifeval.main as ifeval
-    import lighteval.tasks.extended.lcb.main as lcb
-    import lighteval.tasks.extended.mix_eval.main as mix_eval
-    import lighteval.tasks.extended.mt_bench.main as mt_bench
-    import lighteval.tasks.extended.olympiade_bench.main as olympiad_bench
-    import lighteval.tasks.extended.tiny_benchmarks.main as tiny_benchmarks
 
-    AVAILABLE_EXTENDED_TASKS_MODULES = [ifeval, tiny_benchmarks, mt_bench, mix_eval, olympiad_bench, hle, lcb]
-
-else:
-    AVAILABLE_EXTENDED_TASKS_MODULES = []
+AVAILABLE_EXTENDED_TASKS_MODULES = [ifeval, tiny_benchmarks, mt_bench, mix_eval, olympiad_bench, hle, lcb]
diff --git a/src/lighteval/tasks/extended/ifeval/instructions.py b/src/lighteval/tasks/extended/ifeval/instructions.py
@@ -21,7 +21,11 @@
 import re
 import string
 
-import langdetect
+from ....utils.imports import is_package_installed
+
+
+if is_package_installed("langdetect"):
+    import langdetect
 
 import lighteval.tasks.extended.ifeval.instructions_utils as instructions_util
 
diff --git a/src/lighteval/tasks/extended/ifeval/main.py b/src/lighteval/tasks/extended/ifeval/main.py
@@ -32,9 +32,11 @@
 from lighteval.models.model_output import ModelResponse
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc, SamplingMethod
+from lighteval.utils.imports import requires
 
 
 # Very specific task where there are no precise outputs but instead we test if the format obeys rules
+@requires("langdetect")
 def ifeval_prompt(line, task_name: str = ""):
     return Doc(
         task_name=task_name,
diff --git a/src/lighteval/tasks/registry.py b/src/lighteval/tasks/registry.py
diff --git a/src/lighteval/utils/imports.py b/src/lighteval/utils/imports.py
diff --git a/src/lighteval/utils/parallelism.py b/src/lighteval/utils/parallelism.py
diff --git a/tests/models/test_transformers_model.py b/tests/models/test_transformers_model.py
diff --git a/tests/pipeline/test_reasoning_tags.py b/tests/pipeline/test_reasoning_tags.py
diff --git a/tests/utils.py b/tests/utils.py
diff --git a/tests/utils/test_caching.py b/tests/utils/test_caching.py
