Fix PEFT example and base-model config resolution for adapter models

examples/model_configs/peft_model.yaml

@@ -1,14 +1,12 @@
 model_parameters:
-  model_name: "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B" # pretrained=model_name,trust_remote_code=boolean,revision=revision_to_use,model_parallel=True ... For a PEFT model, the pretrained model should be the one trained with PEFT and the base model below will contain the original model on which the adapters will be applied.
-  tokenizer: null # name of tokenizer to use if different from the model's default
-  subfolder: null # subfolder in the model's directory to use
-  dtype: "float16"  # Specifying the model to be loaded in 4 bit uses BitsAndBytesConfig. The other option is to use "8bit" quantization.
-  compile: true
-  revision: "main" # revision to use
-  trust_remote_code: true # Trust remote code
-  model_parallel: null # Model parallel
-  max_length: 2048 # maximum length of the input text and the generated text
-
-  # should go in generation
-  max_generation_toks: 256 # maximum number of tokens to generate
-  batch_size: 10 # batch size to use
+  adapter_weights: true # Select PEFT adapter loading rather than the standard Transformers backend.
+  model_name: "ybelkada/opt-350m-lora" # Hub ID or local path containing the adapter weights.
+  base_model: "facebook/opt-350m" # Base model on which the adapter was trained.
+  revision: "main"
+  dtype: "float16"
+  compile: false
+  model_parallel: false
+  batch_size: 1
+  generation_parameters:
+    max_new_tokens: 256
+    temperature: 0.0

src/lighteval/models/transformers/adapter_model.py

@@ -39,7 +39,6 @@
     from peft import PeftModel
 
 
-@requires("peft")
 class AdapterModelConfig(TransformersModelConfig):
     """Configuration class for PEFT (Parameter-Efficient Fine-Tuning) adapter models.
 
@@ -59,15 +58,24 @@ class AdapterModelConfig(TransformersModelConfig):
 
     base_model: str
 
+    def model_post_init(self, __context):
+        if self.tokenizer is None:
+            self.tokenizer = self.base_model
+        super().model_post_init(__context)
+
+    def get_transformers_config(self):
+        return super().get_transformers_config(model_name=self.base_model)
 
+
+@requires("peft")
 class AdapterModel(TransformersModel):
     def _create_auto_model(self) -> transformers.PreTrainedModel:
         """Returns a PeftModel from a base model and a version fined tuned using PEFT."""
         torch_dtype = _get_dtype(self.config.dtype)
         model_parallel, max_memory, device_map = self.init_model_parallel(self.config.model_parallel)
         self.config.model_parallel = model_parallel
 
-        adapter_weights = self.config.pretrained
+        adapter_weights = self.config.model_name
         merged_path = f"{adapter_weights}-adapter-applied"
 
         if self.config.dtype == "4bit":

src/lighteval/models/transformers/transformers_model.py

@@ -179,14 +179,14 @@ def model_post_init(self, __context):
                 "You set `multichoice_continuations_start_space` to false. This will remove a leading space from multichoice continuations, if present."
             )
 
-    def get_transformers_config(self) -> PretrainedConfig:
+    def get_transformers_config(self, model_name: str | None = None) -> PretrainedConfig:
         revision = self.revision
 
         if self.subfolder:
             revision = f"{self.revision}/{self.subfolder}"
 
         auto_config = AutoConfig.from_pretrained(
-            self.model_name,
+            model_name or self.model_name,
             revision=revision,
             trust_remote_code=self.trust_remote_code,
         )

tests/unit/models/test_adapter_model.py

@@ -0,0 +1,93 @@
+# MIT License
+
+# Copyright (c) 2024 The HuggingFace Team
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+import yaml
+
+from lighteval.models.transformers.adapter_model import AdapterModel, AdapterModelConfig
+from lighteval.utils.imports import is_package_available
+
+
+REPO_ROOT = Path(__file__).parents[3]
+
+
+def load_example_config() -> AdapterModelConfig:
+    with (REPO_ROOT / "examples/model_configs/peft_model.yaml").open() as config_file:
+        config = yaml.safe_load(config_file)["model_parameters"]
+
+    assert config.pop("adapter_weights") is True
+    return AdapterModelConfig(**config)
+
+
+def test_peft_example_uses_base_model_for_config_and_tokenizer():
+    config = load_example_config()
+
+    assert config.model_name == "ybelkada/opt-350m-lora"
+    assert config.base_model == "facebook/opt-350m"
+    assert config.tokenizer == config.base_model
+    assert config.generation_parameters.max_new_tokens == 256
+
+    expected_config = MagicMock()
+    with patch(
+        "lighteval.models.transformers.transformers_model.AutoConfig.from_pretrained",
+        return_value=expected_config,
+    ) as from_pretrained:
+        assert config.get_transformers_config() is expected_config
+
+    from_pretrained.assert_called_once_with(
+        config.base_model,
+        revision="main",
+        trust_remote_code=False,
+    )
+
+
+@pytest.mark.skipif(not is_package_available("peft"), reason="requires the adapters extra")
+def test_adapter_model_loads_weights_from_model_name():
+    config = load_example_config()
+    adapter_model = AdapterModel.__new__(AdapterModel)
+    adapter_model.config = config
+    adapter_model.accelerator = MagicMock(is_local_main_process=True)
+    adapter_model._tokenizer = [0, 1]
+    adapter_model.init_model_parallel = MagicMock(return_value=(False, None, None))
+
+    base_model = MagicMock()
+    base_model.config.vocab_size = len(adapter_model._tokenizer)
+    peft_model = MagicMock()
+    peft_model.merge_and_unload.return_value = MagicMock()
+    loaded_model = MagicMock()
+
+    with (
+        patch(
+            "lighteval.models.transformers.adapter_model.AutoModelForCausalLM.from_pretrained",
+            side_effect=[base_model, loaded_model],
+        ),
+        patch(
+            "lighteval.models.transformers.adapter_model.PeftModel.from_pretrained",
+            return_value=peft_model,
+        ) as from_pretrained,
+    ):
+        assert adapter_model._create_auto_model() is loaded_model
+
+    from_pretrained.assert_called_once_with(base_model, config.model_name)