🚨 Remove imports from other models (#49473)

* Remove imports from other models

* remove imports in more cases

Author: guarin

src/transformers/models/embedding_gemma2/modular_embedding_gemma2.py

@@ -36,7 +36,6 @@
     TransformersKwargs,
     auto_docstring,
     can_return_tuple,
-    is_vision_available,
     logging,
     torch_compilable_check,
 )
@@ -55,14 +54,11 @@
     eager_attention_forward,
 )
 from ..gemma4.processing_gemma4 import Gemma4Processor, Gemma4ProcessorKwargs
-from ..gemma4.video_processing_gemma4 import Gemma4VideoProcessor, Gemma4VideoProcessorKwargs
-
-
-if is_vision_available():
-    from ..gemma4.image_processing_gemma4 import (
-        Gemma4ImageProcessorKwargs,
-        get_aspect_ratio_preserving_size,  # noqa: F401  # trf-ignore: TRF039
-    )
+from ..gemma4.video_processing_gemma4 import (
+    Gemma4VideoProcessor,
+    Gemma4VideoProcessorKwargs,
+    get_aspect_ratio_preserving_size,  # noqa: F401
+)
 
 
 logger = logging.get_logger(__name__)
@@ -845,7 +841,8 @@ def sample_frames(
 
 
 class EmbeddingGemma2ProcessorKwargs(Gemma4ProcessorKwargs):
-    images_kwargs: Gemma4ImageProcessorKwargs
+    images_kwargs = AttributeError()
+
     _defaults = {
         "text_kwargs": {
             "padding": True,

src/transformers/models/embedding_gemma2/processing_embedding_gemma2.py

@@ -18,6 +18,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import math
+
 import numpy as np
 
 from ...audio_utils import AudioInput, is_valid_audio
@@ -26,18 +28,12 @@
 from ...tokenization_utils_base import PreTokenizedInput, TextInput
 from ...utils import (
     auto_docstring,
-    is_vision_available,
 )
 from ...utils.import_utils import requires
 from ...video_utils import VideoInput, is_valid_video, make_batched_videos
 
 
-if is_vision_available():
-    from ..gemma4.image_processing_gemma4 import Gemma4ImageProcessorKwargs, get_aspect_ratio_preserving_size
-
-
 class EmbeddingGemma2ProcessorKwargs(ProcessingKwargs, total=False):
-    images_kwargs: Gemma4ImageProcessorKwargs
     _defaults = {
         "text_kwargs": {
             "padding": True,
@@ -50,6 +46,60 @@ class EmbeddingGemma2ProcessorKwargs(ProcessingKwargs, total=False):
     }
 
 
+def get_aspect_ratio_preserving_size(
+    height: int,
+    width: int,
+    patch_size: int,
+    max_patches: int,
+    pooling_kernel_size: int,
+) -> tuple[int, int]:
+    """
+    Image is resized to preserve aspect ratio so it fits within the patch budget.
+    Target dimensions are the largest that:
+    1) Produce at most `max_patches` patches when patchified with `patch_size`
+    2) Have height and width divisible by `pooling_kernel_size * patch_size`
+    """
+    total_px = height * width
+    target_px = max_patches * (patch_size**2)
+    factor = math.sqrt(target_px / total_px)
+    ideal_height = factor * height
+    ideal_width = factor * width
+    side_mult = pooling_kernel_size * patch_size
+
+    # Round down to nearest multiple of side_mult
+    target_height = int(math.floor(ideal_height / side_mult)) * side_mult
+    target_width = int(math.floor(ideal_width / side_mult)) * side_mult
+
+    # Handle edge cases where one or both dimensions round to 0
+    if target_height == 0 and target_width == 0:
+        raise ValueError(
+            "Attempting to resize to a 0 x 0 image. Resized height should be divisible by "
+            f"`pooling_kernel_size * patch_size`={pooling_kernel_size * patch_size}."
+        )
+
+    max_side_length = (max_patches // pooling_kernel_size**2) * side_mult
+    if target_height == 0:
+        target_height = side_mult
+        target_width = min(
+            int(math.floor(width / height)) * side_mult,
+            max_side_length,
+        )
+    elif target_width == 0:
+        target_width = side_mult
+        target_height = min(
+            int(math.floor(height / width)) * side_mult,
+            max_side_length,
+        )
+
+    if target_height * target_width > target_px:
+        raise ValueError(
+            f"Resizing [{height}x{width}] to [{target_height}x{target_width}] "
+            f"but this exceeds {max_patches} patches with patch_size {patch_size}"
+        )
+
+    return target_height, target_width
+
+
 @auto_docstring
 @requires(backends=("vision",))
 class EmbeddingGemma2Processor(ProcessorMixin):

src/transformers/models/ijepa/modular_ijepa.py

@@ -1,14 +1,13 @@
 import torch
 import torch.nn as nn
 
-from transformers.models.ijepa.configuration_ijepa import IJepaConfig
-
 from ... import initialization as init
 from ...modeling_outputs import BaseModelOutputWithPooling, ImageClassifierOutput
 from ...modeling_utils import PreTrainedModel
 from ...processing_utils import Unpack
 from ...utils import TransformersKwargs, auto_docstring, torch_int
 from ..vit.modeling_vit import ViTEmbeddings, ViTForImageClassification, ViTModel, ViTPreTrainedModel
+from .configuration_ijepa import IJepaConfig
 
 
 class IJepaEmbeddings(ViTEmbeddings):

src/transformers/models/lightglue/configuration_lightglue.py

@@ -23,7 +23,6 @@
 from ...configuration_utils import PreTrainedConfig, SubConfigSpec
 from ...utils import auto_docstring
 from ..auto import AutoConfig
-from ..superpoint import SuperPointConfig
 
 
 @auto_docstring(checkpoint="ETH-CVG/lightglue_superpoint")
@@ -61,7 +60,7 @@ class LightGlueConfig(PreTrainedConfig):
         "keypoint_detector_config": SubConfigSpec(config_class=AutoConfig, model_type="superpoint"),
     }
 
-    keypoint_detector_config: dict | SuperPointConfig | None = None
+    keypoint_detector_config: dict | PreTrainedConfig | None = None
     descriptor_dim: int = 256
     num_hidden_layers: int = 9
     num_attention_heads: int = 4

src/transformers/models/lightglue/modular_lightglue.py

@@ -35,7 +35,6 @@
 from ..llama.modeling_llama import LlamaAttention, eager_attention_forward
 from ..superglue.image_processing_pil_superglue import SuperGlueImageProcessorPil
 from ..superglue.image_processing_superglue import SuperGlueImageProcessor, SuperGlueImageProcessorKwargs
-from ..superpoint import SuperPointConfig
 
 
 logger = logging.get_logger(__name__)
@@ -76,7 +75,7 @@ class LightGlueConfig(PreTrainedConfig):
         "keypoint_detector_config": SubConfigSpec(config_class=AutoConfig, model_type="superpoint"),
     }
 
-    keypoint_detector_config: dict | SuperPointConfig | None = None
+    keypoint_detector_config: dict | PreTrainedConfig | None = None
     descriptor_dim: int = 256
     num_hidden_layers: int = 9
     num_attention_heads: int = 4

src/transformers/models/nemotron_h/modeling_nemotron_h.py

@@ -42,7 +42,6 @@
 from ...modeling_layers import GradientCheckpointingLayer
 from ...modeling_outputs import MoeCausalLMOutputWithPast, MoeModelOutputWithPast
 from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
-from ...models.zamba2.modeling_zamba2 import Zamba2RMSNormGated
 from ...processing_utils import Unpack
 from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging
 from ...utils.generic import merge_with_config_defaults
@@ -53,6 +52,27 @@
 logger = logging.get_logger(__name__)
 
 
+class NemotronHRMSNormGated(torch.nn.Module):
+    def __init__(self, hidden_size, group_size, eps=1e-6):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+        self.group_size = group_size
+
+    def forward(self, hidden_states, gate=None):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        if gate is not None:
+            hidden_states = hidden_states * nn.functional.silu(gate.to(torch.float32))
+        *prefix_dims, last_dim = hidden_states.shape
+        group_count = last_dim // self.group_size
+        hidden_states_group = hidden_states.view(*prefix_dims, group_count, self.group_size)
+        variance = hidden_states_group.pow(2).mean(-1, keepdim=True)
+        hidden_states_group = hidden_states_group * torch.rsqrt(variance + self.variance_epsilon)
+        hidden_states = hidden_states_group.view(*prefix_dims, group_count * self.group_size)
+        return self.weight * hidden_states.to(input_dtype)
+
+
 # Helper methods for segment sum computation
 
 
@@ -418,12 +438,12 @@ def __init__(self, config: NemotronHConfig, layer_idx: int | None = None, initia
         # S4D real initialization. These are not discretized!
         # The core is to load them, compute the discrete states, then write the updated state. Keeps the memory bounded
         self.A_log = nn.Parameter(torch.empty(self.num_heads))
-        self.norm = Zamba2RMSNormGated(
+        self.norm = NemotronHRMSNormGated(
             self.intermediate_size, group_size=self.intermediate_size // self.n_groups, eps=config.layer_norm_epsilon
         )
         self.D = nn.Parameter(torch.empty(self.num_heads))
         if initialize_mixer_weights and self.dt_bias.device.type != "meta":
-            self.init_nemotron_h_mamba2_weights()
+            self.init_nemotron_h_weights()
         self.out_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.use_bias)
 
         self.layer_type = config.layer_types[layer_idx]
@@ -436,7 +456,7 @@ def __init__(self, config: NemotronHConfig, layer_idx: int | None = None, initia
         self.use_mem_eff_path = True
 
     @torch.no_grad()
-    def init_nemotron_h_mamba2_weights(self):
+    def init_nemotron_h_weights(self):
         A = torch.arange(1, self.num_heads + 1, device=self.A_log.device, dtype=torch.float32)
         init.copy_(self.A_log, torch.log(A))
         init.ones_(self.D)

src/transformers/models/nemotron_h/modular_nemotron_h.py

@@ -44,6 +44,10 @@
 logger = logging.get_logger(__name__)
 
 
+class NemotronHRMSNormGated(Zamba2RMSNormGated):
+    pass
+
+
 class NemotronHMamba2Mixer(Zamba2MambaMixer):
     def __init__(self, config: NemotronHConfig, layer_idx: int | None = None, initialize_mixer_weights: bool = True):
         super().__init__(config, layer_idx, initialize_mixer_weights)
@@ -74,7 +78,7 @@ def __init__(self, config: NemotronHConfig, layer_idx: int | None = None, initia
             projection_size,
             bias=config.use_bias,
         )
-        self.norm = Zamba2RMSNormGated(
+        self.norm = NemotronHRMSNormGated(
             self.intermediate_size, group_size=self.intermediate_size // self.n_groups, eps=config.layer_norm_epsilon
         )
         self.out_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.use_bias)

src/transformers/models/sam3_tracker/modular_sam3_tracker.py

@@ -20,8 +20,7 @@
 from ...configuration_utils import SubConfigSpec
 from ...modeling_utils import PreTrainedModel
 from ...utils import auto_docstring
-from ..auto import AutoModel
-from ..auto.configuration_auto import AutoConfig
+from ..auto import AutoConfig, AutoModel
 from ..sam2.configuration_sam2 import (
     Sam2Config,
     Sam2MaskDecoderConfig,

utils/modular_model_converter.py

@@ -26,6 +26,7 @@
 from create_dependency_mapping import find_priority_list
 from libcst import ClassDef, CSTVisitor
 from libcst import matchers as m
+from libcst.helpers import get_absolute_module_from_package_for_import
 from libcst.metadata import MetadataWrapper, ScopeProvider
 from modular_integrations import (
     EXCLUDED_EXTERNAL_FILES,
@@ -1221,7 +1222,30 @@ def find_file_type(class_name: str, model_name: str) -> str:
 )
 
 # These specific modeling imports should not be visited as other modeling files
-IMPORTS_TO_SKIP_IN_MODULAR = ("auto.modeling_auto",)
+MODEL_NAMES_TO_SKIP_IN_MODULAR = {"auto"}
+
+
+# Stands for the model of the file containing the import, e.g. the current model directory name
+# in `from .configuration_llama import LlamaConfig`
+CURRENT_MODEL = "__current_model__"
+
+
+def get_imported_model_name(node: cst.ImportFrom) -> str | None:
+    """Return the model directory `node` imports from, e.g. `llama` for `from ..llama.modeling_llama import X`,
+    `CURRENT_MODEL` for `from .configuration_llama import LlamaConfig`, or None if it does not import from a model."""
+    absolute_name = get_absolute_module_from_package_for_import(f"transformers.models.{CURRENT_MODEL}", node)
+    match = re.search(r"(?:^|\.)transformers\.models\.(\w+)", absolute_name or "")
+    return match.group(1) if match is not None else None
+
+
+def is_import_from_other_model(node: cst.CSTNode) -> bool:
+    """Whether `node` is an import statement from another model directory than the current model or MODEL_NAMES_TO_SKIP_IN_MODULAR."""
+    is_import_from = m.matches(node, m.SimpleStatementLine(body=[m.ImportFrom()]))
+    return is_import_from and get_imported_model_name(node.body[0]) not in {
+        None,
+        CURRENT_MODEL,
+        *MODEL_NAMES_TO_SKIP_IN_MODULAR,
+    }
 
 
 def append_new_import_node(
@@ -1480,9 +1504,16 @@ def visit_ImportFrom(self, node: cst.ImportFrom) -> None:
         # `node.module` is None for fully relative imports, e.g. `from ... import initialization as init`
         import_module = self.python_module.code_for_node(node.module) if node.module is not None else ""
         import_statement = "." * len(node.relative) + import_module
-        if any(import_to_skip in import_statement for import_to_skip in IMPORTS_TO_SKIP_IN_MODULAR):
+        imported_model_name = get_imported_model_name(node)
+        if imported_model_name in MODEL_NAMES_TO_SKIP_IN_MODULAR:
             return
-        if m.matches(node.module, m.Attribute()):
+        elif get_imported_model_name(node) == self.model_name:
+            # do not allow imports like `from transformers.models.current_model import ...` or `from ...models.current_model import ...`
+            raise ValueError(
+                f"`{self.python_module.code_for_node(node)}` imports from `{self.model_name}` itself. Import files from "
+                f"`{self.model_name}` with a single-dot relative import instead, e.g. `from .configuration_{self.model_name} import ...`."
+            )
+        elif m.matches(node.module, m.Attribute()):
             for imported_ in node.names:
                 # If we match here, it's an import from 3rd party lib that we need to skip
                 if any(external_file["name"] in import_statement for external_file in self.excluded_external_files):
@@ -1517,6 +1548,11 @@ def visit_ImportFrom(self, node: cst.ImportFrom) -> None:
                     f"You are importing from {import_module} directly using global imports. Import from the correct local path"
                 )
 
+    def visit_If(self, node):
+        # do not add imports from other models to the list of imports
+        statements = [stmt for stmt in node.body.body if not is_import_from_other_model(stmt)]
+        super().visit_If(node.with_changes(body=node.body.with_changes(body=statements)))
+
     def visit_SimpleStatementLine(self, node):
         """If we visit an import statement not previously visited, record it. If we visit a module-scope assignment,
         simply record it or, if it is `__all__`, split it between files where we should dispatch it.
@@ -1538,11 +1574,8 @@ def visit_SimpleStatementLine(self, node):
                     self.python_module.code_for_node(node.body[0].module) if node.body[0].module is not None else ""
                 )
                 import_statement = "." * len(node.body[0].relative) + import_module
-                if any(
+                if not is_import_from_other_model(node) or any(
                     external_file["name"] in import_statement for external_file in self.excluded_external_files
-                ) or not (
-                    re.search(rf"(?:transformers\.models\.)|(?:\.\.)\w+\.({self.match_patterns}).*", import_statement)
-                    and not any(import_to_skip in import_statement for import_to_skip in IMPORTS_TO_SKIP_IN_MODULAR)
                 ):
                     self.imports.append(node)
             elif m.matches(node, simple_top_level_assign_structure):
@@ -1617,6 +1650,11 @@ def leave_Module(self, node):
 
             self.imported_objects_per_file[file_type].update(mapper.objects_imported_from_modeling)
 
+        # Functions explicitly imported in the modular are inlined, even if the file of an inherited class imports them
+        # from a sibling file
+        for imported_objects in self.imported_objects_per_file.values():
+            imported_objects.difference_update(self.added_objects_file_mapping)
+
     def merge_model_specific_imports(self, visited_modules):
         """Merge the functions and assignments imported from the modeling files to the modular nodes and dependency graph,
         based on the visited files."""