diff --git a/vllm/model_executor/models/transformers/base.py b/vllm/model_executor/models/transformers/base.py
index c964cb721d65..f8f1ead2c492 100644
--- a/vllm/model_executor/models/transformers/base.py
+++ b/vllm/model_executor/models/transformers/base.py
@@ -591,6 +591,8 @@ def init_parameters(self, module: nn.Module, dtype: torch.dtype | None = None):
         def _init_parameters(module: nn.Module, dtype: torch.dtype | None):
             for name, param in module.named_parameters(recurse=False):
                 if param.device == torch.device("meta"):
+                    if hasattr(param, "weight_loader"):
+                        continue
                     new_param = nn.Parameter(
                         torch.empty_like(
                             param.data,