enable lora modules

Author: yli1 user

matrix/app_server/deploy_utils.py

@@ -19,6 +19,7 @@
 from jinja2 import Template
 from ray import serve
 
+from matrix.app_server.llm.ray_serve_vllm import BaseDeployment
 from matrix.common.cluster_info import ClusterInfo
 from matrix.utils.ray import Action, get_ray_address, kill_matrix_actors
 
@@ -132,6 +133,19 @@
         "quantization": "compressed-tensors",
         "use_v1_engine": "true",
     },
+    "unsloth/mistral-7b-instruct-v0.2-bnb-4bit": {
+        "name": "unsloth-mistral-7B",
+        "tensor-parallel-size": 1,
+        "pipeline-parallel-size": 1,
+        "enable-prefix-caching": True,
+        "max_ongoing_requests": 256,
+        "max-model-len": 32768,
+        "gpu-memory-utilization": 0.4,
+        "enable-lora": True,
+        "quantization": "bitsandbytes",
+        "load-format": "bitsandbytes",
+        "max_lora_rank": 32,
+    },
 }
 
 non_model_params = [
@@ -372,14 +386,6 @@ def get_yaml_for_deployment(
                 yaml_str += "\n" + yaml.dump([found_app[0]], indent=2, sort_keys=False)
                 continue
 
-            unknown = {
-                k: v
-                for k, v in app.items()
-                if k not in non_model_params
-                and not hasattr(AsyncEngineArgs, k.replace("-", "_"))
-            }
-            assert not unknown, f"unknown vllm model args {unknown}"
-
             app_type = app.get("app_type", "llm")
             assert app_type in [
                 "llm",
@@ -397,6 +403,19 @@ def get_yaml_for_deployment(
             if "max_replica" not in app:
                 app["max_replica"] = app["min_replica"]
 
+            if app_type in ["llm", "sglang_llm"]:
+                unknown = {
+                    k: v
+                    for k, v in app.items()
+                    if k not in non_model_params
+                    and not hasattr(AsyncEngineArgs, k.replace("-", "_"))
+                    and not hasattr(BaseDeployment, k.replace("-", "_"))
+                }
+                assert not unknown, f"unknown vllm model args {unknown}"
+            else:
+                unknown = {k: v for k, v in app.items() if k not in non_model_params}
+                assert not unknown, f"unknown {app_type} model args {unknown}"
+
             if app_type in ["llm", "sglang_llm"]:
                 update_vllm_app_params(app)
                 yaml_str += Template(vllm_app_template).render(

matrix/app_server/llm/ray_serve_vllm.py

@@ -380,6 +380,13 @@ async def CreateChatCompletion(self, request):
         )
         logger.debug(f"Request: {chat}")
         try:
+            if (
+                self.openai_serving_chat.models.static_lora_modules
+                and len(self.openai_serving_chat.models.lora_requests) == 0
+            ):
+                # only need for lora modules, at vllm >= v0.7.0
+                # due to https://github.com/vllm-project/vllm/commit/ac2f3f7fee93cf9cd97c0078e362feab7b6c8299
+                await self.openai_serving_chat.models.init_static_loras()
             generator = await self.openai_serving_chat.create_chat_completion(chat)
             if isinstance(generator, ErrorResponse):
                 status_code = self.http_to_grpc_status(generator.code)
@@ -417,6 +424,13 @@ async def CreateCompletion(self, request):
         )
         logger.debug(f"Request: {completion_request}")
         try:
+            if (
+                self.openai_serving_chat.models.static_lora_modules
+                and len(self.openai_serving_chat.models.lora_requests) == 0
+            ):
+                # only need for lora modules, at vllm >= v0.7.0
+                # due to https://github.com/vllm-project/vllm/commit/ac2f3f7fee93cf9cd97c0078e362feab7b6c8299
+                await self.openai_serving_chat.models.init_static_loras()
             generator = await self.openai_serving_completion.create_completion(
                 completion_request,
                 Request(  # this Request is purely dummy, it is changed to optional in vllm's recent pull https://github.com/vllm-project/vllm/pull/12503