start engine loop on server startup event

lmdeploy/pytorch/engine/engine.py

@@ -1125,3 +1125,10 @@ def create_instance(self, cuda_stream_id=0):
         """
         from .engine_instance import EngineInstance
         return EngineInstance(self)
+
+    def start_loop(self):
+        """start engine loop."""
+        if self.req_manager.is_loop_alive():
+            return True
+        self.req_manager.create_loop_task()
+        return True

lmdeploy/serve/async_engine.py

@@ -897,6 +897,13 @@ def _gen():
 
         return session
 
+    def start_loop(self):
+        """start engine loop."""
+        if hasattr(self.engine, 'start_loop'):
+            return self.engine.start_loop()
+        else:
+            return True
+
     """ DistServe Async Engine API Begin """
 
     def free_cache(self, session_id: int):

lmdeploy/serve/openai/api_server.py

@@ -1051,6 +1051,9 @@ def dummy_get_device_id():
 
 @router.on_event('startup')
 async def startup_event():
+    async_engine = VariableInterface.async_engine
+    async_engine.start_loop()
+
     if VariableInterface.proxy_url is None:
         return
     try: