Update model.py

Author: Smartappli

llama_cpp/server/model.py

@@ -9,7 +9,7 @@
 
 
 class LlamaProxy:
-    def __init__(self, models: list[ModelSettings]) -> None:
+    def __init__(self, models: List[ModelSettings]) -> None:
         assert len(models) > 0, "No models provided!"
 
         self._model_settings_dict: dict[str, ModelSettings] = {}
@@ -18,19 +18,19 @@ def __init__(self, models: list[ModelSettings]) -> None:
                 model.model_alias = model.model
             self._model_settings_dict[model.model_alias] = model
 
-        self._current_model: llama_cpp.Llama | None = None
-        self._current_model_alias: str | None = None
+        self._current_model: Optional[llama_cpp.Llama] = None
+        self._current_model_alias: Optional[str] = None
 
         self._default_model_settings: ModelSettings = models[0]
         self._default_model_alias: str = self._default_model_settings.model_alias  # type: ignore
 
         # Load default model
         self._current_model = self.load_llama_from_model_settings(
-            self._default_model_settings,
+            self._default_model_settings
         )
         self._current_model_alias = self._default_model_alias
 
-    def __call__(self, model: str | None = None) -> llama_cpp.Llama:
+    def __call__(self, model: Optional[str] = None) -> llama_cpp.Llama:
         if model is None:
             model = self._default_model_alias
 
@@ -53,7 +53,7 @@ def __call__(self, model: str | None = None) -> llama_cpp.Llama:
     def __getitem__(self, model: str):
         return self._model_settings_dict[model].model_dump()
 
-    def __setitem__(self, model: str, settings: ModelSettings | str | bytes):
+    def __setitem__(self, model: str, settings: Union[ModelSettings, str, bytes]):
         if isinstance(settings, (bytes, str)):
             settings = ModelSettings.model_validate_json(settings)
         self._model_settings_dict[model] = settings
@@ -82,7 +82,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama:
                 )
             else:
                 chat_handler = llama_cpp.llama_chat_format.Llava15ChatHandler(
-                    clip_model_path=settings.clip_model_path, verbose=settings.verbose,
+                    clip_model_path=settings.clip_model_path, verbose=settings.verbose
                 )
         elif settings.chat_format == "obsidian":
             assert settings.clip_model_path is not None, "clip model not found"
@@ -96,7 +96,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama:
                 )
             else:
                 chat_handler = llama_cpp.llama_chat_format.ObsidianChatHandler(
-                    clip_model_path=settings.clip_model_path, verbose=settings.verbose,
+                    clip_model_path=settings.clip_model_path, verbose=settings.verbose
                 )
         elif settings.chat_format == "llava-1-6":
             assert settings.clip_model_path is not None, "clip model not found"
@@ -110,7 +110,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama:
                 )
             else:
                 chat_handler = llama_cpp.llama_chat_format.Llava16ChatHandler(
-                    clip_model_path=settings.clip_model_path, verbose=settings.verbose,
+                    clip_model_path=settings.clip_model_path, verbose=settings.verbose
                 )
         elif settings.chat_format == "moondream":
             assert settings.clip_model_path is not None, "clip model not found"
@@ -124,7 +124,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama:
                 )
             else:
                 chat_handler = llama_cpp.llama_chat_format.MoondreamChatHandler(
-                    clip_model_path=settings.clip_model_path, verbose=settings.verbose,
+                    clip_model_path=settings.clip_model_path, verbose=settings.verbose
                 )
         elif settings.chat_format == "nanollava":
             assert settings.clip_model_path is not None, "clip model not found"
@@ -138,7 +138,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama:
                 )
             else:
                 chat_handler = llama_cpp.llama_chat_format.NanoLlavaChatHandler(
-                    clip_model_path=settings.clip_model_path, verbose=settings.verbose,
+                    clip_model_path=settings.clip_model_path, verbose=settings.verbose
                 )
         elif settings.chat_format == "llama-3-vision-alpha":
             assert settings.clip_model_path is not None, "clip model not found"
@@ -152,38 +152,38 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama:
                 )
             else:
                 chat_handler = llama_cpp.llama_chat_format.Llama3VisionAlpha(
-                    clip_model_path=settings.clip_model_path, verbose=settings.verbose,
+                    clip_model_path=settings.clip_model_path, verbose=settings.verbose
                 )
         elif settings.chat_format == "hf-autotokenizer":
             assert (
                 settings.hf_pretrained_model_name_or_path is not None
             ), "hf_pretrained_model_name_or_path must be set for hf-autotokenizer"
             chat_handler = (
                 llama_cpp.llama_chat_format.hf_autotokenizer_to_chat_completion_handler(
-                    settings.hf_pretrained_model_name_or_path,
+                    settings.hf_pretrained_model_name_or_path
                 )
             )
         elif settings.chat_format == "hf-tokenizer-config":
             assert (
                 settings.hf_tokenizer_config_path is not None
             ), "hf_tokenizer_config_path must be set for hf-tokenizer-config"
             chat_handler = llama_cpp.llama_chat_format.hf_tokenizer_config_to_chat_completion_handler(
-                json.load(open(settings.hf_tokenizer_config_path)),
+                json.load(open(settings.hf_tokenizer_config_path))
             )
 
-        tokenizer: llama_cpp.BaseLlamaTokenizer | None = None
+        tokenizer: Optional[llama_cpp.BaseLlamaTokenizer] = None
         if settings.hf_pretrained_model_name_or_path is not None:
             tokenizer = llama_tokenizer.LlamaHFTokenizer.from_pretrained(
-                settings.hf_pretrained_model_name_or_path,
+                settings.hf_pretrained_model_name_or_path
             )
 
         draft_model = None
         if settings.draft_model is not None:
             draft_model = llama_speculative.LlamaPromptLookupDecoding(
-                num_pred_tokens=settings.draft_model_num_pred_tokens,
+                num_pred_tokens=settings.draft_model_num_pred_tokens
             )
 
-        kv_overrides: dict[str, bool | int | float | str] | None = None
+        kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None
         if settings.kv_overrides is not None:
             assert isinstance(settings.kv_overrides, list)
             kv_overrides = {}