Merge pull request #1689 from ScriptScientist/main

feat: novita llms support

Author: dartpain

application/api/answer/routes.py

@@ -42,6 +42,8 @@
     gpt_model = "claude-2"
 elif settings.LLM_NAME == "groq":
     gpt_model = "llama3-8b-8192"
+elif settings.LLM_NAME == "novita":
+    gpt_model = "deepseek/deepseek-r1"
 
 if settings.MODEL_NAME:  # in case there is particular model name configured
     gpt_model = settings.MODEL_NAME

application/llm/llm_creator.py

@@ -7,7 +7,7 @@
 from application.llm.docsgpt_provider import DocsGPTAPILLM
 from application.llm.premai import PremAILLM
 from application.llm.google_ai import GoogleLLM
-
+from application.llm.novita import NovitaLLM
 
 class LLMCreator:
     llms = {
@@ -20,7 +20,8 @@ class LLMCreator:
         "docsgpt": DocsGPTAPILLM,
         "premai": PremAILLM,
         "groq": GroqLLM,
-        "google": GoogleLLM
+        "google": GoogleLLM,
+        "novita": NovitaLLM
     }
 
     @classmethod

application/llm/novita.py

@@ -0,0 +1,32 @@
+from application.llm.base import BaseLLM
+from openai import OpenAI
+
+
+class NovitaLLM(BaseLLM):
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.client = OpenAI(api_key=api_key, base_url="https://api.novita.ai/v3/openai")
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+
+    def _raw_gen(self, baseself, model, messages, stream=False, tools=None, **kwargs):
+        if tools:
+            response = self.client.chat.completions.create(
+                model=model, messages=messages, stream=stream, tools=tools, **kwargs
+            )
+            return response.choices[0]
+        else:
+            response = self.client.chat.completions.create(
+                model=model, messages=messages, stream=stream, **kwargs
+            )
+            return response.choices[0].message.content
+
+    def _raw_gen_stream(
+        self, baseself, model, messages, stream=True, tools=None, **kwargs
+    ):
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
+        )
+        for line in response:
+            if line.choices[0].delta.content is not None:
+                yield line.choices[0].delta.content

setup.sh

@@ -148,6 +148,7 @@ prompt_cloud_api_provider_options() {
     echo -e "${YELLOW}4) Groq${NC}"
     echo -e "${YELLOW}5) HuggingFace Inference API${NC}"
     echo -e "${YELLOW}6) Azure OpenAI${NC}"
+    echo -e "${YELLOW}7) Novita${NC}"
     echo -e "${YELLOW}b) Back to Main Menu${NC}"
     echo
     read -p "$(echo -e "${DEFAULT_FG}Choose option (1-6, or b): ${NC}")" provider_choice
@@ -428,6 +429,12 @@ connect_cloud_api_provider() {
                 model_name="gpt-4o"
                 get_api_key
                 break ;;
+            7) # Novita
+                provider_name="Novita"
+                llm_name="novita"
+                model_name="deepseek/deepseek-r1"
+                get_api_key
+                break ;;
             b|B) clear; return ;; # Clear screen and Back to Main Menu
             *) echo -e "\n${RED}Invalid choice. Please choose 1-6, or b.${NC}" ; sleep 1 ;;
         esac