Feat/add litellm provider

custom_components/llmvision/config_flow.py

@@ -15,6 +15,7 @@
     LocalAI,
     Ollama,
     AWSBedrock,
+    LiteLLM,
 )
 from .const import (
     DOMAIN,
@@ -54,6 +55,7 @@
     DEFAULT_AWS_MODEL,
     DEFAULT_OPENWEBUI_MODEL,
     DEFAULT_OPENROUTER_MODEL,
+    DEFAULT_LITELLM_MODEL,
     ENDPOINT_OPENWEBUI,
     ENDPOINT_AZURE,
     ENDPOINT_OPENROUTER,
@@ -84,6 +86,7 @@ async def handle_provider(self, provider):
             "Groq": self.async_step_groq,
             "LocalAI": self.async_step_localai,
             "Ollama": self.async_step_ollama,
+            "LiteLLM": self.async_step_litellm,
             "OpenAI": self.async_step_openai,
             "OpenWebUI": self.async_step_openwebui,
             "OpenRouter": self.async_step_openrouter,
@@ -120,6 +123,7 @@ async def async_step_user(self, user_input=None):
                                 "Azure",
                                 "Google",
                                 "Groq",
+                                "LiteLLM",
                                 "LocalAI",
                                 "Ollama",
                                 "OpenAI",
@@ -1632,6 +1636,101 @@ async def async_step_openrouter(self, user_input=None):
             data_schema=data_schema,
         )
 
+    async def async_step_litellm(self, user_input=None):
+        data_schema = vol.Schema(
+            {
+                vol.Optional("connection_section"): section(
+                    vol.Schema(
+                        {
+                            vol.Required(CONF_API_KEY): selector(
+                                {"text": {"type": "password"}}
+                            ),
+                        }
+                    ),
+                    {"collapsed": False},
+                ),
+                vol.Optional("model_section"): section(
+                    vol.Schema(
+                        {
+                            vol.Required(
+                                CONF_DEFAULT_MODEL, default=DEFAULT_LITELLM_MODEL
+                            ): str,
+                            vol.Optional(CONF_TEMPERATURE, default=0.5): selector(
+                                {
+                                    "number": {
+                                        "min": 0,
+                                        "max": 1,
+                                        "step": 0.1,
+                                        "mode": "slider",
+                                    }
+                                }
+                            ),
+                            vol.Optional(CONF_TOP_P, default=0.9): selector(
+                                {
+                                    "number": {
+                                        "min": 0,
+                                        "max": 1,
+                                        "step": 0.1,
+                                        "mode": "slider",
+                                    }
+                                }
+                            ),
+                        }
+                    ),
+                    {"collapsed": False},
+                ),
+            }
+        )
+
+        if self.source == config_entries.SOURCE_RECONFIGURE:
+            self.init_info = self._get_reconfigure_entry().data
+            suggested = {
+                "connection_section": {
+                    CONF_API_KEY: self.init_info.get(CONF_API_KEY),
+                },
+                "model_section": {
+                    CONF_DEFAULT_MODEL: self.init_info.get(
+                        CONF_DEFAULT_MODEL, DEFAULT_LITELLM_MODEL
+                    ),
+                    CONF_TEMPERATURE: self.init_info.get(CONF_TEMPERATURE, 0.5),
+                    CONF_TOP_P: self.init_info.get(CONF_TOP_P, 0.9),
+                },
+            }
+            data_schema = self.add_suggested_values_to_schema(data_schema, suggested)
+
+        if user_input is not None:
+            user_input[CONF_PROVIDER] = self.init_info[CONF_PROVIDER]
+            user_input = flatten_dict(user_input)
+            try:
+                provider = LiteLLM(
+                    self.hass,
+                    api_key=user_input[CONF_API_KEY],
+                    model=user_input[CONF_DEFAULT_MODEL],
+                )
+                await provider.validate()
+                user_input[CONF_PROVIDER] = self.init_info[CONF_PROVIDER]
+                if self.source == config_entries.SOURCE_RECONFIGURE:
+                    return self.async_update_reload_and_abort(
+                        self._get_reconfigure_entry(),
+                        data_updates=user_input,
+                    )
+                else:
+                    return self.async_create_entry(
+                        title="LiteLLM", data=user_input
+                    )
+            except ServiceValidationError as e:
+                _LOGGER.error(f"Validation failed: {e}")
+                return self.async_show_form(
+                    step_id="litellm",
+                    data_schema=data_schema,
+                    errors={"base": "handshake_failed"},
+                )
+
+        return self.async_show_form(
+            step_id="litellm",
+            data_schema=data_schema,
+        )
+
     async def async_step_reconfigure(self, user_input):
         data = self._get_reconfigure_entry().data
         provider = data[CONF_PROVIDER]

custom_components/llmvision/const.py

@@ -146,6 +146,7 @@
 DEFAULT_AWS_MODEL = "us.amazon.nova-pro-v1:0"
 DEFAULT_OPENWEBUI_MODEL = "gemma3:4b"
 DEFAULT_OPENROUTER_MODEL = "google/gemma-3-4b-it:free"
+DEFAULT_LITELLM_MODEL = "gpt-4o-mini"
 
 DEFAULT_SUMMARY_PROMPT = "Provide a brief summary for the following titles. Focus on the key actions or changes that occurred over time and avoid unnecessary details or subjective interpretations. The summary should be concise, objective, and relevant to the content of the images. Keep the summary under 50 words and ensure it captures the main events or activities described in the descriptions. Here are the descriptions:\n "
 

custom_components/llmvision/manifest.json

@@ -8,6 +8,6 @@
     "integration_type": "service",
     "iot_class": "cloud_polling",
     "issue_tracker": "https://github.com/valentinfrlch/ha-llmvision/issues",
-    "requirements": ["boto3==1.37.1", "aiosqlite==0.21.0", "aiofile==3.9.0"],
+    "requirements": ["boto3==1.37.1", "aiosqlite==0.21.0", "aiofile==3.9.0", "litellm>=1.80,<1.88"],
     "version": "1.7.0"
 }

custom_components/llmvision/providers.py

@@ -50,6 +50,7 @@
     DEFAULT_AWS_MODEL,
     DEFAULT_OPENWEBUI_MODEL,
     DEFAULT_OPENROUTER_MODEL,
+    DEFAULT_LITELLM_MODEL,
     CONF_KEEP_ALIVE,
     CONF_CONTEXT_WINDOW,
     CONF_TEMPERATURE,
@@ -130,6 +131,7 @@ def get_default_model(self, provider):
             "AWS": DEFAULT_AWS_MODEL,  # For backwards compatibility
             "Open WebUI": DEFAULT_OPENWEBUI_MODEL,
             "OpenRouter": DEFAULT_OPENROUTER_MODEL,
+            "LiteLLM": DEFAULT_LITELLM_MODEL,
         }.get(provider_name)
 
     def validate(self, call: Any) -> None | ServiceValidationError:
@@ -2045,6 +2047,144 @@ def supports_structured_output(self) -> bool:
         return True
 
 
+class LiteLLM(Provider):
+
+    def __init__(self, hass: HomeAssistant, api_key: str, model: str):
+        super().__init__(hass, api_key, model)
+
+    def _build_litellm_kwargs(self, data: dict) -> dict:
+        kwargs = {
+            "model": data.get("model"),
+            "messages": data.get("messages"),
+            "drop_params": True,
+            "timeout": self.request_timeout,
+        }
+        for k in ("max_tokens", "temperature", "top_p"):
+            if k in data:
+                kwargs[k] = data[k]
+        if self.api_key:
+            kwargs["api_key"] = self.api_key
+        return kwargs
+
+    async def _make_request(self, data: dict) -> str:
+        import litellm
+        from litellm.exceptions import (
+            AuthenticationError,
+            BadRequestError,
+            ContextWindowExceededError,
+            NotFoundError,
+            RateLimitError,
+            Timeout,
+        )
+
+        kwargs = self._build_litellm_kwargs(data)
+
+        try:
+            response = await litellm.acompletion(**kwargs)
+        except AuthenticationError:
+            raise ServiceValidationError("invalid_api_key")
+        except NotFoundError:
+            raise ServiceValidationError(
+                f"Model '{self.model}' not found. Use LiteLLM format: openai/gpt-4o-mini, anthropic/claude-sonnet-4-6"
+            )
+        except ContextWindowExceededError:
+            raise ServiceValidationError("context_window_exceeded")
+        except RateLimitError:
+            raise ServiceValidationError("rate_limit")
+        except Timeout:
+            raise ServiceValidationError(
+                f"Request timed out after {self.request_timeout}s"
+            )
+        except BadRequestError as e:
+            raise ServiceValidationError(f"LiteLLM bad request: {e}")
+        except Exception as e:
+            raise ServiceValidationError(f"LiteLLM error: {e}")
+
+        choices = getattr(response, "choices", None)
+        if not choices:
+            raise ServiceValidationError("empty_response")
+        message = choices[0].message
+        if message.content is None:
+            raise ServiceValidationError("invalid_response")
+        return message.content
+
+    def _prepare_vision_data(self, call: Any) -> dict:
+        default_parameters = self._get_default_parameters(call)
+        payload = {
+            "model": self.model,
+            "messages": [{"role": "user", "content": []}],
+            "max_tokens": call.max_tokens,
+            "temperature": default_parameters.get("temperature"),
+            "top_p": default_parameters.get("top_p"),
+        }
+
+        for image, filename in zip(call.base64_images, call.filenames):
+            tag = (
+                ("Image " + str(call.base64_images.index(image) + 1))
+                if filename == ""
+                else filename
+            )
+            payload["messages"][0]["content"].append(
+                {"type": "text", "text": tag + ":"}
+            )
+            payload["messages"][0]["content"].append(
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/jpeg;base64,{image}"},
+                }
+            )
+
+        payload["messages"][0]["content"].append({"type": "text", "text": call.message})
+        system_prompt = self._get_system_prompt()
+        payload["messages"].insert(0, {"role": "system", "content": system_prompt})
+
+        if getattr(call, "use_memory", False):
+            memory_content = call.memory._get_memory_images(memory_type="OpenAI")
+            if memory_content:
+                payload["messages"].insert(
+                    1, {"role": "user", "content": memory_content}
+                )
+
+        return payload
+
+    def _prepare_text_data(self, call: Any) -> dict:
+        default_parameters = self._get_default_parameters(call)
+        title_prompt = self._get_title_prompt()
+        payload = {
+            "model": self.model,
+            "messages": [
+                {"role": "user", "content": [{"type": "text", "text": title_prompt}]},
+                {"role": "user", "content": [{"type": "text", "text": call.message}]},
+            ],
+            "max_tokens": call.max_tokens,
+            "temperature": default_parameters.get("temperature"),
+            "top_p": default_parameters.get("top_p"),
+        }
+        return payload
+
+    async def validate(self) -> None | ServiceValidationError:
+        import litellm
+        from litellm.exceptions import AuthenticationError, NotFoundError
+
+        try:
+            await litellm.acompletion(
+                model=self.model,
+                messages=[{"role": "user", "content": "Hi"}],
+                max_tokens=1,
+                drop_params=True,
+                timeout=self.request_timeout,
+                api_key=self.api_key if self.api_key else None,
+            )
+        except AuthenticationError:
+            raise ServiceValidationError("invalid_api_key")
+        except NotFoundError:
+            raise ServiceValidationError(
+                f"Model '{self.model}' not found. Use LiteLLM format: openai/gpt-4o-mini, anthropic/claude-sonnet-4-6"
+            )
+        except Exception as e:
+            raise ServiceValidationError(f"handshake_failed: {e}")
+
+
 class ProviderFactory:
     """
     Factory to create provider instances from a provider name and config
@@ -2160,4 +2300,11 @@ def create(
                 endpoint={"base_url": ENDPOINT_OPENROUTER},
             )
 
+        if provider_name == "LiteLLM":
+            return LiteLLM(
+                hass,
+                api_key=cast(str, config.get(CONF_API_KEY) or ""),
+                model=model,
+            )
+
         raise ServiceValidationError("invalid_provider")

custom_components/llmvision/strings.json

@@ -359,6 +359,36 @@
                     }
                 }
             },
+            "litellm": {
+                "title": "Configure LiteLLM",
+                "description": "LiteLLM provides access to 100+ LLM providers (OpenAI, Anthropic, Google, AWS Bedrock, Ollama, etc.) through a unified SDK. Use the provider-prefixed model format.",
+                "sections": {
+                    "connection_section": {
+                        "name": "Connection",
+                        "description": "API key for your LLM provider",
+                        "data": {
+                            "api_key": "API key"
+                        },
+                        "data_description": {
+                            "api_key": "Your LLM provider API key (e.g. OpenAI, Anthropic, Google). LiteLLM routes to the correct provider based on the model name."
+                        }
+                    },
+                    "model_section": {
+                        "name": "Model",
+                        "description": "Set default model parameters",
+                        "data": {
+                            "default_model": "Default model",
+                            "temperature": "Temperature",
+                            "top_p": "Top P"
+                        },
+                        "data_description": {
+                            "default_model": "Use the LiteLLM model format: openai/gpt-4o-mini, anthropic/claude-sonnet-4-6, gemini/gemini-2.0-flash, etc.",
+                            "temperature": "Controls the randomness of the output. Lower values make the output more deterministic.",
+                            "top_p": "Controls the diversity of the output. Lower values make the output more focused."
+                        }
+                    }
+                }
+            },
             "settings": {
                 "title": "Settings",
                 "description": "Configure the LLM Vision integration. This entry is required before setting up other providers. If you wish to use the default settings, just press 'Submit'.",

custom_components/llmvision/translations/en.json

@@ -359,6 +359,36 @@
                     }
                 }
             },
+            "litellm": {
+                "title": "Configure LiteLLM",
+                "description": "LiteLLM provides access to 100+ LLM providers (OpenAI, Anthropic, Google, AWS Bedrock, Ollama, etc.) through a unified SDK. Use the provider-prefixed model format.",
+                "sections": {
+                    "connection_section": {
+                        "name": "Connection",
+                        "description": "API key for your LLM provider",
+                        "data": {
+                            "api_key": "API key"
+                        },
+                        "data_description": {
+                            "api_key": "Your LLM provider API key (e.g. OpenAI, Anthropic, Google). LiteLLM routes to the correct provider based on the model name."
+                        }
+                    },
+                    "model_section": {
+                        "name": "Model",
+                        "description": "Set default model parameters",
+                        "data": {
+                            "default_model": "Default model",
+                            "temperature": "Temperature",
+                            "top_p": "Top P"
+                        },
+                        "data_description": {
+                            "default_model": "Use the LiteLLM model format: openai/gpt-4o-mini, anthropic/claude-sonnet-4-6, gemini/gemini-2.0-flash, etc.",
+                            "temperature": "Controls the randomness of the output. Lower values make the output more deterministic.",
+                            "top_p": "Controls the diversity of the output. Lower values make the output more focused."
+                        }
+                    }
+                }
+            },
             "settings": {
                 "title": "Settings",
                 "description": "Configure the LLM Vision integration. This entry is required before setting up other providers. If you wish to use the default settings, just press 'Submit'.",