Merge pull request #137 from jekalmin/v1.0.2

1.0.2

Author: jekalmin

README.md

@@ -16,7 +16,7 @@ Extended OpenAI Conversation uses OpenAI API's feature of [function calling](htt
 Since "gpt-3.5-turbo" model already knows how to call service of Home Assistant in general, you just have to let model know what devices you have by [exposing entities](https://github.com/jekalmin/extended_openai_conversation#preparation)
 
 ## Installation
-1. Install via HACS or by copying `extended_openai_conversation` folder into `<config directory>/custom_components`
+1. Install via registering as a custom repository of HACS or by copying `extended_openai_conversation` folder into `<config directory>/custom_components`
 2. Restart Home Assistant
 3. Go to Settings > Devices & Services.
 4. In the bottom right corner, select the Add Integration button.

custom_components/extended_openai_conversation/__init__.py

@@ -46,6 +46,9 @@
     CONF_BASE_URL,
     CONF_API_VERSION,
     CONF_SKIP_AUTHENTICATION,
+    CONF_USE_TOOLS,
+    CONF_CONTEXT_THRESHOLD,
+    CONF_CONTEXT_TRUNCATE_STRATEGY,
     DEFAULT_ATTACH_USERNAME,
     DEFAULT_CHAT_MODEL,
     DEFAULT_MAX_TOKENS,
@@ -55,6 +58,9 @@
     DEFAULT_MAX_FUNCTION_CALLS_PER_CONVERSATION,
     DEFAULT_CONF_FUNCTIONS,
     DEFAULT_SKIP_AUTHENTICATION,
+    DEFAULT_USE_TOOLS,
+    DEFAULT_CONTEXT_THRESHOLD,
+    DEFAULT_CONTEXT_TRUNCATE_STRATEGY,
     DOMAIN,
 )
 
@@ -153,7 +159,6 @@ def supported_languages(self) -> list[str] | Literal["*"]:
     async def async_process(
         self, user_input: conversation.ConversationInput
     ) -> conversation.ConversationResult:
-        raw_prompt = self.entry.options.get(CONF_PROMPT, DEFAULT_PROMPT)
         exposed_entities = self.get_exposed_entities()
 
         if user_input.conversation_id in self.history:
@@ -163,7 +168,9 @@ async def async_process(
             conversation_id = ulid.ulid()
             user_input.conversation_id = conversation_id
             try:
-                prompt = self._async_generate_prompt(raw_prompt, exposed_entities)
+                system_message = self._generate_system_message(
+                    exposed_entities, user_input
+                )
             except TemplateError as err:
                 _LOGGER.error("Error rendering prompt: %s", err)
                 intent_response = intent.IntentResponse(language=user_input.language)
@@ -174,7 +181,7 @@ async def async_process(
                 return conversation.ConversationResult(
                     response=intent_response, conversation_id=conversation_id
                 )
-            messages = [{"role": "system", "content": prompt}]
+            messages = [system_message]
         user_message = {"role": "user", "content": user_input.text}
         if self.entry.options.get(CONF_ATTACH_USERNAME, DEFAULT_ATTACH_USERNAME):
             user = await self.hass.auth.async_get_user(user_input.context.user_id)
@@ -215,12 +222,25 @@ async def async_process(
             response=intent_response, conversation_id=conversation_id
         )
 
-    def _async_generate_prompt(self, raw_prompt: str, exposed_entities) -> str:
+    def _generate_system_message(
+        self, exposed_entities, user_input: conversation.ConversationInput
+    ):
+        raw_prompt = self.entry.options.get(CONF_PROMPT, DEFAULT_PROMPT)
+        prompt = self._async_generate_prompt(raw_prompt, exposed_entities, user_input)
+        return {"role": "system", "content": prompt}
+
+    def _async_generate_prompt(
+        self,
+        raw_prompt: str,
+        exposed_entities,
+        user_input: conversation.ConversationInput,
+    ) -> str:
         """Generate a prompt for the user."""
         return template.Template(raw_prompt, self.hass).async_render(
             {
                 "ha_name": self.hass.config.location_name,
                 "exposed_entities": exposed_entities,
+                "current_device_id": user_input.device_id,
             },
             parse_result=False,
         )
@@ -269,6 +289,28 @@ def get_functions(self):
         except:
             raise FunctionLoadFailed()
 
+    async def truncate_message_history(
+        self, messages, exposed_entities, user_input: conversation.ConversationInput
+    ):
+        """Truncate message history."""
+        strategy = self.entry.options.get(
+            CONF_CONTEXT_TRUNCATE_STRATEGY, DEFAULT_CONTEXT_TRUNCATE_STRATEGY
+        )
+
+        if strategy == "clear":
+            last_user_message_index = None
+            for i in reversed(range(len(messages))):
+                if messages[i]["role"] == "user":
+                    last_user_message_index = i
+                    break
+
+            if last_user_message_index is not None:
+                del messages[1:last_user_message_index]
+                # refresh system prompt when all messages are deleted
+                messages[0] = self._generate_system_message(
+                    exposed_entities, user_input
+                )
+
     async def query(
         self,
         user_input: conversation.ConversationInput,
@@ -281,16 +323,27 @@ async def query(
         max_tokens = self.entry.options.get(CONF_MAX_TOKENS, DEFAULT_MAX_TOKENS)
         top_p = self.entry.options.get(CONF_TOP_P, DEFAULT_TOP_P)
         temperature = self.entry.options.get(CONF_TEMPERATURE, DEFAULT_TEMPERATURE)
+        use_tools = self.entry.options.get(CONF_USE_TOOLS, DEFAULT_USE_TOOLS)
+        context_threshold = self.entry.options.get(
+            CONF_CONTEXT_THRESHOLD, DEFAULT_CONTEXT_THRESHOLD
+        )
         functions = list(map(lambda s: s["spec"], self.get_functions()))
         function_call = "auto"
         if n_requests == self.entry.options.get(
             CONF_MAX_FUNCTION_CALLS_PER_CONVERSATION,
             DEFAULT_MAX_FUNCTION_CALLS_PER_CONVERSATION,
         ):
             function_call = "none"
+
+        tool_kwargs = {"functions": functions, "function_call": function_call}
+        if use_tools:
+            tool_kwargs = {
+                "tools": [{"type": "function", "function": func} for func in functions],
+                "tool_choice": function_call,
+            }
+
         if len(functions) == 0:
-            functions = None
-            function_call = None
+            tool_kwargs = {}
 
         _LOGGER.info("Prompt for %s: %s", model, messages)
 
@@ -301,20 +354,28 @@ async def query(
             top_p=top_p,
             temperature=temperature,
             user=user_input.conversation_id,
-            functions=functions,
-            function_call=function_call,
+            **tool_kwargs,
         )
 
         _LOGGER.info("Response %s", response.model_dump(exclude_none=True))
+
+        if response.usage.total_tokens > context_threshold:
+            await self.truncate_message_history(messages, exposed_entities, user_input)
+
         choice: Choice = response.choices[0]
         message = choice.message
+
         if choice.finish_reason == "function_call":
             message = await self.execute_function_call(
                 user_input, messages, message, exposed_entities, n_requests + 1
             )
+        if choice.finish_reason == "tool_calls":
+            message = await self.execute_tool_calls(
+                user_input, messages, message, exposed_entities, n_requests + 1
+            )
         return message
 
-    def execute_function_call(
+    async def execute_function_call(
         self,
         user_input: conversation.ConversationInput,
         messages,
@@ -328,7 +389,7 @@ def execute_function_call(
             None,
         )
         if function is not None:
-            return self.execute_function(
+            return await self.execute_function(
                 user_input,
                 messages,
                 message,
@@ -366,3 +427,57 @@ async def execute_function(
             }
         )
         return await self.query(user_input, messages, exposed_entities, n_requests)
+
+    async def execute_tool_calls(
+        self,
+        user_input: conversation.ConversationInput,
+        messages,
+        message: ChatCompletionMessage,
+        exposed_entities,
+        n_requests,
+    ):
+        messages.append(message.model_dump(exclude_none=True))
+        for tool in message.tool_calls:
+            function_name = tool.function.name
+            function = next(
+                (s for s in self.get_functions() if s["spec"]["name"] == function_name),
+                None,
+            )
+            if function is not None:
+                result = await self.execute_tool_function(
+                    user_input,
+                    tool,
+                    exposed_entities,
+                    function,
+                )
+
+                messages.append(
+                    {
+                        "tool_call_id": tool.id,
+                        "role": "tool",
+                        "name": function_name,
+                        "content": str(result),
+                    }
+                )
+            else:
+                raise FunctionNotFound(function_name)
+        return await self.query(user_input, messages, exposed_entities, n_requests)
+
+    async def execute_tool_function(
+        self,
+        user_input: conversation.ConversationInput,
+        tool,
+        exposed_entities,
+        function,
+    ):
+        function_executor = get_function_executor(function["function"]["type"])
+
+        try:
+            arguments = json.loads(tool.function.arguments)
+        except json.decoder.JSONDecodeError as err:
+            raise ParseArgumentsFailed(tool.function.arguments) from err
+
+        result = await function_executor.execute(
+            self.hass, function["function"], arguments, user_input, exposed_entities
+        )
+        return result

custom_components/extended_openai_conversation/config_flow.py

@@ -19,7 +19,6 @@
     NumberSelector,
     NumberSelectorConfig,
     TemplateSelector,
-    AttributeSelector,
     SelectSelector,
     SelectSelectorConfig,
     SelectOptionDict,
@@ -40,6 +39,9 @@
     CONF_BASE_URL,
     CONF_API_VERSION,
     CONF_SKIP_AUTHENTICATION,
+    CONF_USE_TOOLS,
+    CONF_CONTEXT_THRESHOLD,
+    CONF_CONTEXT_TRUNCATE_STRATEGY,
     DEFAULT_ATTACH_USERNAME,
     DEFAULT_CHAT_MODEL,
     DEFAULT_MAX_TOKENS,
@@ -50,6 +52,10 @@
     DEFAULT_CONF_FUNCTIONS,
     DEFAULT_CONF_BASE_URL,
     DEFAULT_SKIP_AUTHENTICATION,
+    DEFAULT_USE_TOOLS,
+    DEFAULT_CONTEXT_THRESHOLD,
+    DEFAULT_CONTEXT_TRUNCATE_STRATEGY,
+    CONTEXT_TRUNCATE_STRATEGIES,
     DOMAIN,
     DEFAULT_NAME,
 )
@@ -80,6 +86,9 @@
         CONF_TEMPERATURE: DEFAULT_TEMPERATURE,
         CONF_FUNCTIONS: DEFAULT_CONF_FUNCTIONS_STR,
         CONF_ATTACH_USERNAME: DEFAULT_ATTACH_USERNAME,
+        CONF_USE_TOOLS: DEFAULT_USE_TOOLS,
+        CONF_CONTEXT_THRESHOLD: DEFAULT_CONTEXT_THRESHOLD,
+        CONF_CONTEXT_TRUNCATE_STRATEGY: DEFAULT_CONTEXT_TRUNCATE_STRATEGY,
     }
 )
 
@@ -222,4 +231,29 @@ def openai_config_option_schema(self, options: MappingProxyType[str, Any]) -> di
                 description={"suggested_value": options.get(CONF_ATTACH_USERNAME)},
                 default=DEFAULT_ATTACH_USERNAME,
             ): BooleanSelector(),
+            vol.Optional(
+                CONF_USE_TOOLS,
+                description={"suggested_value": options.get(CONF_USE_TOOLS)},
+                default=DEFAULT_USE_TOOLS,
+            ): BooleanSelector(),
+            vol.Optional(
+                CONF_CONTEXT_THRESHOLD,
+                description={"suggested_value": options.get(CONF_CONTEXT_THRESHOLD)},
+                default=DEFAULT_CONTEXT_THRESHOLD,
+            ): int,
+            vol.Optional(
+                CONF_CONTEXT_TRUNCATE_STRATEGY,
+                description={
+                    "suggested_value": options.get(CONF_CONTEXT_TRUNCATE_STRATEGY)
+                },
+                default=DEFAULT_CONTEXT_TRUNCATE_STRATEGY,
+            ): SelectSelector(
+                SelectSelectorConfig(
+                    options=[
+                        SelectOptionDict(value=strategy["key"], label=strategy["label"])
+                        for strategy in CONTEXT_TRUNCATE_STRATEGIES
+                    ],
+                    mode=SelectSelectorMode.DROPDOWN,
+                )
+            ),
         }

custom_components/extended_openai_conversation/const.py

@@ -84,5 +84,12 @@
 ]
 CONF_ATTACH_USERNAME = "attach_username"
 DEFAULT_ATTACH_USERNAME = False
+CONF_USE_TOOLS = "use_tools"
+DEFAULT_USE_TOOLS = False
+CONF_CONTEXT_THRESHOLD = "context_threshold"
+DEFAULT_CONTEXT_THRESHOLD = 13000
+CONTEXT_TRUNCATE_STRATEGIES = [{"key": "clear", "label": "Clear All Messages"}]
+CONF_CONTEXT_TRUNCATE_STRATEGY = "context_truncate_strategy"
+DEFAULT_CONTEXT_TRUNCATE_STRATEGY = CONTEXT_TRUNCATE_STRATEGIES[0]["key"]
 
 SERVICE_QUERY_IMAGE = "query_image"