feat: sensecap watcher add inference (#1312)

* feat: Wake up when a person is detected

* fix: Solve the problem of no sound when using WakeWordInvoke

* fix: Solve the problem of triggering dialogue when the person has not left

* feat(vision): 优化视觉检测逻辑并增加配置接口

本次提交旨在优化视觉检测功能,使其行为更自然、更智能,并为用户提供灵活的配置选项。

主要更新包括:

1. 引入了更精细的检测状态机:

   - IDLE: 空闲状态,等待检测目标。

   - VALIDATING: 验证状态,在检测到目标后,持续一段时间(可配置)以确认其存在,防止误触发。

   - COOLDOWN: 冷却状态,在一次成功交互后进入,避免过于频繁的打扰。

2. 新增了用于配置视觉检测的 MCP 工具:

   - self.vision.get_detection_config: 获取当前的检测参数(阈值、冷却间隔、验证时长、目标类型)。

   - self.vision.set_detection_config: 允许用户动态修改这些参数,以适应不同场景。

3. 性能优化:

   - 增加了配置参数的内存缓存,避免了在检测循环中对 NVS 的频繁访问。

* feat: Inference using Model 4

* feat: default inference disable

* feat: version cmd change to output json

* fix: fix image display

* Fix include directives for esp_check and esp_app_desc

---------

Co-authored-by: Spencer <[email protected]>
Co-authored-by: Xiaoxia <[email protected]>

Author: 3 people

main/application.cc

@@ -788,13 +788,35 @@ bool Application::UpgradeFirmware(Ota& ota, const std::string& url) {
 }
 
 void Application::WakeWordInvoke(const std::string& wake_word) {
+    if (!protocol_) {
+        return;
+    }
+
     if (device_state_ == kDeviceStateIdle) {
-        ToggleChatState();
-        Schedule([this, wake_word]() {
-            if (protocol_) {
-                protocol_->SendWakeWordDetected(wake_word); 
+        audio_service_.EncodeWakeWord();
+
+        if (!protocol_->IsAudioChannelOpened()) {
+            SetDeviceState(kDeviceStateConnecting);
+            if (!protocol_->OpenAudioChannel()) {
+                audio_service_.EnableWakeWordDetection(true);
+                return;
             }
-        }); 
+        }
+
+        ESP_LOGI(TAG, "Wake word detected: %s", wake_word.c_str());
+#if CONFIG_USE_AFE_WAKE_WORD || CONFIG_USE_CUSTOM_WAKE_WORD
+        // Encode and send the wake word data to the server
+        while (auto packet = audio_service_.PopWakeWordPacket()) {
+            protocol_->SendAudio(std::move(packet));
+        }
+        // Set the chat state to wake word detected
+        protocol_->SendWakeWordDetected(wake_word);
+        SetListeningMode(aec_mode_ == kAecOff ? kListeningModeAutoStop : kListeningModeRealtime);
+#else
+        SetListeningMode(aec_mode_ == kAecOff ? kListeningModeAutoStop : kListeningModeRealtime);
+        // Play the pop up sound to indicate the wake word is detected
+        audio_service_.PlaySound(Lang::Sounds::OGG_POPUP);
+#endif
     } else if (device_state_ == kDeviceStateSpeaking) {
         Schedule([this]() {
             AbortSpeaking(kAbortReasonNone);

main/boards/sensecap-watcher/sensecap_watcher.cc

@@ -12,7 +12,7 @@
 #include "lvgl_theme.h"
 
 #include <esp_log.h>
-#include "esp_check.h"
+#include <esp_check.h>
 #include <esp_lcd_panel_io.h>
 #include <esp_lcd_panel_ops.h>
 #include <esp_lcd_spd2010.h>
@@ -28,6 +28,7 @@
 #include <esp_console.h>
 #include <esp_mac.h>
 #include <nvs_flash.h>
+#include <esp_app_desc.h>
 
 #include "assets/lang_config.h"
 
@@ -492,6 +493,47 @@ class SensecapWatcher : public WifiBoard {
         };
         ESP_ERROR_CHECK(esp_console_cmd_register(&cmd5));
 
+        const esp_console_cmd_t cmd6 = {
+            .command = "version",
+            .help = "Read version info",
+            .hint = NULL,
+            .func = NULL,
+            .argtable = NULL,
+            .func_w_context = [](void *context,int argc, char** argv) -> int {
+                auto self = static_cast<SensecapWatcher*>(context);
+                auto app_desc = esp_app_get_description();
+                const char* region = "UNKNOWN";
+                #if defined(CONFIG_LANGUAGE_ZH_CN)
+                    region = "CN";
+                #elif defined(CONFIG_LANGUAGE_EN_US)
+                    region = "US";
+                #elif defined(CONFIG_LANGUAGE_JA_JP)
+                    region = "JP";
+                #elif defined(CONFIG_LANGUAGE_ES_ES)
+                    region = "ES";
+                #elif defined(CONFIG_LANGUAGE_DE_DE)
+                    region = "DE";
+                #elif defined(CONFIG_LANGUAGE_FR_FR)
+                    region = "FR";
+                #elif defined(CONFIG_LANGUAGE_IT_IT)
+                    region = "IT";
+                #elif defined(CONFIG_LANGUAGE_PT_PT)
+                    region = "PT";
+                #elif defined(CONFIG_LANGUAGE_RU_RU)
+                    region = "RU";
+                #elif defined(CONFIG_LANGUAGE_KO_KR)
+                    region = "KR";
+                #endif
+                printf("{\"type\":0,\"name\":\"VER?\",\"code\":0,\"data\":{\"software\":\"%s\",\"hardware\":\"watcher xiaozhi agent\",\"camera\":%d,\"region\":\"%s\"}}\n",
+                       app_desc->version,
+                       self->GetCamera() == nullptr ? 0 : 1,
+                       region);
+                return 0;
+            },
+            .context =this
+        };
+        ESP_ERROR_CHECK(esp_console_cmd_register(&cmd6));
+
         esp_console_dev_uart_config_t hw_config = ESP_CONSOLE_DEV_UART_CONFIG_DEFAULT();
         ESP_ERROR_CHECK(esp_console_new_repl_uart(&hw_config, &repl_config, &repl));
         ESP_ERROR_CHECK(esp_console_start_repl(repl));