Skip to content

Commit d80f943

Browse files
7878
authored
v1.6.6: Set MCP as default IoT Protocol (#690)
1 parent 0c83263 commit d80f943

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+523
-427
lines changed

‎CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
# CMakeLists in this exact order for cmake to work correctly
55
cmake_minimum_required(VERSION 3.16)
66

7-
set(PROJECT_VER "1.6.5")
7+
set(PROJECT_VER "1.6.6")
88

99
# Add this line to disable the specific warning
1010
add_compile_options(-Wno-missing-field-initializers)

‎main/Kconfig.projbuild‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -158,10 +158,10 @@ choice BOARD_TYPE
158158
bool "乐鑫ESP S3 LCD EV Board开发板"
159159
config BOARD_TYPE_ZHENGCHEN_1_54TFT_WIFI
160160
bool "征辰科技1.54(WIFI)"
161-
config BOARD_TYPE_MINSI_K08_DUAL
162-
bool "敏思科技K08(DUAL)"
163161
config BOARD_TYPE_ZHENGCHEN_1_54TFT_ML307
164162
bool "征辰科技1.54(ML307)"
163+
config BOARD_TYPE_MINSI_K08_DUAL
164+
bool "敏思科技K08(DUAL)"
165165
config BOARD_TYPE_ESP32_S3_1_54_MUMA
166166
bool "Spotpear ESP32-S3-1.54-MUMA"
167167
endchoice
@@ -284,7 +284,7 @@ config USE_SERVER_AEC
284284

285285
choice IOT_PROTOCOL
286286
prompt "IoT Protocol"
287-
default IOT_PROTOCOL_XIAOZHI
287+
default IOT_PROTOCOL_MCP
288288
help
289289
IoT 协议，用于获取设备状态与发送控制指令
290290
config IOT_PROTOCOL_MCP

‎main/application.cc‎

Lines changed: 44 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,14 @@ Application::Application() {
4444
event_group_ = xEventGroupCreate();
4545
background_task_ = new BackgroundTask(4096 * 7);
4646

47+
#if CONFIG_USE_DEVICE_AEC
48+
aec_mode_ = kAecOnDeviceSide;
49+
#elif CONFIG_USE_SERVER_AEC
50+
aec_mode_ = kAecOnServerSide;
51+
#else
52+
aec_mode_ = kAecOff;
53+
#endif
54+
4755
#if CONFIG_USE_AUDIO_PROCESSOR
4856
audio_processor_ = std::make_unique<AfeAudioProcessor>();
4957
#else
@@ -285,7 +293,7 @@ void Application::ToggleChatState() {
285293
return;
286294
}
287295

288-
SetListeningMode(realtime_chat_enabled_ ? kListeningModeRealtime : kListeningModeAutoStop);
296+
SetListeningMode(aec_mode_ == kAecOff ? kListeningModeAutoStop : kListeningModeRealtime);
289297
});
290298
} else if (device_state_ == kDeviceStateSpeaking) {
291299
Schedule([this]() {
@@ -358,8 +366,8 @@ void Application::Start() {
358366
auto codec = board.GetAudioCodec();
359367
opus_decoder_ = std::make_unique<OpusDecoderWrapper>(codec->output_sample_rate(), 1, OPUS_FRAME_DURATION_MS);
360368
opus_encoder_ = std::make_unique<OpusEncoderWrapper>(16000, 1, OPUS_FRAME_DURATION_MS);
361-
if (realtime_chat_enabled_) {
362-
ESP_LOGI(TAG, "Realtime chat enabled, setting opus encoder complexity to 0");
369+
if (aec_mode_ != kAecOff) {
370+
ESP_LOGI(TAG, "AEC mode: %d, setting opus encoder complexity to 0", aec_mode_);
363371
opus_encoder_->SetComplexity(0);
364372
} else if (board.GetBoardType() == "ml307") {
365373
ESP_LOGI(TAG, "ML307 board detected, setting opus encoder complexity to 5");
@@ -404,6 +412,11 @@ void Application::Start() {
404412
// Initialize the protocol
405413
display->SetStatus(Lang::Strings::LOADING_PROTOCOL);
406414

415+
// Add MCP common tools before initializing the protocol
416+
#if CONFIG_IOT_PROTOCOL_MCP
417+
McpServer::GetInstance().AddCommonTools();
418+
#endif
419+
407420
if (ota_.HasMqttConfig()) {
408421
protocol_ = std::make_unique<MqttProtocol>();
409422
} else if (ota_.HasWebsocketConfig()) {
@@ -608,7 +621,7 @@ void Application::Start() {
608621
// Set the chat state to wake word detected
609622
protocol_->SendWakeWordDetected(wake_word);
610623
ESP_LOGI(TAG, "Wake word detected: %s", wake_word.c_str());
611-
SetListeningMode(realtime_chat_enabled_ ? kListeningModeRealtime : kListeningModeAutoStop);
624+
SetListeningMode(aec_mode_ == kAecOff ? kListeningModeAutoStop : kListeningModeRealtime);
612625
} else if (device_state_ == kDeviceStateSpeaking) {
613626
AbortSpeaking(kAbortReasonWakeWordDetected);
614627
} else if (device_state_ == kDeviceStateActivating) {
@@ -1002,3 +1015,30 @@ void Application::SendMcpMessage(const std::string& payload) {
10021015
}
10031016
});
10041017
}
1018+
1019+
void Application::SetAecMode(AecMode mode) {
1020+
aec_mode_ = mode;
1021+
Schedule([this]() {
1022+
auto& board = Board::GetInstance();
1023+
auto display = board.GetDisplay();
1024+
switch (aec_mode_) {
1025+
case kAecOff:
1026+
audio_processor_->EnableDeviceAec(false);
1027+
display->ShowNotification(Lang::Strings::RTC_MODE_OFF);
1028+
break;
1029+
case kAecOnServerSide:
1030+
audio_processor_->EnableDeviceAec(false);
1031+
display->ShowNotification(Lang::Strings::RTC_MODE_ON);
1032+
break;
1033+
case kAecOnDeviceSide:
1034+
audio_processor_->EnableDeviceAec(true);
1035+
display->ShowNotification(Lang::Strings::RTC_MODE_ON);
1036+
break;
1037+
}
1038+
1039+
// If the AEC mode is changed, close the audio channel
1040+
if (protocol_ && protocol_->IsAudioChannelOpened()) {
1041+
protocol_->CloseAudioChannel();
1042+
}
1043+
});
1044+
}

‎main/application.h‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,12 @@
3030
#define SEND_AUDIO_EVENT (1 << 1)
3131
#define CHECK_NEW_VERSION_DONE_EVENT (1 << 2)
3232

33+
enum AecMode {
34+
kAecOff,
35+
kAecOnDeviceSide,
36+
kAecOnServerSide,
37+
};
38+
3339
enum DeviceState {
3440
kDeviceStateUnknown,
3541
kDeviceStateStarting,
@@ -73,6 +79,8 @@ class Application {
7379
void PlaySound(const std::string_view& sound);
7480
bool CanEnterSleepMode();
7581
void SendMcpMessage(const std::string& payload);
82+
void SetAecMode(AecMode mode);
83+
AecMode GetAecMode() const { return aec_mode_; }
7684

7785
private:
7886
Application();
@@ -90,11 +98,8 @@ class Application {
9098
esp_timer_handle_t clock_timer_handle_ = nullptr;
9199
volatile DeviceState device_state_ = kDeviceStateUnknown;
92100
ListeningMode listening_mode_ = kListeningModeAutoStop;
93-
#if CONFIG_USE_DEVICE_AEC || CONFIG_USE_SERVER_AEC
94-
bool realtime_chat_enabled_ = true;
95-
#else
96-
bool realtime_chat_enabled_ = false;
97-
#endif
101+
AecMode aec_mode_ = kAecOff;
102+
98103
bool aborted_ = false;
99104
bool voice_detected_ = false;
100105
bool busy_decoding_audio_ = false;

‎main/assets/en-US/language.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@
5151

5252
"VOLUME": "Volume ",
5353
"MUTED": "Muted",
54-
"MAX_VOLUME": "Max volume"
54+
"MAX_VOLUME": "Max volume",
55+
56+
"RTC_MODE_OFF": "AEC Off",
57+
"RTC_MODE_ON": "AEC On"
5558
}
5659
}

‎main/assets/ja-JP/language.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@
5050

5151
"VOLUME": "音量 ",
5252
"MUTED": "ミュートされています",
53-
"MAX_VOLUME": "最大音量"
53+
"MAX_VOLUME": "最大音量",
54+
55+
"RTC_MODE_OFF": "AEC 無効",
56+
"RTC_MODE_ON": "AEC 有効"
5457
}
5558
}

‎main/assets/zh-CN/language.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@
5050

5151
"VOLUME":"音量 ",
5252
"MUTED":"已静音",
53-
"MAX_VOLUME":"最大音量"
53+
"MAX_VOLUME":"最大音量",
54+
55+
"RTC_MODE_OFF":"AEC 关闭",
56+
"RTC_MODE_ON":"AEC 开启"
5457
}
5558
}

‎main/assets/zh-TW/language.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@
5050

5151
"VOLUME": "音量 ",
5252
"MUTED": "已靜音",
53-
"MAX_VOLUME": "最大音量"
53+
"MAX_VOLUME": "最大音量",
54+
55+
"RTC_MODE_OFF": "AEC 關閉",
56+
"RTC_MODE_ON": "AEC 開啟"
5457
}
5558
}

‎main/audio_processing/afe_audio_processor.cc‎

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,26 +26,25 @@ void AfeAudioProcessor::Initialize(AudioCodec* codec) {
2626
char* ns_model_name = esp_srmodel_filter(models, ESP_NSNET_PREFIX, NULL);
2727

2828
afe_config_t* afe_config = afe_config_init(input_format.c_str(), NULL, AFE_TYPE_VC, AFE_MODE_HIGH_PERF);
29-
#ifdef CONFIG_USE_DEVICE_AEC
30-
afe_config->aec_init = true;
3129
afe_config->aec_mode = AEC_MODE_VOIP_HIGH_PERF;
32-
#else
33-
afe_config->aec_init = false;
34-
#endif
30+
afe_config->vad_mode = VAD_MODE_0;
31+
afe_config->vad_min_noise_ms = 100;
3532
afe_config->ns_init = true;
3633
afe_config->ns_model_name = ns_model_name;
3734
afe_config->afe_ns_mode = AFE_NS_MODE_NET;
35+
36+
afe_config->afe_perferred_core = 1;
37+
afe_config->afe_perferred_priority = 1;
38+
afe_config->agc_init = false;
39+
afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM;
40+
3841
#ifdef CONFIG_USE_DEVICE_AEC
42+
afe_config->aec_init = true;
3943
afe_config->vad_init = false;
4044
#else
45+
afe_config->aec_init = false;
4146
afe_config->vad_init = true;
42-
afe_config->vad_mode = VAD_MODE_0;
43-
afe_config->vad_min_noise_ms = 100;
4447
#endif
45-
afe_config->afe_perferred_core = 1;
46-
afe_config->afe_perferred_priority = 1;
47-
afe_config->agc_init = false;
48-
afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM;
4948

5049
afe_iface_ = esp_afe_handle_from_config(afe_config);
5150
afe_data_ = afe_iface_->create_from_config(afe_config);
@@ -136,4 +135,18 @@ void AfeAudioProcessor::AudioProcessorTask() {
136135
output_callback_(std::vector<int16_t>(res->data, res->data + res->data_size / sizeof(int16_t)));
137136
}
138137
}
139-
}
138+
}
139+
140+
void AfeAudioProcessor::EnableDeviceAec(bool enable) {
141+
if (enable) {
142+
#if CONFIG_USE_DEVICE_AEC
143+
afe_iface_->disable_vad(afe_data_);
144+
afe_iface_->enable_aec(afe_data_);
145+
#else
146+
ESP_LOGE(TAG, "Device AEC is not supported");
147+
#endif
148+
} else {
149+
afe_iface_->disable_aec(afe_data_);
150+
afe_iface_->enable_vad(afe_data_);
151+
}
152+
}

‎main/audio_processing/afe_audio_processor.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ class AfeAudioProcessor : public AudioProcessor {
2626
void OnOutput(std::function<void(std::vector<int16_t>&& data)> callback) override;
2727
void OnVadStateChange(std::function<void(bool speaking)> callback) override;
2828
size_t GetFeedSize() override;
29+
void EnableDeviceAec(bool enable) override;
2930

3031
private:
3132
EventGroupHandle_t event_group_ = nullptr;

0 commit comments

Comments
 (0)