Merge branch 'feature/add_audio_render' into 'main'

feat(esp_audio_render): Add initial version of esp_audio_render

See merge request adf/multimedia/esp-gmf!121

Author: jason-mao

.github/workflows/upload_component.yml

@@ -25,6 +25,7 @@ jobs:
             esp_capture: ./packages/esp_capture
             gmf_app_utils: ./packages/gmf_app_utils
             esp_board_manager: ./packages/esp_board_manager
+            esp_audio_render: ./packages/esp_audio_render
             gmf_examples: ./gmf_examples
           namespace: "espressif"
           api_token: ${{ secrets.IDF_COMPONENT_API_TOKEN }}

README.md

@@ -45,6 +45,7 @@ Advanced components are encapsulation modules in ESP-GMF that target specific ap
 | [gmf_app_utils](./packages/gmf_app_utils) | Common peripheral configuration, unit <br>testing tools Memory leak detection tools | - `gmf_core`<br>- `protocol_examples_common`<br>- `codec_board`|
 | [esp_capture](./packages/esp_capture) | High-level multimedia capture module | - `gmf_core`<br>- `gmf-audio`<br>- `gmf-video`<br>- `esp_muxer`<br>- `esp_codec_dev`<br>- `esp-sr`<br>- `esp_video`<br>- `esp32-camera`|
 | [esp_board_manager](./packages/esp_board_manager) | Intelligent, automated board configuration and management utilities with YAML-based setup | Dependencies vary based on selected board |
+| [esp_audio_render](./packages/esp_audio_render) | Audio render module with mixer support | - `gmf_core`<br>- `gmf-audio`<br>|
 
 
 # ESP-GMF Usage Guide

README_CN.md

@@ -45,6 +45,7 @@ ESP-GMF 各个模块以组件的形式存在，组件又按功能分为**原子
 |  [gmf_app_utils](./packages/gmf_app_utils) | 常用外设配置，单元测试工具<br>内存泄漏检测工具 | - `gmf_core`<br>- `protocol_examples_common`<br>- `codec_board` |
 |  [esp_capture](./packages/esp_capture) | 易用的音视频采集器 | - `gmf_core`<br>- `gmf-audio`<br>- `gmf-video`<br>- `esp_muxer`<br>- `esp_codec_dev`<br>- `esp-sr`<br>- `esp_video`<br>- `esp32-camera`|
 |  [esp_board_manager](./packages/esp_board_manager) | 智能、自动化的板子配置和管理工具，支持基于 YAML 的设置 | 根据选择的板子变化依赖 |
+| [esp_audio_render](./packages/esp_audio_render) | 支持混音的音频渲染器 | - `gmf_core`<br>- `gmf-audio`<br>|
 
 # ESP-GMF 使用说明
 

elements/gmf_audio/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## v0.7.1
+
+### Bug Fixes
+- Fixed sonic bypass failed for first payload
+
+### Features
+
+- Added audio param setting for `aud_alc`, `aud_fade`, `aud_sonic`
+
 ## v0.7.0
 
 ### Features

elements/gmf_audio/esp_gmf_alc.c

@@ -44,7 +44,19 @@ static esp_gmf_err_t __alc_set_gain(esp_gmf_element_handle_t handle, esp_gmf_arg
     uint8_t idx = (uint8_t)(*buf);
     alc_desc = alc_desc->next;
     int8_t gain = (int8_t)(*(buf + alc_desc->offset));
-    return esp_gmf_alc_set_gain(handle, idx, gain);
+    esp_gmf_err_t ret = ESP_GMF_ERR_OK;
+    if (idx == 0xFF) {
+        // Apply gain to all channels
+        esp_ae_alc_cfg_t *config = (esp_ae_alc_cfg_t *)OBJ_GET_CFG(handle);
+        if (config) {
+            for (uint8_t i = 0; i < config->channel; i++) {
+                ret |= esp_gmf_alc_set_gain(handle, i, gain);
+            }
+        }
+    } else {
+        ret = esp_gmf_alc_set_gain(handle, idx, gain);
+    }
+    return ret;
 }
 
 static esp_gmf_err_t __alc_get_gain(esp_gmf_element_handle_t handle, esp_gmf_args_desc_t *arg_desc,

elements/gmf_audio/esp_gmf_audio_param.c

@@ -54,3 +54,49 @@ esp_gmf_err_t esp_gmf_audio_param_set_dest_ch(esp_gmf_element_handle_t self, uin
 
     RELEASE_AMETHOD_SETTING();
 }
+
+esp_gmf_err_t esp_gmf_audio_param_set_speed(esp_gmf_element_handle_t self, float speed)
+{
+    const char *method_name = AMETHOD(SONIC, SET_SPEED);
+    PREPARE_AMETHOD_SETTING();
+
+    SET_AMETHOD_ARG(AMETHOD_ARG(SONIC, SET_SPEED, SPEED), speed);
+
+    RELEASE_AMETHOD_SETTING();
+}
+
+esp_gmf_err_t esp_gmf_audio_param_set_pitch(esp_gmf_element_handle_t self, float pitch)
+{
+    const char *method_name = AMETHOD(SONIC, SET_PITCH);
+    PREPARE_AMETHOD_SETTING();
+
+    SET_AMETHOD_ARG(AMETHOD_ARG(SONIC, SET_PITCH, PITCH), pitch);
+
+    RELEASE_AMETHOD_SETTING();
+}
+
+esp_gmf_err_t esp_gmf_audio_param_set_alc_channel_gain(esp_gmf_element_handle_t self, uint8_t ch_idx, float db)
+{
+    const char *method_name = AMETHOD(ALC, SET_GAIN);
+    // ALC currently only support int8 db
+    int8_t gain = (int8_t) db;
+    PREPARE_AMETHOD_SETTING();
+
+    SET_AMETHOD_ARG(AMETHOD_ARG(ALC, SET_GAIN, IDX), ch_idx);
+    SET_AMETHOD_ARG(AMETHOD_ARG(ALC, SET_GAIN, GAIN), gain);
+
+    RELEASE_AMETHOD_SETTING();
+}
+
+esp_gmf_err_t esp_gmf_audio_param_set_fade(esp_gmf_element_handle_t self, bool is_fade_in)
+{
+    const char *method_name = AMETHOD(FADE, SET_MODE);
+    // Aligned with ESP_AE_FADE_MODE_FADE_IN  = 1, ESP_AE_FADE_MODE_FADE_OUT = 2
+    int32_t mode = is_fade_in ? 1 : 2;
+
+    PREPARE_AMETHOD_SETTING();
+
+    SET_AMETHOD_ARG(AMETHOD_ARG(FADE, SET_MODE, MODE), mode);
+
+    RELEASE_AMETHOD_SETTING();
+}

elements/gmf_audio/esp_gmf_sonic.c

@@ -157,6 +157,7 @@ static esp_gmf_job_err_t esp_gmf_sonic_process(esp_gmf_element_handle_t self, vo
     esp_gmf_payload_t *in_load = NULL;
     esp_gmf_payload_t *out_load = NULL;
     bool is_done = false;
+    sonic->out_data_hd.needed_num = sonic->out_size / sonic->bytes_per_sample;
     if (sonic->speed == 1.0f && sonic->pitch == 1.0f && (sonic->out_data_hd.out_num < sonic->out_data_hd.needed_num)) {
         out_len = gmf_sonic_bypass_process(sonic, in_port, out_port, &in_load, &out_load);
         goto __sonic_release;
@@ -188,7 +189,6 @@ static esp_gmf_job_err_t esp_gmf_sonic_process(esp_gmf_element_handle_t self, vo
         }
         load_ret = esp_gmf_port_acquire_out(out_port, &out_load, sonic->out_size, ESP_GMF_MAX_DELAY);
         ESP_GMF_PORT_ACQUIRE_OUT_CHECK(TAG, load_ret, out_len, goto __sonic_release);
-        sonic->out_data_hd.needed_num = sonic->out_size / sonic->bytes_per_sample;
         sonic->out_data_hd.samples = out_load->buf;
         out_load->valid_size = 0;
         out_load->pts = sonic->cur_pts;

elements/gmf_audio/idf_component.yml

@@ -1,4 +1,4 @@
-version: "0.7.0"
+version: "0.7.1"
 description: Espressif GMF Audio Elements Module
 url: https://github.com/espressif/esp-gmf/tree/main/elements/gmf_audio
 documentation: "https://github.com/espressif/esp-gmf/blob/main/elements/gmf_audio/README.md"

elements/gmf_audio/include/esp_gmf_audio_param.h

@@ -55,6 +55,74 @@ esp_gmf_err_t esp_gmf_audio_param_set_dest_bits(esp_gmf_element_handle_t self, u
  */
 esp_gmf_err_t esp_gmf_audio_param_set_dest_ch(esp_gmf_element_handle_t self, uint8_t dest_ch);
 
+/**
+ * @brief  Set speed for audio element
+ *
+ * @param[in]  handle  Audio element handle
+ * @param[in]  speed   Audio playback speed
+ *
+ * @return
+ *       - ESP_GMF_ERR_OK           On success
+ *       - ESP_GMF_ERR_NOT_FOUND    Not found the method
+ *       - ESP_GMF_ERR_MEMORY_LACK  Failed to allocate memory
+ *       - Others                   Failed to apply method
+ */
+esp_gmf_err_t esp_gmf_audio_param_set_speed(esp_gmf_element_handle_t self, float speed);
+
+/**
+ * @brief  Set pitch for audio element
+ *
+ * @param[in]  handle  Audio element handle
+ * @param[in]  pitch   Audio pitch to set
+ *
+ * @return
+ *       - ESP_GMF_ERR_OK           On success
+ *       - ESP_GMF_ERR_NOT_FOUND    Not found the method
+ *       - ESP_GMF_ERR_MEMORY_LACK  Failed to allocate memory
+ *       - Others                   Failed to apply method
+ */
+esp_gmf_err_t esp_gmf_audio_param_set_pitch(esp_gmf_element_handle_t self, float pitch);
+
+/**
+ * @brief  Set ALC certain channel gain for audio element
+ *
+ * @param[in]  handle   Audio element handle
+ * @param[in]  ch_idx   Channel index (0: Left channel 1: Right Channel 0xFF: All channels)
+ * @param[in]  gain_db  ALC gain (unit decibel)
+ *
+ * @return
+ *       - ESP_GMF_ERR_OK           On success
+ *       - ESP_GMF_ERR_NOT_FOUND    Not found the method
+ *       - ESP_GMF_ERR_MEMORY_LACK  Failed to allocate memory
+ *       - Others                   Failed to apply method
+ */
+esp_gmf_err_t esp_gmf_audio_param_set_alc_channel_gain(esp_gmf_element_handle_t self, uint8_t ch_idx, float gain_db);
+
+/**
+ * @brief  Set ALC gain for all channels
+ */
+#define esp_gmf_audio_param_set_alc_gain(self, gain_db) esp_gmf_audio_param_set_alc_channel_gain(self, 0xFF, gain_db)
+
+/**
+ * @brief  Set fade in/out direction for audio element
+ *
+ * @param[in]  handle      Audio element handle
+ * @param[in]  is_fade_in  Whether fade in or out
+ *
+ * @return
+ *       - ESP_GMF_ERR_OK           On success
+ *       - ESP_GMF_ERR_NOT_FOUND    Not found the method
+ *       - ESP_GMF_ERR_MEMORY_LACK  Failed to allocate memory
+ *       - Others                   Failed to apply method
+ */
+esp_gmf_err_t esp_gmf_audio_param_set_fade(esp_gmf_element_handle_t self, bool is_fade_in);
+
+/**
+ * @brief  Helper wrapper for fade in and fade out setting
+ */
+#define esp_gmf_audio_param_set_fade_in(self)  esp_gmf_audio_param_set_fade(self, true);
+#define esp_gmf_audio_param_set_fade_out(self) esp_gmf_audio_param_set_fade(self, false);
+
 #ifdef __cplusplus
 }
 #endif

packages/esp_audio_render/CMakeLists.txt

@@ -0,0 +1,13 @@
+# Gather common sources
+set(COMPONENT_SRC_DIRS "src")
+
+set(COMPONENT_INCLUDE "./include")
+
+set(COMPONENT_PRIV_INCLUDE "private_inc")
+
+idf_component_register(
+    SRC_DIRS ${COMPONENT_SRC_DIRS}
+    INCLUDE_DIRS ${COMPONENT_INCLUDE}
+    PRIV_INCLUDE_DIRS ${COMPONENT_PRIV_INCLUDE}
+    REQUIRES esp_timer
+)