Merge branch 'feature/add_mixer_in_acquire_wait_time_check' into 'main'

Add mixer in acquire wait time check

See merge request adf/multimedia/esp-gmf!94

Author: jason-mao

elements/gmf_audio/CHANGELOG.md

@@ -19,6 +19,7 @@
 - Added TRUNCATE support for SONIC element to enable its flexible placement at any position within the pipeline
 - Supported scenarios where the input data for audio encoding is not a complete frame
 - Added PTS (Presentation Time Stamp) calculation and propagation for the `aud_alc`, `aud_eq`, `aud_fade`, `aud_enc`
+- Supported user-configurable input frame length and load acquisition timeout for the mixer
 
 ### Bug Fixes
 
@@ -30,6 +31,7 @@
 - Fixed the issue where the encoder modified the configuration during runtime
 - Fixed the issue where the mixer did not forward the out_load parameter to downstream components when all input stream were no data
 - Fixed bugs that not check validation of in and out buffer for mixer which will cause crash
+- Fixed bugs where setting parameters before opening had no effect for `aud_fade` and `aud_mixer`
 
 ## v0.6.3
 

elements/gmf_audio/CMakeLists.txt

@@ -1,3 +1,4 @@
 idf_component_register(SRC_DIRS "./"
                        INCLUDE_DIRS "./include"
-                       PRIV_INCLUDE_DIRS "./private_include")
+                       PRIV_INCLUDE_DIRS "./private_include"
+                       REQUIRES esp_timer)

elements/gmf_audio/esp_gmf_fade.c

@@ -24,7 +24,6 @@ typedef struct {
     esp_gmf_audio_element_t parent;            /*!< The GMF fade handle */
     esp_ae_fade_handle_t    fade_hd;           /*!< The audio effects fade handle */
     uint8_t                 bytes_per_sample;  /*!< Bytes number of per sampling point */
-    esp_ae_fade_mode_t      mode;              /*!< The current fade mode */
     bool                    need_reopen : 1;   /*!< Whether need to reopen.
                                                     True: Execute the close function first, then execute the open function
                                                     False: Do nothing */
@@ -236,6 +235,8 @@ esp_gmf_err_t esp_gmf_fade_set_mode(esp_gmf_element_handle_t handle, esp_ae_fade
 {
     ESP_GMF_NULL_CHECK(TAG, handle, { return ESP_GMF_ERR_INVALID_ARG;});
     esp_gmf_fade_t *fade = (esp_gmf_fade_t *)handle;
+    esp_ae_fade_cfg_t *cfg = (esp_ae_fade_cfg_t *)OBJ_GET_CFG(handle);
+    ESP_GMF_NULL_CHECK(TAG, cfg, return ESP_GMF_ERR_FAIL);
     if (fade->fade_hd) {
         esp_gmf_oal_mutex_lock(((esp_gmf_audio_element_t *)handle)->lock);
         esp_ae_err_t ret = esp_ae_fade_set_mode(fade->fade_hd, mode);
@@ -244,29 +245,33 @@ esp_gmf_err_t esp_gmf_fade_set_mode(esp_gmf_element_handle_t handle, esp_ae_fade
             return ESP_GMF_JOB_ERR_FAIL;
         }
     }
-    fade->mode = mode;
+    cfg->mode = mode;
     return ESP_GMF_JOB_ERR_OK;
 }
 
 esp_gmf_err_t esp_gmf_fade_get_mode(esp_gmf_element_handle_t handle, esp_ae_fade_mode_t *mode)
 {
     ESP_GMF_NULL_CHECK(TAG, handle, { return ESP_GMF_ERR_INVALID_ARG;});
     ESP_GMF_NULL_CHECK(TAG, mode, { return ESP_GMF_ERR_INVALID_ARG;});
+    esp_ae_fade_cfg_t *cfg = (esp_ae_fade_cfg_t *)OBJ_GET_CFG(handle);
+    ESP_GMF_NULL_CHECK(TAG, cfg, return ESP_GMF_ERR_FAIL);
     esp_gmf_fade_t *fade = (esp_gmf_fade_t *)handle;
     if (fade->fade_hd) {
         esp_ae_err_t ret = esp_ae_fade_get_mode(fade->fade_hd, mode);
         if (ret != ESP_AE_ERR_OK) {
             return ESP_GMF_JOB_ERR_FAIL;
         }
     } else {
-        *mode = fade->mode;
+        *mode = cfg->mode;
     }
     return ESP_GMF_JOB_ERR_OK;
 }
 
 esp_gmf_err_t esp_gmf_fade_reset_weight(esp_gmf_element_handle_t handle)
 {
     ESP_GMF_NULL_CHECK(TAG, handle, { return ESP_GMF_ERR_INVALID_ARG;});
+    esp_ae_fade_cfg_t *cfg = (esp_ae_fade_cfg_t *)OBJ_GET_CFG(handle);
+    ESP_GMF_NULL_CHECK(TAG, cfg, return ESP_GMF_ERR_FAIL);
     esp_gmf_fade_t *fade = (esp_gmf_fade_t *)handle;
     if(fade->fade_hd) {
         esp_gmf_oal_mutex_lock(((esp_gmf_audio_element_t *)handle)->lock);
@@ -275,7 +280,7 @@ esp_gmf_err_t esp_gmf_fade_reset_weight(esp_gmf_element_handle_t handle)
         if (ret != ESP_AE_ERR_OK) {
             return ESP_GMF_JOB_ERR_FAIL;
         }
-        esp_ae_fade_get_mode(fade->fade_hd, &fade->mode);
+        esp_ae_fade_get_mode(fade->fade_hd, &cfg->mode);
     }
     return ESP_GMF_JOB_ERR_OK;
 }

elements/gmf_audio/esp_gmf_mixer.c

@@ -17,18 +17,24 @@
 #include "esp_gmf_cap.h"
 #include "esp_gmf_caps_def.h"
 #include "esp_gmf_audio_element.h"
+#ifdef MIXER_DEBUG
+#include "esp_timer.h"
+#endif
 
-#define MIXER_DEFAULT_PROC_TIME_MS (10)
+#define MIXER_DEFAULT_PROC_TIME_MS      (10)
+#define MIXER_GET_FRAME_BYTE_SIZE(info) (MIXER_DEFAULT_PROC_TIME_MS * info->sample_rate * info->channel * info->bits_per_sample / 8000)
 
 typedef struct {
     esp_gmf_audio_element_t parent;            /*!< The GMF mixer handle */
     esp_ae_mixer_handle_t   mixer_hd;          /*!< The audio effects mixer handle */
     uint8_t                 bytes_per_sample;  /*!< Bytes number of per sampling point */
     uint32_t                process_num;       /*!< The data number of mixer processing */
+    uint32_t                frame_time;        /*!< The time of one frame, unit: ms */
     esp_gmf_payload_t     **in_load;           /*!< The array of input payload */
     esp_gmf_payload_t      *out_load;          /*!< The output payload */
     uint8_t               **in_arr;            /*!< The input buffer pointer array of mixer */
     esp_ae_mixer_mode_t    *mode;              /*!< The mixer mode */
+    uint8_t                 src_num;           /*!< The number of source */
     bool                    need_reopen : 1;   /*!< Whether need to reopen.
                                                     True: Execute the close function first, then execute the open function
                                                     False: Do nothing */
@@ -102,9 +108,15 @@ static esp_gmf_job_err_t esp_gmf_mixer_open(esp_gmf_element_handle_t self, void
     esp_ae_mixer_cfg_t *mixer_info = (esp_ae_mixer_cfg_t *)OBJ_GET_CFG(self);
     ESP_GMF_NULL_CHECK(TAG, mixer_info, {return ESP_GMF_JOB_ERR_FAIL;})
     mixer->bytes_per_sample = (mixer_info->bits_per_sample >> 3) * mixer_info->channel;
-    mixer->process_num = MIXER_DEFAULT_PROC_TIME_MS * mixer_info->sample_rate * mixer->bytes_per_sample / 1000;
     esp_ae_mixer_open(mixer_info, &mixer->mixer_hd);
     ESP_GMF_CHECK(TAG, mixer->mixer_hd, {return ESP_GMF_JOB_ERR_FAIL;}, "Failed to create mixer handle");
+    for (int i = 0; i < mixer_info->src_num; i++) {
+        esp_ae_mixer_set_mode(mixer->mixer_hd, i, mixer->mode[i]);
+    }
+    uint32_t process_num = MIXER_GET_FRAME_BYTE_SIZE(mixer_info);
+    mixer->process_num = (ESP_GMF_ELEMENT_GET(mixer)->in_attr.data_size == 0) ? (process_num) : (ESP_GMF_ELEMENT_GET(mixer)->in_attr.data_size);
+    mixer->frame_time = GMF_AUDIO_CALC_PTS(mixer->process_num, mixer_info->sample_rate, mixer_info->channel, mixer_info->bits_per_sample);
+    mixer->src_num = mixer_info->src_num;
     mixer->in_load = esp_gmf_oal_calloc(1, sizeof(esp_gmf_payload_t *) * mixer_info->src_num);
     ESP_GMF_MEM_VERIFY(TAG, mixer->in_load, {return ESP_GMF_JOB_ERR_FAIL;},
                        "in load", sizeof(esp_gmf_payload_t *) * mixer_info->src_num);
@@ -156,16 +168,27 @@ static esp_gmf_job_err_t esp_gmf_mixer_process(esp_gmf_element_handle_t self, vo
     esp_gmf_port_handle_t in = ESP_GMF_ELEMENT_GET(self)->in;
     esp_gmf_port_handle_t in_port = in;
     esp_gmf_port_handle_t out_port = ESP_GMF_ELEMENT_GET(self)->out;
-    esp_ae_mixer_cfg_t *mixer_info = (esp_ae_mixer_cfg_t *)OBJ_GET_CFG(self);
-    ESP_GMF_NULL_CHECK(TAG, mixer_info, return ESP_GMF_JOB_ERR_FAIL);
-    int index = mixer_info->src_num;
+    int index = mixer->src_num;
     memset(mixer->in_load, 0, sizeof(esp_gmf_payload_t *) * index);
     mixer->out_load = NULL;
     int i = 0;
-    int wait_time = 0;
+#ifdef MIXER_DEBUG
+    uint64_t start = 0;
+    uint64_t end = 0;
+    uint64_t acquire_in_time = 0;
+    uint32_t frame_time = mixer->frame_time;
+#endif
     while (in_port != NULL) {
-        wait_time = i == 0 ? ESP_GMF_MAX_DELAY : 0;
-        ret = esp_gmf_port_acquire_in(in_port, &(mixer->in_load[i]), mixer->process_num, wait_time);
+#ifdef MIXER_DEBUG
+        start = esp_timer_get_time();
+#endif
+        int wait_ticks = ((in_port->wait_ticks < (mixer->frame_time / index)) ? (mixer->frame_time / index) : (in_port->wait_ticks));
+        ret = esp_gmf_port_acquire_in(in_port, &(mixer->in_load[i]), mixer->process_num, wait_ticks);
+#ifdef MIXER_DEBUG
+        end = esp_timer_get_time();
+        acquire_in_time += ((end - start) / 1000);
+        ESP_LOGD(TAG, "Port %d acquire in time: %lld ms, wait ticks: %d", i, (end - start) / 1000, wait_ticks);
+#endif
         if (ret == ESP_GMF_IO_FAIL || mixer->in_load[i]->buf == NULL) {
             ESP_LOGE(TAG, "Acquire in failed, idx:%d, ret: %d", i, ret);
             out_len = ESP_GMF_JOB_ERR_FAIL;
@@ -180,11 +203,16 @@ static esp_gmf_job_err_t esp_gmf_mixer_process(esp_gmf_element_handle_t self, vo
             memset(mixer->in_arr[i] + read_len, 0, mixer->process_num - read_len);
         }
         in_port = in_port->next;
-        i++;
         ESP_LOGV(TAG, "IN: idx: %d load: %p, buf: %p, valid size: %d, buf length: %d, done: %d",
                  i, mixer->in_load[i], mixer->in_load[i]->buf, mixer->in_load[i]->valid_size,
                  mixer->in_load[i]->buf_length, mixer->in_load[i]->is_done);
+        i++;
+    }
+#ifdef MIXER_DEBUG
+    if (acquire_in_time > frame_time) {
+        ESP_LOGW(TAG, "Total acquire in time: %lld ms, frame time: %ld ms", acquire_in_time, frame_time);
     }
+#endif
     ret = esp_gmf_port_acquire_out(out_port, &mixer->out_load, mixer->process_num, ESP_GMF_MAX_DELAY);
     ESP_GMF_PORT_ACQUIRE_OUT_CHECK(TAG, ret, out_len, { goto __mixer_release;});
     // Down-mixer never stop in gmf, only user can set to stop
@@ -380,13 +408,17 @@ esp_gmf_err_t esp_gmf_mixer_init(esp_ae_mixer_cfg_t *config, esp_gmf_element_han
     esp_gmf_obj_set_config(obj, cfg, sizeof(esp_ae_mixer_cfg_t));
     mixer->mode = esp_gmf_oal_calloc(cfg->src_num, sizeof(esp_ae_mixer_mode_t));
     ESP_GMF_MEM_VERIFY(TAG, mixer->mode, {ret = ESP_GMF_ERR_MEMORY_LACK; goto MIXER_INIT_FAIL;}, "Allocate(%d) failed", cfg->src_num * sizeof(esp_ae_mixer_mode_t));
+    for (int i = 0; i < cfg->src_num; i++) {
+        mixer->mode[i] = ESP_AE_MIXER_MODE_FADE_UPWARD;
+    }
     ret = esp_gmf_obj_set_tag(obj, "aud_mixer");
     ESP_GMF_RET_ON_NOT_OK(TAG, ret, goto MIXER_INIT_FAIL, "Failed to set obj tag");
     esp_gmf_element_cfg_t el_cfg = {0};
+    mixer->process_num = MIXER_GET_FRAME_BYTE_SIZE(cfg);
     ESP_GMF_ELEMENT_IN_PORT_ATTR_SET(el_cfg.in_attr, ESP_GMF_EL_PORT_CAP_MULTI, 0, 0,
-        ESP_GMF_PORT_TYPE_BLOCK | ESP_GMF_PORT_TYPE_BYTE, ESP_GMF_ELEMENT_PORT_DATA_SIZE_DEFAULT);
+        ESP_GMF_PORT_TYPE_BLOCK | ESP_GMF_PORT_TYPE_BYTE, mixer->process_num);
     ESP_GMF_ELEMENT_IN_PORT_ATTR_SET(el_cfg.out_attr, ESP_GMF_EL_PORT_CAP_SINGLE, 0, 0,
-        ESP_GMF_PORT_TYPE_BLOCK | ESP_GMF_PORT_TYPE_BYTE, ESP_GMF_ELEMENT_PORT_DATA_SIZE_DEFAULT);
+        ESP_GMF_PORT_TYPE_BLOCK | ESP_GMF_PORT_TYPE_BYTE, mixer->process_num);
     el_cfg.dependency = true;
     ret = esp_gmf_audio_el_init(mixer, &el_cfg);
     ESP_GMF_RET_ON_NOT_OK(TAG, ret, goto MIXER_INIT_FAIL, "Failed to initialize mixer element");

elements/test_apps/main/elements/gmf_audio_effects_test.c

@@ -498,11 +498,10 @@ TEST_CASE("Audio mixer Play", "[ESP_GMF_Effects]")
     TEST_ASSERT_EQUAL(ESP_GMF_ERR_OK, esp_gmf_pipeline_run(pipe3));
     esp_gmf_element_handle_t mixer_hd = NULL;
     TEST_ASSERT_EQUAL(ESP_GMF_ERR_OK, esp_gmf_pipeline_get_el_by_name(pipe3, "aud_mixer", &mixer_hd));
-    esp_ae_mixer_mode_t mode = ESP_AE_MIXER_MODE_FADE_UPWARD;
     xEventGroupWaitBits(pipe_sync_evt3, PIPELINE_BLOCK_RUN_BIT, pdTRUE, pdFALSE, portMAX_DELAY);
     // set mode
-    TEST_ASSERT_EQUAL(ESP_GMF_ERR_OK, esp_gmf_mixer_set_mode(mixer_hd, 0, mode));
-    TEST_ASSERT_EQUAL(ESP_GMF_ERR_OK, esp_gmf_mixer_set_mode(mixer_hd, 1, mode));
+    TEST_ASSERT_EQUAL(ESP_GMF_ERR_OK, esp_gmf_mixer_set_mode(mixer_hd, 0, ESP_AE_MIXER_MODE_FADE_UPWARD));
+    TEST_ASSERT_EQUAL(ESP_GMF_ERR_OK, esp_gmf_mixer_set_mode(mixer_hd, 1, ESP_AE_MIXER_MODE_FADE_DOWNWARD));
 
     xEventGroupWaitBits(pipe_sync_evt1, PIPELINE_BLOCK_BIT, pdTRUE, pdFALSE, portMAX_DELAY);
     xEventGroupWaitBits(pipe_sync_evt2, PIPELINE_BLOCK_BIT, pdTRUE, pdFALSE, portMAX_DELAY);

packages/esp_audio_simple_player/CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Features
 
 - Replaced the interface for decoding reconfig
+- Supported placing the audio processing task stack in RAM or SPI-RAM using `stack_in_ext` in `esp_asp_cfg_t`
 
 ## v0.9.3
 

packages/esp_audio_simple_player/include/esp_audio_simple_player.h

@@ -69,13 +69,14 @@ typedef struct {
  * @brief  Configuration structure for the audio simple player
  */
 typedef struct {
-    esp_asp_func_t       in;          /*!< Input data callback, it is required only for raw data(raw://xxx/xxx.mp3), and is ignored in other cases */
-    esp_asp_func_t       out;         /*!< Output data callback configuration, it is required for all cases */
-    int                  task_prio;   /*!< Priority of the player task */
-    int                  task_stack;  /*!< Size of the task stack */
-    uint8_t              task_core;   /*!< CPU core affinity for the task */
-    esp_asp_prev_func_t  prev;        /*!< An optional callback invoked before the pipeline starts (e.g., configure linked elements before running) */
-    void                *prev_ctx;    /*!< User context passed to the previous action callback */
+    esp_asp_func_t       in;                    /*!< Input data callback, it is required only for raw data(raw://xxx/xxx.mp3), and is ignored in other cases */
+    esp_asp_func_t       out;                   /*!< Output data callback configuration, it is required for all cases */
+    int                  task_prio;             /*!< Priority of the player task */
+    int                  task_stack;            /*!< Size of the task stack */
+    uint8_t              task_core;             /*!< CPU core affinity for the task */
+    bool                 task_stack_in_ext : 1; /*!< Whether the task stack is in external memory */
+    esp_asp_prev_func_t  prev;                  /*!< An optional callback invoked before the pipeline starts (e.g., configure linked elements before running) */
+    void                *prev_ctx;              /*!< User context passed to the previous action callback */
 } esp_asp_cfg_t;
 
 /**

packages/esp_audio_simple_player/src/esp_audio_simple_player.c

@@ -300,6 +300,7 @@ esp_gmf_err_t esp_audio_simple_player_new(esp_asp_cfg_t *cfg, esp_asp_handle_t *
         task_cfg.thread.prio = cfg->task_prio;
     }
     task_cfg.thread.core = cfg->task_core;
+    task_cfg.thread.stack_in_ext = cfg->task_stack_in_ext;
     esp_gmf_task_init(&task_cfg, &player->work_task);
     if (player->work_task == NULL) {
         ESP_LOGE(TAG, "Failed to create the pipeline task");