Merge branch 'bugfix/fix_capture_audio_bypass_err' into 'main'

bugfix(esp_capture): Fix audio bypass fail and dynamic enable failed

See merge request adf/multimedia/esp-gmf!138

Author: jason-mao

packages/esp_capture/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## v0.7.4
+
+### Bug Fixes
+
+- Fixed failed to start audio capture with PCM format
+- Fixed dynamically enable disable sink failed after started
+- Added get sink handle with same configuration after started
+- Fixed sync lost if dynamical enable and disable sink
+- Added test cases for audio bypass
+- Added test cases for dynamically enable and disable sink
+- Added multiple start and stop test cases
+
 ## v0.7.3
 
 ### Bug Fixes

packages/esp_capture/idf_component.yml

@@ -1,4 +1,4 @@
-version: "0.7.3"
+version: "0.7.4"
 description: Espressif Capture is a module for capture media stream from camera and microphone
 url: https://github.com/espressif/esp-gmf/tree/main/packages/esp_capture
 documentation: "https://github.com/espressif/esp-gmf/blob/main/packages/esp_capture/README.md"

packages/esp_capture/impl/capture_gmf_path/src/elements/gmf_audio_src.c

@@ -29,6 +29,7 @@ typedef struct {
     esp_gmf_audio_element_t      parent;
     esp_gmf_port_handle_t        in_port;
     esp_capture_sync_handle_t    sync_handle;
+    uint32_t                     base_pts;
     esp_capture_audio_src_if_t  *audio_src_if;
     esp_gmf_info_sound_t         aud_info;
     data_q_t                    *audio_src_q;
@@ -99,14 +100,22 @@ static void audio_src_thread(void *arg)
         }
         if (audio_src->frame_reached == false) {
             CAPTURE_PERF_MON(0, "Audio Src Frame Reached", {});
+            // Get base pts
+            if (audio_src->sync_handle) {
+                uint32_t cur_pts = 0;
+                esp_capture_sync_get_current(audio_src->sync_handle, &cur_pts);
+                if (cur_pts > CAPTURE_SYNC_TOLERANCE) {
+                    audio_src->base_pts = cur_pts;
+                }
+            }
             audio_src->frame_reached = true;
         }
         esp_capture_stream_frame_t *frame = (esp_capture_stream_frame_t *)data;
         frame->stream_type = ESP_CAPTURE_STREAM_TYPE_AUDIO;
         frame->data = (data + sizeof(esp_capture_stream_frame_t));
         frame->size = audio_src->audio_frame_size;
         int ret = audio_src->audio_src_if->read_frame(audio_src->audio_src_if, frame);
-        frame->pts = calc_audio_pts(audio_src, audio_src->audio_frames);
+        frame->pts = calc_audio_pts(audio_src, audio_src->audio_frames) + audio_src->base_pts;
         if (ret != ESP_CAPTURE_ERR_OK) {
             data_q_send_buffer(audio_src->audio_src_q, 0);
             ESP_LOGE(TAG, "Failed to read audio frame ret %d", ret);
@@ -126,6 +135,7 @@ static void audio_src_thread(void *arg)
         data_q_send_buffer(audio_src->audio_src_q, frame_size);
         audio_src->audio_frames++;
     }
+    audio_src->audio_frames = 0;
     // Wakeup reader if read from device failed
     if (err_exit) {
         data_q_wakeup(audio_src->audio_src_q);

packages/esp_capture/impl/capture_gmf_path/src/gmf_capture_audio_path.c

@@ -54,6 +54,21 @@ static esp_capture_err_t get_audio_encoder(gmf_capture_path_mngr_t *mngr, uint8_
     return ESP_CAPTURE_ERR_NOT_FOUND;
 }
 
+static esp_gmf_element_handle_t get_sink_tail_element(gmf_capture_path_mngr_t *mngr, audio_path_res_t *res)
+{
+    uint8_t path_mask = (1 << res->base.path);
+    for (int i = 0; i < mngr->pipeline_num; i++) {
+        esp_capture_gmf_pipeline_t *pipeline = &mngr->pipeline[i];
+        if ((pipeline->path_mask & path_mask) == 0) {
+            continue;
+        }
+        if (capture_pipeline_is_sink(pipeline->pipeline)) {
+            return ESP_GMF_PIPELINE_GET_LAST_ELEMENT(((esp_gmf_pipeline_handle_t)pipeline->pipeline));
+        }
+    }
+    return NULL;
+}
+
 static esp_capture_err_t set_audio_source_sync_handle(gmf_capture_path_mngr_t *mngr, esp_capture_sync_handle_t sync_handle)
 {
     for (int i = 0; i < mngr->pipeline_num; i++) {
@@ -99,10 +114,16 @@ static esp_gmf_err_io_t audio_sink_acquire(void *handle, esp_gmf_payload_t *load
     if (aud_frame == NULL) {
         return ESP_GMF_IO_FAIL;
     }
+
     aud_frame->stream_type = ESP_CAPTURE_STREAM_TYPE_AUDIO;
     aud_frame->data = ((void *)aud_frame) + sizeof(esp_capture_stream_frame_t);
-    load->buf = aud_frame->data;
-    load->buf_length = wanted_size;
+    if (load->buf) {
+        // In bypass case copy data directly
+        memcpy(aud_frame->data, load->buf, load->valid_size);
+    } else {
+        load->buf = aud_frame->data;
+        load->buf_length = wanted_size;
+    }
     return ESP_GMF_IO_OK;
 }
 
@@ -124,6 +145,10 @@ static esp_gmf_err_io_t audio_sink_release(void *handle, esp_gmf_payload_t *load
             ESP_LOGI(TAG, "Drop for disable");
             data_q_send_buffer(q, 0);
         }
+        if (load->buf == aud_frame->data) {
+            // Clear buf when not bypass case
+            load->buf = NULL;
+        }
     }
     return ESP_GMF_IO_OK;
 }
@@ -144,7 +169,11 @@ static esp_capture_err_t audio_path_prepare(gmf_capture_path_res_t *mngr_res)
         if (res->audio_q == NULL) {
             return ESP_CAPTURE_ERR_NO_MEM;
         }
-        esp_gmf_element_register_out_port(res->aenc_el, res->sink_port);
+        if (res->aenc_el) {
+            esp_gmf_element_register_out_port(res->aenc_el, res->sink_port);
+        } else {
+            esp_gmf_element_register_out_port(get_sink_tail_element(mngr_res->parent, res), res->sink_port);
+        }
     }
     return ESP_CAPTURE_ERR_OK;
 }

packages/esp_capture/impl/capture_gmf_path/src/gmf_capture_auto_audio_pipeline.c

@@ -321,6 +321,15 @@ static esp_capture_err_t buildup_pipelines(audio_pipeline_t *audio_pipe)
                 // TODO currently only support encoder
                 proc_elements[proc_num++] = element_name;
             }
+            if (proc_num == 0) {
+                // Add a dummy element in case no element can not form pipeline
+                const char *element_name = get_ops_element(audio_pipe, AUDIO_PATH_OPS_BIT_CVT);
+                if (element_name == NULL) {
+                    ESP_LOGE(TAG, "Can not find element for bit convert");
+                    return ESP_CAPTURE_ERR_INTERNAL;
+                }
+                proc_elements[proc_num++] = element_name;
+            }
 #if 0
             printf("Audio Pipeline %d: ", i);
             for (int j = 0; j < proc_num; j++) {

packages/esp_capture/impl/capture_gmf_path/src/gmf_capture_path_mngr.c

@@ -160,15 +160,27 @@ static esp_capture_err_t prepare_pipeline(gmf_capture_path_mngr_t *mngr, uint8_t
     gmf_capture_path_res_t *res = gmf_capture_path_mngr_get_path(mngr, path);
     uint8_t path_mask = (1 << path);
     if (res->negotiated == false) {
+        bool for_all = true;
+        for (int i = 0; i < mngr->path_num; i++) {
+            gmf_capture_path_res_t *each = gmf_capture_path_mngr_get_path(mngr, i);
+            if (each->negotiated) {
+                for_all = false;
+                break;
+            }
+        }
         if (mngr->pipeline_builder->negotiate) {
             int ret = ESP_CAPTURE_ERR_OK;
+            uint8_t nego_mask = for_all ? ESP_CAPTURE_PIPELINE_NEGO_ALL_MASK : path_mask;
             CAPTURE_PERF_MON(path, (mngr->stream_type == ESP_CAPTURE_STREAM_TYPE_AUDIO) ? "Negotiate Audio Sink" : "Negotiate Video Sink", {
-                ret = mngr->pipeline_builder->negotiate(mngr->pipeline_builder, path_mask);
+                ret = mngr->pipeline_builder->negotiate(mngr->pipeline_builder, nego_mask);
             });
             if (ret != ESP_CAPTURE_ERR_OK) {
                 ESP_LOGE(TAG, "Fail to negotiate audio pipeline");
                 return ret;
             }
+            if (res->configured) {
+                res->negotiated = true;
+            }
         }
     }
     for (int i = 0; i < mngr->pipeline_num; i++) {
@@ -298,6 +310,12 @@ static void stop_pipelines(gmf_capture_path_mngr_t *mngr, uint8_t path, gmf_capt
             CAPTURE_PERF_MON(res->path, (mngr->stream_type == ESP_CAPTURE_STREAM_TYPE_AUDIO) ? "Prepare Stop Audio Sink" : "Prepare Stop Video Sink", {
                 stop_cb(res);
             });
+        } else {
+            // Src is stopped all related sink need re-negotiate
+            for (int i = 0; i < mngr->path_num; i++) {
+                gmf_capture_path_res_t *each = gmf_capture_path_mngr_get_idx(mngr, i);
+                each->negotiated = false;
+            }
         }
         ESP_LOGD(TAG, "Start to stop pipeline %d", sel_pipe);
         CAPTURE_PERF_MON(res->path, (mngr->stream_type == ESP_CAPTURE_STREAM_TYPE_AUDIO) ? "Stop Audio Pipeline" : "Stop Video Pipeline", {
@@ -353,6 +371,7 @@ static esp_capture_err_t stop_path(gmf_capture_path_mngr_t *mngr, uint8_t path,
         return ESP_CAPTURE_ERR_OK;
     }
     res->started = false;
+    res->negotiated = false;
     stop_pipelines(mngr, path, stop_cb);
     release_pipelines(mngr, path, release_cb);
     ESP_LOGD(TAG, "Path %d stop finished\n", path);

packages/esp_capture/src/esp_capture.c

@@ -510,6 +510,29 @@ static esp_capture_path_mngr_if_t *capture_get_mngr_by_stream_type(capture_t *ca
     return mngr;
 }
 
+static bool capture_same_sink_cfg(esp_capture_sink_cfg_t *old, esp_capture_sink_cfg_t *sink_cfg)
+{
+    if (old->audio_info.format_id != sink_cfg->audio_info.format_id ||
+        old->video_info.format_id != sink_cfg->video_info.format_id) {
+        return false;
+    }
+    if (old->audio_info.format_id) {
+        if (old->audio_info.sample_rate != sink_cfg->audio_info.sample_rate ||
+            old->audio_info.channel != sink_cfg->audio_info.channel ||
+            old->audio_info.bits_per_sample != sink_cfg->audio_info.bits_per_sample) {
+            return false;
+        }
+    }
+    if (old->video_info.format_id) {
+        if (old->video_info.width != sink_cfg->video_info.width ||
+            old->video_info.height != sink_cfg->video_info.height ||
+            old->video_info.fps != sink_cfg->video_info.fps) {
+            return false;
+        }
+    }
+    return true;
+}
+
 esp_capture_err_t esp_capture_set_thread_scheduler(esp_capture_thread_scheduler_cb_t thread_scheduler)
 {
     capture_thread_set_scheduler(thread_scheduler);
@@ -736,15 +759,21 @@ esp_capture_err_t esp_capture_sink_setup(esp_capture_handle_t h, uint8_t type, e
     capture_mutex_lock(capture->api_lock, CAPTURE_MAX_LOCK_TIME);
     esp_capture_err_t ret = ESP_CAPTURE_ERR_OK;
     do {
+        capture_path_t *cur = capture_get_path_by_index(capture, type);
         if (capture->started) {
+            if (cur && capture_same_sink_cfg(&cur->sink_cfg, sink_info)) {
+                // Allow get sink handle use same setup
+                *path = (esp_capture_sink_handle_t)cur;
+                ret = ESP_CAPTURE_ERR_OK;
+                break;
+            }
             ESP_LOGE(TAG, "Not support add path after started");
             CAPTURE_BREAK_SET_RETURN(ret, ESP_CAPTURE_ERR_INVALID_STATE);
         }
         if (capture->cfg.audio_path == NULL && capture->cfg.video_path == NULL) {
             ESP_LOGE(TAG, "Only support add sink for no path manager");
             CAPTURE_BREAK_SET_RETURN(ret, ESP_CAPTURE_ERR_NOT_SUPPORTED);
         }
-        capture_path_t *cur = capture_get_path_by_index(capture, type);
         // Path already added
         if (cur) {
             if (cur->enable && capture->started) {
@@ -1104,7 +1133,6 @@ esp_capture_err_t esp_capture_sink_acquire_frame(esp_capture_sink_handle_t h, es
     }
     // TODO not add lock user need care the timing
     if (path->enable == false) {
-        ESP_LOGE(TAG, "Capture path %d is not enabled", path->path_type);
         return ESP_CAPTURE_ERR_INVALID_STATE;
     }
     int ret = ESP_CAPTURE_ERR_NOT_SUPPORTED;
@@ -1153,7 +1181,6 @@ esp_capture_err_t esp_capture_sink_release_frame(esp_capture_sink_handle_t h, es
     }
     // TODO not add lock user need care the timing
     if (path->enable == false) {
-        ESP_LOGE(TAG, "Capture path %d is not enabled", path->path_type);
         return ESP_CAPTURE_ERR_INVALID_STATE;
     }
     int ret = ESP_CAPTURE_ERR_OK;

packages/esp_capture/src/esp_capture_sync.c

@@ -52,6 +52,7 @@ esp_capture_err_t esp_capture_sync_on(esp_capture_sync_handle_t handle)
     }
     sync_t *sync = (sync_t *)handle;
     sync->started = true;
+    sync->last_update_pts = 0;
     sync->last_update_time = CUR();
     return ESP_CAPTURE_ERR_OK;
 }

packages/esp_capture/test_apps/main/capture_builder.c

@@ -318,24 +318,13 @@ int build_advance_av_capture_sys(capture_sys_t *capture_sys)
     return 0;
 }
 
-int read_all_frames(capture_sys_t *capture_sys, bool dual_sink, int timeout)
+void read_with_timeout(capture_sys_t *capture_sys, bool dual_sink, int timeout)
 {
     capture_run_result_t *res = &capture_sys->run_result;
     memset(res, 0, sizeof(capture_run_result_t));
     uint32_t start_time = esp_timer_get_time() / 1000;
     uint32_t cur_time = start_time;
     int sink_num = dual_sink ? 2 : 1;
-    if (capture_sys->capture_sink[0]) {
-        esp_capture_sink_enable(capture_sys->capture_sink[0], ESP_CAPTURE_RUN_MODE_ALWAYS);
-    }
-    if (dual_sink && capture_sys->capture_sink[1]) {
-        esp_capture_sink_enable(capture_sys->capture_sink[1], ESP_CAPTURE_RUN_MODE_ALWAYS);
-    }
-    int ret = esp_capture_start(capture_sys->capture);
-    if (ret != 0) {
-        ESP_LOGE(TAG, "Fail to start capture");
-        return -1;
-    }
     while (cur_time < start_time + timeout) {
         // Following code acquire frame without wait for all supported sink
         for (int i = 0; i < sink_num; i++) {
@@ -345,7 +334,7 @@ int read_all_frames(capture_sys_t *capture_sys, bool dual_sink, int timeout)
             while (esp_capture_sink_acquire_frame(capture_sys->capture_sink[i], &frame, true) == ESP_CAPTURE_ERR_OK) {
                 res->audio_frame_count[i]++;
                 if (res->audio_frame_count[i] == 1) {
-                    ESP_LOGI(TAG, "[%d] First audio frame received", i);
+                    ESP_LOGI(TAG, "[%d] First audio frame received pts %d", i, (int)frame.pts);
                 }
                 res->audio_frame_size[i] += frame.size;
                 res->audio_pts[i] = frame.pts;
@@ -355,7 +344,7 @@ int read_all_frames(capture_sys_t *capture_sys, bool dual_sink, int timeout)
             while (esp_capture_sink_acquire_frame(capture_sys->capture_sink[i], &frame, true) == ESP_CAPTURE_ERR_OK) {
                 res->video_frame_count[i]++;
                 if (res->video_frame_count[i] == 1) {
-                    ESP_LOGI(TAG, "[%d] First video frame received", i);
+                    ESP_LOGI(TAG, "[%d] First video frame received pts %d", i, (int)frame.pts);
                 }
                 res->video_frame_size[i] += frame.size;
                 res->video_pts[i] = frame.pts;
@@ -365,7 +354,7 @@ int read_all_frames(capture_sys_t *capture_sys, bool dual_sink, int timeout)
             while (esp_capture_sink_acquire_frame(capture_sys->capture_sink[i], &frame, true) == ESP_CAPTURE_ERR_OK) {
                 res->muxer_frame_count[i]++;
                 if (res->muxer_frame_count[i] == 1) {
-                    ESP_LOGI(TAG, "[%d] First muxed frame received", i);
+                    ESP_LOGI(TAG, "[%d] First muxed frame received pts %d", i, (int)frame.pts);
                 }
                 res->muxer_frame_size[i] += frame.size;
                 res->muxer_pts[i] = frame.pts;
@@ -375,23 +364,39 @@ int read_all_frames(capture_sys_t *capture_sys, bool dual_sink, int timeout)
         vTaskDelay(10 / portTICK_PERIOD_MS);
         cur_time = esp_timer_get_time() / 1000;
     }
-    esp_capture_stop(capture_sys->capture);
 
     // Show capture results
     for (int i = 0; i < sink_num; i++) {
         if (res->audio_frame_count[i]) {
-            ESP_LOGI(TAG, "Audio Path %d frame_count:%d frame_size:%d pts:%d", i,
+            ESP_LOGW(TAG, "Audio Path %d frame_count:%d frame_size:%d pts:%d", i,
                      res->audio_frame_count[i], res->audio_frame_size[i], res->audio_pts[i]);
         }
         if (res->video_frame_count[i]) {
-            ESP_LOGI(TAG, "Video Path %d frame_count:%d frame_size:%d pts:%d", i,
+            ESP_LOGW(TAG, "Video Path %d frame_count:%d frame_size:%d pts:%d", i,
                      res->video_frame_count[i], res->video_frame_size[i], res->video_pts[i]);
         }
         if (res->muxer_frame_count[i]) {
-            ESP_LOGI(TAG, "Muxer Path %d frame_count:%d frame_size:%d pts:%d", i,
+            ESP_LOGW(TAG, "Muxer Path %d frame_count:%d frame_size:%d pts:%d", i,
                      res->muxer_frame_count[i], res->muxer_frame_size[i], res->muxer_pts[i]);
         }
     }
+}
+
+int read_all_frames(capture_sys_t *capture_sys, bool dual_sink, int timeout)
+{
+    if (capture_sys->capture_sink[0]) {
+        esp_capture_sink_enable(capture_sys->capture_sink[0], ESP_CAPTURE_RUN_MODE_ALWAYS);
+    }
+    if (dual_sink && capture_sys->capture_sink[1]) {
+        esp_capture_sink_enable(capture_sys->capture_sink[1], ESP_CAPTURE_RUN_MODE_ALWAYS);
+    }
+    int ret = esp_capture_start(capture_sys->capture);
+    if (ret != 0) {
+        ESP_LOGE(TAG, "Fail to start capture");
+        return -1;
+    }
+    read_with_timeout(capture_sys, dual_sink, timeout);
+    esp_capture_stop(capture_sys->capture);
     return 0;
 }
 

packages/esp_capture/test_apps/main/capture_builder.h

@@ -58,6 +58,8 @@ int build_advance_video_only_capture_sys(capture_sys_t *capture_sys);
 
 int build_advance_av_capture_sys(capture_sys_t *capture_sys);
 
+void read_with_timeout(capture_sys_t *capture_sys, bool dual_sink, int timeout);
+
 int read_all_frames(capture_sys_t *capture_sys, bool dual_sink, int timeout);
 
 void destroy_capture_sys(capture_sys_t *capture_sys);