Merge branch 'bugfix/fix_missing_store_vid_src_info' into 'main'

jason-mao · jason-mao · commit 41d061009b58 · 2025-06-11T17:44:34.000+08:00
gmf_video: Fix video processing stability and logging issues

See merge request adf/multimedia/esp-gmf!91
diff --git a/elements/gmf_video/esp_gmf_video_color_convert.c b/elements/gmf_video/esp_gmf_video_color_convert.c
@@ -31,10 +31,10 @@ static esp_gmf_job_err_t video_cc_el_open(esp_gmf_element_handle_t self, void *p
     esp_gmf_color_convert_hd_t *video_el = (esp_gmf_color_convert_hd_t *)self;
     // Get and check configuration
     esp_imgfx_color_convert_cfg_t *cfg = (esp_imgfx_color_convert_cfg_t *)OBJ_GET_CFG(self);
-    ESP_GMF_MEM_CHECK(TAG, cfg, return ESP_GMF_ERR_INVALID_ARG);
+    ESP_GMF_CHECK(TAG, cfg, return ESP_GMF_JOB_ERR_FAIL, "Failed to get color convert config");
     // Open color convert module
     esp_imgfx_color_convert_open(cfg, &video_el->hd);
-    ESP_GMF_MEM_CHECK(TAG, video_el->hd, return ESP_GMF_JOB_ERR_FAIL);
+    ESP_GMF_CHECK(TAG, video_el->hd, return ESP_GMF_JOB_ERR_FAIL, "Failed to create color convert handle");
     // Get video size
     esp_imgfx_get_image_size(cfg->in_pixel_fmt, &cfg->in_res, (uint32_t *)&(ESP_GMF_ELEMENT_GET(video_el)->in_attr.data_size));
     esp_imgfx_get_image_size(cfg->out_pixel_fmt, &cfg->in_res, (uint32_t *)&(ESP_GMF_ELEMENT_GET(video_el)->out_attr.data_size));
@@ -232,6 +232,7 @@ static esp_gmf_err_t video_cc_el_received_event_handler(esp_gmf_event_pkt_t *evt
     esp_gmf_event_state_t state = ESP_GMF_EVENT_STATE_NONE;
     esp_gmf_element_get_state(self, &state);
     esp_gmf_info_video_t *info = (esp_gmf_info_video_t *)evt->payload;
+    esp_gmf_video_el_set_src_info(self, info);
     esp_imgfx_color_convert_cfg_t *config = (esp_imgfx_color_convert_cfg_t *)OBJ_GET_CFG(self);
     esp_gmf_color_convert_hd_t *video_el = (esp_gmf_color_convert_hd_t *)self;
     GMF_VIDEO_UPDATE_CONFIG(config, info, video_el->need_recfg);
diff --git a/elements/gmf_video/esp_gmf_video_crop.c b/elements/gmf_video/esp_gmf_video_crop.c
@@ -32,10 +32,10 @@ static esp_gmf_job_err_t video_crop_el_open(esp_gmf_element_handle_t self, void
     esp_gmf_crop_hd_t *video_el = (esp_gmf_crop_hd_t *)self;
     // Get and check config
     esp_imgfx_crop_cfg_t *cfg = (esp_imgfx_crop_cfg_t *)OBJ_GET_CFG(self);
-    ESP_GMF_MEM_CHECK(TAG, cfg, return ESP_GMF_ERR_INVALID_ARG);
+    ESP_GMF_CHECK(TAG, cfg, return ESP_GMF_JOB_ERR_FAIL, "Failed to get crop config");
     // Open crop module
     esp_imgfx_crop_open(cfg, &video_el->hd);
-    ESP_GMF_MEM_CHECK(TAG, video_el->hd, return ESP_GMF_JOB_ERR_FAIL);
+    ESP_GMF_CHECK(TAG, video_el->hd, return ESP_GMF_JOB_ERR_FAIL, "Failed to create crop handle");
     // Get video size
     esp_imgfx_get_image_size(cfg->in_pixel_fmt, &cfg->in_res, (uint32_t *)&(ESP_GMF_ELEMENT_GET(video_el)->in_attr.data_size));
     esp_imgfx_get_image_size(cfg->in_pixel_fmt, &cfg->cropped_res, (uint32_t *)&(ESP_GMF_ELEMENT_GET(video_el)->out_attr.data_size));
@@ -240,6 +240,7 @@ static esp_gmf_err_t video_crop_el_received_event_handler(esp_gmf_event_pkt_t *e
     esp_gmf_event_state_t state = ESP_GMF_EVENT_STATE_NONE;
     esp_gmf_element_get_state(self, &state);
     esp_gmf_info_video_t *info = (esp_gmf_info_video_t *)evt->payload;
+    esp_gmf_video_el_set_src_info(self, info);
     esp_imgfx_crop_cfg_t *config = (esp_imgfx_crop_cfg_t *)OBJ_GET_CFG(self);
     esp_gmf_crop_hd_t *video_el = (esp_gmf_crop_hd_t *)self;
     GMF_VIDEO_UPDATE_CONFIG(config, info, video_el->need_recfg);
diff --git a/elements/gmf_video/esp_gmf_video_rotate.c b/elements/gmf_video/esp_gmf_video_rotate.c
@@ -31,10 +31,10 @@ static esp_gmf_job_err_t video_rotate_el_open(esp_gmf_element_handle_t self, voi
     esp_gmf_rotate_hd_t *video_el = (esp_gmf_rotate_hd_t *)self;
     // Get and check config
     esp_imgfx_rotate_cfg_t *cfg = (esp_imgfx_rotate_cfg_t *)OBJ_GET_CFG(self);
-    ESP_GMF_MEM_CHECK(TAG, cfg, return ESP_GMF_ERR_INVALID_ARG);
+    ESP_GMF_CHECK(TAG, cfg, return ESP_GMF_JOB_ERR_FAIL, "Failed to get rotate config");
     // Open rotate module
     esp_imgfx_rotate_open(cfg, &video_el->hd);
-    ESP_GMF_MEM_CHECK(TAG, video_el->hd, return ESP_GMF_JOB_ERR_FAIL);
+    ESP_GMF_CHECK(TAG, video_el->hd, return ESP_GMF_JOB_ERR_FAIL, "Failed to create rotate handle");
     // Get video size
     esp_imgfx_get_image_size(cfg->in_pixel_fmt, &cfg->in_res, (uint32_t *)&(ESP_GMF_ELEMENT_GET(video_el)->in_attr.data_size));
     esp_imgfx_resolution_t res;
@@ -235,6 +235,7 @@ static esp_gmf_err_t video_rotate_el_received_event_handler(esp_gmf_event_pkt_t
     esp_gmf_event_state_t state = ESP_GMF_EVENT_STATE_NONE;
     esp_gmf_element_get_state(self, &state);
     esp_gmf_info_video_t *info = (esp_gmf_info_video_t *)evt->payload;
+    esp_gmf_video_el_set_src_info(self, info);
     esp_imgfx_rotate_cfg_t *config = (esp_imgfx_rotate_cfg_t *)OBJ_GET_CFG(self);
     esp_gmf_rotate_hd_t *video_el = (esp_gmf_rotate_hd_t *)self;
     GMF_VIDEO_UPDATE_CONFIG(config, info, video_el->need_recfg);
diff --git a/elements/gmf_video/esp_gmf_video_scale.c b/elements/gmf_video/esp_gmf_video_scale.c
@@ -32,10 +32,10 @@ static esp_gmf_job_err_t video_scale_el_open(esp_gmf_element_handle_t self, void
     esp_gmf_scale_hd_t *video_el = (esp_gmf_scale_hd_t *)self;
     // Get and check config
     esp_imgfx_scale_cfg_t *cfg = (esp_imgfx_scale_cfg_t *)OBJ_GET_CFG(self);
-    ESP_GMF_MEM_CHECK(TAG, cfg, return ESP_GMF_ERR_INVALID_ARG);
+    ESP_GMF_CHECK(TAG, cfg, return ESP_GMF_JOB_ERR_FAIL, "Failed to get scale config");
     // Open scale module
     esp_imgfx_scale_open(cfg, &video_el->hd);
-    ESP_GMF_MEM_CHECK(TAG, video_el->hd, return ESP_GMF_JOB_ERR_FAIL);
+    ESP_GMF_CHECK(TAG, video_el->hd, return ESP_GMF_JOB_ERR_FAIL, "Failed to create scale handle");
     // Get video size
     esp_imgfx_get_image_size(cfg->in_pixel_fmt, &cfg->in_res, (uint32_t *)&(ESP_GMF_ELEMENT_GET(video_el)->in_attr.data_size));
     esp_imgfx_get_image_size(cfg->in_pixel_fmt, &cfg->scale_res, (uint32_t *)&(ESP_GMF_ELEMENT_GET(video_el)->out_attr.data_size));
@@ -235,6 +235,7 @@ static esp_gmf_err_t video_scale_el_received_event_handler(esp_gmf_event_pkt_t *
     esp_gmf_event_state_t state = ESP_GMF_EVENT_STATE_NONE;
     esp_gmf_element_get_state(self, &state);
     esp_gmf_info_video_t *info = (esp_gmf_info_video_t *)evt->payload;
+    esp_gmf_video_el_set_src_info(self, info);
     esp_imgfx_scale_cfg_t *config = (esp_imgfx_scale_cfg_t *)OBJ_GET_CFG(self);
     esp_gmf_scale_hd_t *video_el = (esp_gmf_scale_hd_t *)self;
     GMF_VIDEO_UPDATE_CONFIG(config, info, video_el->need_recfg);
diff --git a/elements/gmf_video/private_include/gmf_video_common.h b/elements/gmf_video/private_include/gmf_video_common.h
@@ -115,7 +115,7 @@ static inline esp_gmf_job_err_t video_el_acquire_payload(esp_gmf_port_handle_t i
         if (load_ret == ESP_GMF_IO_ABORT) {
             return ESP_GMF_JOB_ERR_OK;
         }
-        ESP_LOGE("Imgfx_el", "Acquire size: %ld-%d on in port, ret: %d", in_wanted_size, (*in_load)->valid_size, load_ret);
+        ESP_LOGE("Imgfx_el", "Acquire size: %ld on in port, ret: %d", in_wanted_size, load_ret);
         return ESP_GMF_JOB_ERR_FAIL;
     }
     if (((*in_load)->valid_size) < (in_wanted_size)) {
diff --git a/elements/test_apps/main/elements/gmf_image_effects_test.c b/elements/test_apps/main/elements/gmf_image_effects_test.c
@@ -8,6 +8,7 @@
 #include "freertos/FreeRTOS.h"
 #include "freertos/event_groups.h"
 #include "esp_gmf_data_bus.h"
+#include "esp_gmf_video_element.h"
 #include "esp_gmf_video_scale.h"
 #include "esp_gmf_video_crop.h"
 #include "esp_gmf_video_rotate.h"
@@ -67,6 +68,55 @@ static esp_gmf_err_t esp_gmf_create_obj_cfg_pool()
     return ret;
 }
 
+static esp_gmf_err_t event_cb(esp_gmf_event_pkt_t *evt, void *ctx)
+{
+    ESP_GMF_NULL_CHECK(TAG, ctx, return ESP_GMF_ERR_INVALID_ARG);
+    ESP_GMF_NULL_CHECK(TAG, evt, return ESP_GMF_ERR_INVALID_ARG);
+    if ((evt->type != ESP_GMF_EVT_TYPE_REPORT_INFO)
+        || (evt->sub != ESP_GMF_INFO_VIDEO)
+        || (evt->payload == NULL)) {
+        return ESP_GMF_ERR_OK;
+    }
+    esp_gmf_info_video_t *info = (esp_gmf_info_video_t *)evt->payload;
+    // Check received info
+    esp_gmf_info_video_t get_info = {0};
+    esp_gmf_video_el_get_src_info(ctx, &get_info);
+    TEST_ASSERT_EQUAL(info->fps, get_info.fps);
+    TEST_ASSERT_EQUAL(info->bitrate, get_info.bitrate);
+    const char *tag = OBJ_GET_TAG(ctx);
+    if (strcmp(tag, "imgfx_scale") == 0) {
+        esp_imgfx_scale_cfg_t *cfg = (esp_imgfx_scale_cfg_t *)OBJ_GET_CFG(ctx);
+        TEST_ASSERT_EQUAL(info->width, cfg->scale_res.width);
+        TEST_ASSERT_EQUAL(info->height, cfg->scale_res.height);
+        TEST_ASSERT_EQUAL(info->format_id, cfg->in_pixel_fmt);
+    } else if (strcmp(tag, "imgfx_crop") == 0) {
+        esp_imgfx_crop_cfg_t *cfg = (esp_imgfx_crop_cfg_t *)OBJ_GET_CFG(ctx);
+        TEST_ASSERT_EQUAL(info->width, cfg->cropped_res.width);
+        TEST_ASSERT_EQUAL(info->height, cfg->cropped_res.height);
+        TEST_ASSERT_EQUAL(info->format_id, cfg->in_pixel_fmt);
+    } else if (strcmp(tag, "imgfx_rotate") == 0) {
+        esp_imgfx_rotate_cfg_t *cfg = (esp_imgfx_rotate_cfg_t *)OBJ_GET_CFG(ctx);
+        if (cfg->degree % 90 == 0
+            || cfg->degree % 270 == 0) {
+            TEST_ASSERT_EQUAL(info->width, cfg->in_res.height);
+            TEST_ASSERT_EQUAL(info->height, cfg->in_res.width);
+        } else {
+            TEST_ASSERT_EQUAL(info->width, cfg->in_res.width);
+            TEST_ASSERT_EQUAL(info->height, cfg->in_res.height);
+        }
+        TEST_ASSERT_EQUAL(info->format_id, cfg->in_pixel_fmt);
+    } else if (strcmp(tag, "imgfx_color_convert") == 0) {
+        esp_imgfx_color_convert_cfg_t *cfg = (esp_imgfx_color_convert_cfg_t *)OBJ_GET_CFG(ctx);
+        TEST_ASSERT_EQUAL(info->width, cfg->in_res.width);
+        TEST_ASSERT_EQUAL(info->height, cfg->in_res.height);
+        TEST_ASSERT_EQUAL(info->format_id, cfg->out_pixel_fmt);
+    } else {
+        return ESP_GMF_ERR_FAIL;
+    }
+    ESP_LOGI(TAG, "Event callback");
+    return ESP_GMF_ERR_OK;
+}
+
 static esp_gmf_err_io_t imgfx_acquire_read(void *handle, esp_gmf_data_bus_block_t *blk, int wanted_size, int block_ticks)
 {
     blk->buf = test_buffer;
@@ -101,6 +151,20 @@ TEST_CASE("Test for all software video effects", "[ESP_GMF_Effects]")
     esp_gmf_port_handle_t in_port = NULL;
     esp_gmf_port_handle_t out_port = NULL;
     esp_gmf_obj_handle_t obj_hd = {NULL};
+    esp_gmf_info_video_t info = {
+        .fps = 30,
+        .width = 640,
+        .height = 480,
+        .format_id = ESP_IMGFX_PIXEL_FMT_RGB888,
+        .bitrate = 1000000,
+    };
+    esp_gmf_event_pkt_t event_pkt = {
+        .from = NULL,
+        .type = ESP_GMF_EVT_TYPE_REPORT_INFO,
+        .sub = ESP_GMF_INFO_VIDEO,
+        .payload = &info,
+        .payload_size = sizeof(info),
+    };
     // Create obj handle and cfg pool
     TEST_ASSERT_EQUAL(ESP_GMF_ERR_OK, esp_gmf_create_obj_cfg_pool());
     test_buffer = (uint8_t *)malloc(TEST_MAX_IMG_LENGTH);
@@ -121,9 +185,21 @@ TEST_CASE("Test for all software video effects", "[ESP_GMF_Effects]")
             esp_gmf_element_register_in_port(obj_hd, in_port);
             esp_gmf_element_register_out_port(obj_hd, out_port);
             for (size_t i = 0; i < 4; i++) {
+                // set event callback
+                esp_gmf_element_set_event_func(obj_hd, event_cb, obj_hd);
+                // report info
+                TEST_ASSERT_EQUAL(ESP_GMF_ERR_OK, esp_gmf_element_receive_event(obj_hd, &event_pkt, NULL));
                 // Open obj
                 TEST_ASSERT_EQUAL(ESP_GMF_ERR_OK, esp_gmf_element_process_open(obj_hd, NULL));
-                // Run obj
+                // Check received info
+                esp_gmf_info_video_t get_info = {0};
+                esp_gmf_video_el_get_src_info(obj_hd, &get_info);
+                TEST_ASSERT_EQUAL(info.fps, get_info.fps);
+                TEST_ASSERT_EQUAL(info.width, get_info.width);
+                TEST_ASSERT_EQUAL(info.height, get_info.height);
+                TEST_ASSERT_EQUAL(info.format_id, get_info.format_id);
+                TEST_ASSERT_EQUAL(info.bitrate, get_info.bitrate);
+                // Run
                 TEST_ASSERT_EQUAL(ESP_GMF_ERR_OK, esp_gmf_element_process_running(obj_hd, NULL));
                 TEST_ASSERT_EQUAL(ESP_GMF_ERR_OK, esp_gmf_element_process_running(obj_hd, NULL));
                 // Close obj
@@ -140,3 +216,50 @@ TEST_CASE("Test for all software video effects", "[ESP_GMF_Effects]")
     esp_gmf_destory_obj_cfg_pool();
     ESP_GMF_MEM_SHOW(TAG);
 }
+
+TEST_CASE("ERROR: Input load is NULL", "[ESP_GMF_Effects]")
+{
+    esp_log_level_set("*", ESP_LOG_INFO);
+    ESP_GMF_MEM_SHOW(TAG);
+    esp_gmf_port_handle_t in_port = NULL;
+    esp_gmf_port_handle_t out_port = NULL;
+    esp_gmf_obj_handle_t obj_hd = {NULL};
+    // Create obj handle and cfg pool
+    TEST_ASSERT_EQUAL(ESP_GMF_ERR_OK, esp_gmf_create_obj_cfg_pool());
+    test_buffer = (uint8_t *)malloc(TEST_MAX_IMG_LENGTH);
+    TEST_ASSERT_NOT_EQUAL(NULL, test_buffer);
+    test_buffer1 = (uint8_t *)malloc(TEST_MAX_IMG_LENGTH);
+    TEST_ASSERT_NOT_EQUAL(NULL, test_buffer1);
+
+    for (size_t el_index = 0; el_index < TEST_MAX_EL_NUM; el_index++) {
+        // New obj
+        TEST_ASSERT_EQUAL(ESP_GMF_ERR_OK, esp_gmf_obj_dupl(obj_hd_pool[el_index], &obj_hd));
+        // Print obj tag
+        ESP_LOGE(TAG, "%s-%d,obj_hd:%p", (OBJ_GET_TAG(obj_hd)), __LINE__, obj_hd);
+        // Create in/out port
+        in_port = NEW_ESP_GMF_PORT_IN_BLOCK(NULL, NULL, NULL, NULL, TEST_MAX_IMG_LENGTH, TEST_TICK_DELAY);
+        out_port = NEW_ESP_GMF_PORT_OUT_BLOCK(NULL, NULL, NULL, NULL, TEST_MAX_IMG_LENGTH, TEST_TICK_DELAY);
+
+        uint32_t fake_reader_handle;
+        esp_gmf_port_set_writer(in_port, (void *)&fake_reader_handle);
+        // Register in/out port
+        esp_gmf_element_register_in_port(obj_hd, in_port);
+        esp_gmf_element_register_out_port(obj_hd, out_port);
+        for (size_t i = 0; i < 4; i++) {
+            // Open obj
+            TEST_ASSERT_EQUAL(ESP_GMF_ERR_OK, esp_gmf_element_process_open(obj_hd, NULL));
+            // Run obj
+            TEST_ASSERT_EQUAL(ESP_GMF_ERR_FAIL, esp_gmf_element_process_running(obj_hd, NULL));
+            // Close obj
+            TEST_ASSERT_EQUAL(ESP_GMF_ERR_OK, esp_gmf_element_process_close(obj_hd, NULL));
+        }
+        // Delete obj: port will be deleted when obj handle pool is destroyed
+        TEST_ASSERT_EQUAL(ESP_GMF_ERR_OK, esp_gmf_obj_delete(obj_hd));
+    }
+    // free buffer
+    free(test_buffer);
+    free(test_buffer1);
+    // Destroy obj handle and cfg pool
+    esp_gmf_destory_obj_cfg_pool();
+    ESP_GMF_MEM_SHOW(TAG);
+}

Original file line number	Diff line number	Diff line change
`@@ -115,7 +115,7 @@ static inline esp_gmf_job_err_t video_el_acquire_payload(esp_gmf_port_handle_t i`
`115`	`115`	`if (load_ret == ESP_GMF_IO_ABORT) {`
`116`	`116`	`return ESP_GMF_JOB_ERR_OK;`
`117`	`117`	`}`
`118`		`- ESP_LOGE("Imgfx_el", "Acquire size: %ld-%d on in port, ret: %d", in_wanted_size, (*in_load)->valid_size, load_ret);`
	`118`	`+ ESP_LOGE("Imgfx_el", "Acquire size: %ld on in port, ret: %d", in_wanted_size, load_ret);`
`119`	`119`	`return ESP_GMF_JOB_ERR_FAIL;`
`120`	`120`	`}`
`121`	`121`	`if (((*in_load)->valid_size) < (in_wanted_size)) {`