Refactor: Use esp_video component (#1245)

* refactor: migrate camera module to esp-video library

* refactor: migrate boards to esp-video API (1/2)

* refactor: migrate boards to esp-video API (2/2)

* fix: use ESP-IDF 5.5

* refactor: migrate the JPEG encoder to `esp_new_jpeg`

* feat: add YUV422 support

* feat: improve pixelformat and device selection process

* feat: use ESP32-P4 Hardware JPEG Encoder

Author: laride

.github/workflows/build.yml

@@ -92,7 +92,7 @@ jobs:
         include: ${{ fromJson(needs.prepare.outputs.variants) }}
     runs-on: ubuntu-latest
     container:
-      image: espressif/idf:release-v5.4
+      image: espressif/idf:release-v5.5
     steps:
       - name: Checkout
         uses: actions/checkout@v4

main/CMakeLists.txt

@@ -24,7 +24,6 @@ set(SOURCES "audio/audio_codec.cc"
             "display/lvgl_display/gif/lvgl_gif.cc"
             "display/lvgl_display/gif/gifdec.c"
             "display/lvgl_display/jpg/image_to_jpeg.cpp"
-            "display/lvgl_display/jpg/jpeg_encoder.cpp"
             "protocols/protocol.cc"
             "protocols/mqtt_protocol.cc"
             "protocols/websocket_protocol.cc"
@@ -594,6 +593,8 @@ if(CONFIG_IDF_TARGET_ESP32)
                              "audio/codecs/es8388_audio_codec.cc"
                              "audio/codecs/es8389_audio_codec.cc"
                              "led/gpio_led.cc"
+                             "${CMAKE_CURRENT_SOURCE_DIR}/boards/common/esp32_camera.cc"
+                            "display/lvgl_display/jpg/image_to_jpeg.cpp"
                              )
 endif()
 

main/Kconfig.projbuild

@@ -635,6 +635,38 @@ config RECEIVE_CUSTOM_MESSAGE
     help
         Enable custom message reception, allow the device to receive custom messages from the server (preferably through the MQTT protocol)
 
+menu "Camera Configuration"
+    depends on !IDF_TARGET_ESP32
+
+    config XIAOZHI_ENABLE_HARDWARE_JPEG_ENCODER
+        bool "Enable Hardware JPEG Encoder"
+        default y
+        depends on SOC_JPEG_ENCODE_SUPPORTED
+        help
+            Use hardware JPEG encoder on ESP32-P4 to encode image to JPEG.
+            See https://docs.espressif.com/projects/esp-idf/en/stable/esp32p4/api-reference/peripherals/jpeg.html for more details.
+
+    config XIAOZHI_ENABLE_CAMERA_DEBUG_MODE
+        bool "Enable Camera Debug Mode"
+        default n
+        help
+            Enable camera debug mode, print camera debug information to the console.
+            Only works on boards that support camera.
+
+    config XIAOZHI_ENABLE_CAMERA_ENDIANNESS_SWAP
+        bool "Enable software camera buffer endianness swapping (USE WITH CAUTION)"
+        default n
+        depends on !CAMERA_SENSOR_SWAP_PIXEL_BYTE_ORDER
+        help
+            This option treats the camera buffer as a uint16_t[] array and performs byte-swapping (endianness conversion) on each element.
+            
+            Should only be modified by development board integration engineers.
+            
+            **Incorrect usage may result in incorrect image colors!**
+            
+            ATTENTION: If the option CAMERA_SENSOR_SWAP_PIXEL_BYTE_ORDER is available for your sensor, please use that instead.
+endmenu
+
 menu "TAIJIPAI_S3_CONFIG"
     depends on BOARD_TYPE_ESP32S3_Taiji_Pi
     choice I2S_TYPE_TAIJIPI_S3

main/boards/atk-dnesp32s3/atk_dnesp32s3.cc

@@ -133,58 +133,53 @@ class atk_dnesp32s3 : public WifiBoard {
     // 初始化摄像头：ov2640；
     // 根据正点原子官方示例参数
     void InitializeCamera() {
-        
         xl9555_->SetOutputState(OV_PWDN_IO, 0); // PWDN=低 (上电)
         xl9555_->SetOutputState(OV_RESET_IO, 0); // 确保复位
         vTaskDelay(pdMS_TO_TICKS(50));           // 延长复位保持时间
         xl9555_->SetOutputState(OV_RESET_IO, 1); // 释放复位
         vTaskDelay(pdMS_TO_TICKS(50));           // 延长 50ms
 
-        camera_config_t config = {};
-
-        config.pin_pwdn = CAM_PIN_PWDN;  // 实际由 XL9555 控制
-        config.pin_reset = CAM_PIN_RESET;// 实际由 XL9555 控制
-        config.pin_xclk = CAM_PIN_XCLK;
-        config.pin_sccb_sda = CAM_PIN_SIOD;
-        config.pin_sccb_scl = CAM_PIN_SIOC;
-
-        config.pin_d7 = CAM_PIN_D7;
-        config.pin_d6 = CAM_PIN_D6;
-        config.pin_d5 = CAM_PIN_D5;
-        config.pin_d4 = CAM_PIN_D4;
-        config.pin_d3 = CAM_PIN_D3;
-        config.pin_d2 = CAM_PIN_D2;
-        config.pin_d1 = CAM_PIN_D1;
-        config.pin_d0 = CAM_PIN_D0;
-        config.pin_vsync = CAM_PIN_VSYNC;
-        config.pin_href = CAM_PIN_HREF;
-        config.pin_pclk = CAM_PIN_PCLK;
-
-        /* XCLK 20MHz or 10MHz for OV2640 double FPS (Experimental) */
-        config.xclk_freq_hz = 24000000;
-        config.ledc_timer = LEDC_TIMER_0;
-        config.ledc_channel = LEDC_CHANNEL_0;
-
-        config.pixel_format = PIXFORMAT_RGB565;   /* YUV422,GRAYSCALE,RGB565,JPEG */
-        config.frame_size = FRAMESIZE_QVGA;       /* QQVGA-UXGA, For ESP32, do not use sizes above QVGA when not JPEG. The performance of the ESP32-S series has improved a lot, but JPEG mode always gives better frame rates */
-
-        config.jpeg_quality = 12;                 /* 0-63, for OV series camera sensors, lower number means higher quality */
-        config.fb_count = 2;                      /* When jpeg mode is used, if fb_count more than one, the driver will work in continuous mode */
-        config.fb_location = CAMERA_FB_IN_PSRAM;
-        config.grab_mode = CAMERA_GRAB_WHEN_EMPTY;
-
-        esp_err_t err = esp_camera_init(&config); // 测试相机是否存在
-        if (err != ESP_OK) {
-            ESP_LOGE(TAG, "Camera is not plugged in or not supported, error: %s", esp_err_to_name(err));
-            // 如果摄像头初始化失败，设置 camera_ 为 nullptr
-            camera_ = nullptr;
-            return;
-        }else
-        {
-            esp_camera_deinit();// 释放之前的摄像头资源,为正确初始化做准备
-            camera_ = new Esp32Camera(config);
-        }
-        
+        static esp_cam_ctlr_dvp_pin_config_t dvp_pin_config = {
+            .data_width = CAM_CTLR_DATA_WIDTH_8,
+            .data_io = {
+                [0] = CAM_PIN_D0,
+                [1] = CAM_PIN_D1,
+                [2] = CAM_PIN_D2,
+                [3] = CAM_PIN_D3,
+                [4] = CAM_PIN_D4,
+                [5] = CAM_PIN_D5,
+                [6] = CAM_PIN_D6,
+                [7] = CAM_PIN_D7,
+            },
+            .vsync_io = CAM_PIN_VSYNC,
+            .de_io = CAM_PIN_HREF,
+            .pclk_io = CAM_PIN_PCLK,
+            .xclk_io = CAM_PIN_XCLK,
+        };
+
+        esp_video_init_sccb_config_t sccb_config = {
+            .init_sccb = true,
+            .i2c_config = {
+                .port = 1,
+                .scl_pin = CAM_PIN_SIOC,
+                .sda_pin = CAM_PIN_SIOD,
+            },
+            .freq = 100000,
+        };
+
+        esp_video_init_dvp_config_t dvp_config = {
+            .sccb_config = sccb_config,
+            .reset_pin = CAM_PIN_RESET,   // 实际由 XL9555 控制
+            .pwdn_pin = CAM_PIN_PWDN,     // 实际由 XL9555 控制
+            .dvp_pin = dvp_pin_config,
+            .xclk_freq = 24000000,
+        };
+
+        esp_video_init_config_t video_config = {
+            .dvp = &dvp_config,
+        };
+
+        camera_ = new Esp32Camera(video_config);
     }
 
 public:

main/boards/atoms3r-cam-m12-echo-base/README.md

@@ -14,6 +14,12 @@ AtomS3R CAM、AtomS3R M12 是 M5Stack 推出的基于 ESP32-S3-PICO-1-N8R8 的
 
 两款开发版均**不带屏幕、不带额外按键**，需要使用语音唤醒。必要时，需要使用 `idf.py monitor` 查看 log 以确定运行状态。
 
+> ![NOTE]
+>
+> 自版本 [待定] 起，由于依赖库不支持 OV3660 传感器，AtomS3R M12 无法使用摄像头识别功能。
+>
+> AtomS3R CAM 不受影响；使用旧版本小智固件的 AtomS3R M12 不受影响。
+
 ## 配置、编译命令
 
 **配置编译目标为 ESP32S3**

main/boards/atoms3r-cam-m12-echo-base/atoms3r_cam_m12_echo_base.cc

@@ -123,36 +123,51 @@ class AtomS3rCamM12EchoBaseBoard : public WifiBoard {
 
         ESP_LOGI(TAG, "Camera Power Enabled");
 
-        vTaskDelay(pdMS_TO_TICKS(300));
+        vTaskDelay(pdMS_TO_TICKS(1000));
     }
 
-      void InitializeCamera() {
-        camera_config_t config = {};
-        config.pin_d0 = CAMERA_PIN_D0;
-        config.pin_d1 = CAMERA_PIN_D1;
-        config.pin_d2 = CAMERA_PIN_D2;
-        config.pin_d3 = CAMERA_PIN_D3;
-        config.pin_d4 = CAMERA_PIN_D4;
-        config.pin_d5 = CAMERA_PIN_D5;
-        config.pin_d6 = CAMERA_PIN_D6;
-        config.pin_d7 = CAMERA_PIN_D7;
-        config.pin_xclk = CAMERA_PIN_XCLK;
-        config.pin_pclk = CAMERA_PIN_PCLK;
-        config.pin_vsync = CAMERA_PIN_VSYNC;
-        config.pin_href = CAMERA_PIN_HREF;
-        config.pin_sccb_sda = CAMERA_PIN_SIOD;  
-        config.pin_sccb_scl = CAMERA_PIN_SIOC;
-        config.sccb_i2c_port = 1;
-        config.pin_pwdn = CAMERA_PIN_PWDN;
-        config.pin_reset = CAMERA_PIN_RESET;
-        config.xclk_freq_hz = XCLK_FREQ_HZ;
-        config.pixel_format = PIXFORMAT_RGB565;
-        config.frame_size = FRAMESIZE_QVGA;
-        config.jpeg_quality = 12;
-        config.fb_count = 1;
-        config.fb_location = CAMERA_FB_IN_PSRAM;
-        config.grab_mode = CAMERA_GRAB_WHEN_EMPTY;
-        camera_ = new Esp32Camera(config);
+    void InitializeCamera() {
+        static esp_cam_ctlr_dvp_pin_config_t dvp_pin_config = {
+            .data_width = CAM_CTLR_DATA_WIDTH_8,
+            .data_io = {
+                [0] = CAMERA_PIN_D0,
+                [1] = CAMERA_PIN_D1,
+                [2] = CAMERA_PIN_D2,
+                [3] = CAMERA_PIN_D3,
+                [4] = CAMERA_PIN_D4,
+                [5] = CAMERA_PIN_D5,
+                [6] = CAMERA_PIN_D6,
+                [7] = CAMERA_PIN_D7,
+            },
+            .vsync_io = CAMERA_PIN_VSYNC,
+            .de_io = CAMERA_PIN_HREF,
+            .pclk_io = CAMERA_PIN_PCLK,
+            .xclk_io = CAMERA_PIN_XCLK,
+        };
+
+        esp_video_init_sccb_config_t sccb_config = {
+            .init_sccb = true,
+            .i2c_config = {
+                .port = 1,
+                .scl_pin = CAMERA_PIN_SIOC,
+                .sda_pin = CAMERA_PIN_SIOD,
+            },
+            .freq = 100000,
+        };
+
+        esp_video_init_dvp_config_t dvp_config = {
+            .sccb_config = sccb_config,
+            .reset_pin = CAMERA_PIN_RESET,
+            .pwdn_pin = CAMERA_PIN_PWDN,
+            .dvp_pin = dvp_pin_config,
+            .xclk_freq = XCLK_FREQ_HZ,
+        };
+
+        esp_video_init_config_t video_config = {
+            .dvp = &dvp_config,
+        };
+
+        camera_ = new Esp32Camera(video_config);
         camera_->SetHMirror(false);
     }
 

main/boards/atoms3r-cam-m12-echo-base/config.json

@@ -5,7 +5,10 @@
             "name": "atoms3r-cam-m12-echo-base",
             "sdkconfig_append": [
                 "CONFIG_ESPTOOLPY_FLASHSIZE_8MB=y",
-                "CONFIG_PARTITION_TABLE_CUSTOM_FILENAME=\"partitions/v2/8m.csv\""
+                "CONFIG_PARTITION_TABLE_CUSTOM_FILENAME=\"partitions/v2/8m.csv\"",
+                "CONFIG_CAMERA_GC0308=y",
+                "CONFIG_CAMERA_GC0308_AUTO_DETECT_DVP_INTERFACE_SENSOR=y",
+                "CONFIG_CAMERA_GC0308_DVP_YUV422_320X240_20FPS=y"
             ]
         }
     ]

main/boards/bread-compact-wifi-s3cam/compact_wifi_board_s3cam.cc

@@ -126,32 +126,47 @@ class CompactWifiBoardS3Cam : public WifiBoard {
     }
 
     void InitializeCamera() {
-        camera_config_t config = {};
-        config.pin_d0 = CAMERA_PIN_D0;
-        config.pin_d1 = CAMERA_PIN_D1;
-        config.pin_d2 = CAMERA_PIN_D2;
-        config.pin_d3 = CAMERA_PIN_D3;
-        config.pin_d4 = CAMERA_PIN_D4;
-        config.pin_d5 = CAMERA_PIN_D5;
-        config.pin_d6 = CAMERA_PIN_D6;
-        config.pin_d7 = CAMERA_PIN_D7;
-        config.pin_xclk = CAMERA_PIN_XCLK;
-        config.pin_pclk = CAMERA_PIN_PCLK;
-        config.pin_vsync = CAMERA_PIN_VSYNC;
-        config.pin_href = CAMERA_PIN_HREF;
-        config.pin_sccb_sda = CAMERA_PIN_SIOD;  
-        config.pin_sccb_scl = CAMERA_PIN_SIOC;
-        config.sccb_i2c_port = 0;
-        config.pin_pwdn = CAMERA_PIN_PWDN;
-        config.pin_reset = CAMERA_PIN_RESET;
-        config.xclk_freq_hz = XCLK_FREQ_HZ;
-        config.pixel_format = PIXFORMAT_RGB565;
-        config.frame_size = FRAMESIZE_QVGA;
-        config.jpeg_quality = 12;
-        config.fb_count = 1;
-        config.fb_location = CAMERA_FB_IN_PSRAM;
-        config.grab_mode = CAMERA_GRAB_WHEN_EMPTY;
-        camera_ = new Esp32Camera(config);
+        static esp_cam_ctlr_dvp_pin_config_t dvp_pin_config = {
+            .data_width = CAM_CTLR_DATA_WIDTH_8,
+            .data_io = {
+                [0] = CAMERA_PIN_D0,
+                [1] = CAMERA_PIN_D1,
+                [2] = CAMERA_PIN_D2,
+                [3] = CAMERA_PIN_D3,
+                [4] = CAMERA_PIN_D4,
+                [5] = CAMERA_PIN_D5,
+                [6] = CAMERA_PIN_D6,
+                [7] = CAMERA_PIN_D7,
+            },
+            .vsync_io = CAMERA_PIN_VSYNC,
+            .de_io = CAMERA_PIN_HREF,
+            .pclk_io = CAMERA_PIN_PCLK,
+            .xclk_io = CAMERA_PIN_XCLK,
+        };
+
+        esp_video_init_sccb_config_t sccb_config = {
+            .init_sccb = true,
+            .i2c_config = {
+                .port = 0,
+                .scl_pin = CAMERA_PIN_SIOC,
+                .sda_pin = CAMERA_PIN_SIOD,
+            },
+            .freq = 100000,
+        };
+
+        esp_video_init_dvp_config_t dvp_config = {
+            .sccb_config = sccb_config,
+            .reset_pin = CAMERA_PIN_RESET,
+            .pwdn_pin = CAMERA_PIN_PWDN,
+            .dvp_pin = dvp_pin_config,
+            .xclk_freq = XCLK_FREQ_HZ,
+        };
+
+        esp_video_init_config_t video_config = {
+            .dvp = &dvp_config,
+        };
+
+        camera_ = new Esp32Camera(video_config);
         camera_->SetHMirror(false);
     }