Fixed broken person_detection example

- Bumped `esp32_s3_eye` dependency version number
- Made changes to CMakeLists.txt to fix BSP packages not getting installed
- Fixed issues with LVGL which caused the display to fail

Author: ShardulNalegave

examples/person_detection/CMakeLists.txt

@@ -3,27 +3,27 @@ cmake_minimum_required(VERSION 3.5)
 set(EXTRA_COMPONENT_DIRS static_images)
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 
-function(add_bsp SDKCONFIG BSP TARGET)
+function(add_bsp SDKCONFIG BSP)
     string(REGEX MATCH "CONFIG_${BSP}=y" REGEX_RESULT ${SDKCONFIG})
     if (REGEX_RESULT)
-        set(ENV{${BSP}} "${TARGET}")
+        set(ENV{${BSP}} 1)
     endif()
 endfunction()
 
 # 1. Define all variables used in main/idf_component.yml
-set(ENV{TFLITE_USE_BSP_S3_EYE} "false")
-set(ENV{TFLITE_USE_BSP_KORVO_2} "false")
-set(ENV{TFLITE_USE_BSP_KALUGA} "false")
+set(ENV{TFLITE_USE_BSP_S3_EYE} 0)
+set(ENV{TFLITE_USE_BSP_KORVO_2} 0)
+set(ENV{TFLITE_USE_BSP_KALUGA} 0)
 
 # 2. Set correct var to 'target'
 # This is a workaround idf-component-manager limitation, where only
 # target and idf_version can be in the if-clause
 if(EXISTS ${CMAKE_CURRENT_LIST_DIR}/sdkconfig)
     file(READ ${CMAKE_CURRENT_LIST_DIR}/sdkconfig SDKCONFIG_RULE)
 
-    add_bsp("${SDKCONFIG_RULE}" "TFLITE_USE_BSP_S3_EYE" "esp32s3")
-    add_bsp("${SDKCONFIG_RULE}" "TFLITE_USE_BSP_KORVO_2" "esp32s3")
-    add_bsp("${SDKCONFIG_RULE}" "TFLITE_USE_BSP_KALUGA" "esp32s2")
+    add_bsp("${SDKCONFIG_RULE}" "TFLITE_USE_BSP_S3_EYE")
+    add_bsp("${SDKCONFIG_RULE}" "TFLITE_USE_BSP_KORVO_2")
+    add_bsp("${SDKCONFIG_RULE}" "TFLITE_USE_BSP_KALUGA")
 endif()
 
 project(person_detection)

examples/person_detection/main/app_camera_esp.c

@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "app_camera_esp.h"
+#include "esp_log.h"
 #include "sdkconfig.h"
 
 #if (CONFIG_TFLITE_USE_BSP)
@@ -41,6 +42,8 @@ int app_camera_init() {
 
 #if (CONFIG_TFLITE_USE_BSP)
   bsp_i2c_init();
+  bsp_display_start();
+  bsp_display_backlight_on();
   camera_config_t config = BSP_CAMERA_DEFAULT_CONFIG;
 
 #else // CONFIG_TFLITE_USE_BSP
@@ -95,4 +98,4 @@ int app_camera_init() {
   ESP_LOGE(TAG, "Camera is not supported for this device!");
   return -1;
 #endif // ESP_CAMERA_SUPPORTED
-}
+}

examples/person_detection/main/detection_responder.cc

@@ -36,10 +36,8 @@ static lv_obj_t *camera_canvas = NULL;
 static lv_obj_t *person_indicator = NULL;
 static lv_obj_t *label = NULL;
 
-static void create_gui(void)
+void create_gui(void)
 {
-  bsp_display_start();
-  bsp_display_backlight_on(); // Set display brightness to 100%
   bsp_display_lock(0);
   camera_canvas = lv_canvas_create(lv_scr_act());
   assert(camera_canvas);
@@ -62,21 +60,22 @@ void RespondToDetection(float person_score, float no_person_score) {
   int person_score_int = (person_score) * 100 + 0.5;
   (void) no_person_score; // unused
 #if DISPLAY_SUPPORT
-    if (!camera_canvas) {
-      create_gui();
-    }
+  if (!camera_canvas) {
+    create_gui();
+  }
 
-    uint16_t *buf = (uint16_t *) image_provider_get_display_buf();
+  uint16_t *buf = (uint16_t *) image_provider_get_display_buf();
 
-    bsp_display_lock(0);
-    if (person_score_int < 60) { // treat score less than 60% as no person
-      lv_led_off(person_indicator);
-    } else {
-      lv_led_on(person_indicator);
-    }
-    lv_canvas_set_buffer(camera_canvas, buf, IMG_WD, IMG_HT, LV_IMG_CF_TRUE_COLOR);
-    bsp_display_unlock();
+  bsp_display_lock(0);
+  if (person_score_int < 60) { // treat score less than 60% as no person
+    lv_led_off(person_indicator);
+  } else {
+    lv_led_on(person_indicator);
+  }
+
+  lv_canvas_set_buffer(camera_canvas, buf, IMG_WD, IMG_HT, LV_COLOR_FORMAT_RGB565);
+  bsp_display_unlock();
 #endif // DISPLAY_SUPPORT
   MicroPrintf("person score:%d%%, no person score %d%%",
               person_score_int, 100 - person_score_int);
-}
+}

examples/person_detection/main/detection_responder.h

@@ -29,4 +29,7 @@ limitations under the License.
 // particular applications.
 void RespondToDetection(float person_score, float no_person_score);
 
-#endif  // TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_DETECTION_RESPONDER_H_
+// Initialize GUI components
+void create_gui();
+
+#endif  // TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_DETECTION_RESPONDER_H_

examples/person_detection/main/idf_component.yml

@@ -2,20 +2,15 @@ dependencies:
   espressif/esp-tflite-micro:
     version: "*"
     override_path: "../../../"
-
-  espressif/esp32-camera: "~2.0.5"
-
+  espressif/esp32_s2_kaluga_kit:
+    rules:
+      - if: "$TFLITE_USE_BSP_KALUGA == 1"
+    version: 3.*
   espressif/esp32_s3_eye:
-    version: "3.*"
     rules:
-      - if: "target == $TFLITE_USE_BSP_S3_EYE"
-
+      - if: "$TFLITE_USE_BSP_S3_EYE == 1"
+    version: 4.*
   espressif/esp32_s3_korvo_2:
-    version: "2.*"
-    rules:
-      - if: "target == $TFLITE_USE_BSP_KORVO_2"
-
-  espressif/esp32_s2_kaluga_kit:
-    version: "3.*"
     rules:
-      - if: "target == $TFLITE_USE_BSP_KALUGA"
+      - if: "$TFLITE_USE_BSP_KORVO_2 == 1"
+    version: 2.*

examples/person_detection/main/image_provider.cc

@@ -1,4 +1,3 @@
-
 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,16 +12,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include <cstdlib>
-#include <cstring>
-#include <iostream>
 
+#include "string.h"
 #include "freertos/FreeRTOS.h"
 #include "freertos/task.h"
+
+#include "esp_heap_caps.h"
 #include "esp_log.h"
-#include "esp_spi_flash.h"
-#include "esp_system.h"
-#include "esp_timer.h"
+#include "bsp/esp-bsp.h"
 
 #include "app_camera_esp.h"
 #include "esp_camera.h"
@@ -31,8 +28,8 @@ limitations under the License.
 #include "esp_main.h"
 
 static const char* TAG = "app_camera";
-
-static uint16_t *display_buf; // buffer to hold data to be sent to display
+static uint16_t* display_buf;
+static uint16_t* cam_buf;
 
 // Get the camera module ready
 TfLiteStatus InitCamera() {
@@ -51,6 +48,14 @@ TfLiteStatus InitCamera() {
     ESP_LOGE(TAG, "Couldn't allocate display buffer");
     return kTfLiteError;
   }
+
+  if (cam_buf == NULL) {
+    cam_buf = (uint16_t *) heap_caps_malloc(96 * 96 * sizeof(uint16_t), MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
+  }
+  if (display_buf == NULL) {
+    ESP_LOGE(TAG, "Couldn't allocate camera buffer");
+    return kTfLiteError;
+  }
 #endif // DISPLAY_SUPPORT
 
 #if ESP_CAMERA_SUPPORTED
@@ -84,13 +89,18 @@ TfLiteStatus GetImage(int image_width, int image_height, int channels, int8_t* i
   // In case if display support is enabled, we initialise camera in rgb mode
   // Hence, we need to convert this data to grayscale to send it to tf model
   // For display we extra-polate the data to 192X192
+
+  memcpy(cam_buf, fb->buf, fb->len);
+  lv_draw_sw_rgb565_swap(fb->buf, 96 * 96);
+
   for (int i = 0; i < kNumRows; i++) {
     for (int j = 0; j < kNumCols; j++) {
       uint16_t pixel = ((uint16_t *) (fb->buf))[i * kNumCols + j];
+      uint16_t inference_pixel = ((uint16_t *) (cam_buf))[i * kNumCols + j];
 
       // for inference
-      uint8_t hb = pixel & 0xFF;
-      uint8_t lb = pixel >> 8;
+      uint8_t hb = inference_pixel & 0xFF;
+      uint8_t lb = inference_pixel >> 8;
       uint8_t r = (lb & 0x1F) << 3;
       uint8_t g = ((hb & 0x07) << 5) | ((lb & 0xE0) >> 3);
       uint8_t b = (hb & 0xF8);
@@ -103,7 +113,6 @@ TfLiteStatus GetImage(int image_width, int image_height, int channels, int8_t* i
 
       image_data[i * kNumCols + j] = grey_pixel;
 
-      // to display
       display_buf[2 * i * kNumCols * 2 + 2 * j] = pixel;
       display_buf[2 * i * kNumCols * 2 + 2 * j + 1] = pixel;
       display_buf[(2 * i + 1) * kNumCols * 2 + 2 * j] = pixel;

examples/person_detection/main/main_functions.cc

@@ -32,7 +32,6 @@ limitations under the License.
 #include <esp_log.h>
 #include "esp_main.h"
 
-// Globals, used for compatibility with Arduino-style sketches.
 namespace {
 const tflite::Model* model = nullptr;
 tflite::MicroInterpreter* interpreter = nullptr;
@@ -56,7 +55,6 @@ constexpr int kTensorArenaSize = 100 * 1024 + scratchBufSize;
 static uint8_t *tensor_arena;//[kTensorArenaSize]; // Maybe we should move this to external
 }  // namespace
 
-// The name of this function is important for Arduino compatibility.
 void setup() {
   // Map the model into a usable data structure. This doesn't involve any
   // copying or parsing, it's a very lightweight operation.
@@ -68,7 +66,7 @@ void setup() {
   }
 
   if (tensor_arena == NULL) {
-    tensor_arena = (uint8_t *) heap_caps_malloc(kTensorArenaSize, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
+    tensor_arena = (uint8_t *) heap_caps_malloc(kTensorArenaSize, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
   }
   if (tensor_arena == NULL) {
     printf("Couldn't allocate memory of %d bytes\n", kTensorArenaSize);
@@ -113,11 +111,12 @@ void setup() {
     MicroPrintf("InitCamera failed\n");
     return;
   }
+
+  create_gui();
 #endif
 }
 
 #ifndef CLI_ONLY_INFERENCE
-// The name of this function is important for Arduino compatibility.
 void loop() {
   // Get image from provider.
   if (kTfLiteOk != GetImage(kNumCols, kNumRows, kNumChannels, input->data.int8)) {