Fixed broken person_detection example

- Bumped esp32_s3_eye dependency version number
- Made changes to CMakeLists.txt to fix BSP packages not getting installed
- Fixed issues with LVGL which caused the display to fail

Author: ShardulNalegave

examples/person_detection/CMakeLists.txt

@@ -3,27 +3,28 @@ cmake_minimum_required(VERSION 3.5)
 set(EXTRA_COMPONENT_DIRS static_images)
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 
-function(add_bsp SDKCONFIG BSP TARGET)
+# Helper function to set required environment variables as per configuration set by user
+function(add_bsp SDKCONFIG BSP)
     string(REGEX MATCH "CONFIG_${BSP}=y" REGEX_RESULT ${SDKCONFIG})
     if (REGEX_RESULT)
-        set(ENV{${BSP}} "${TARGET}")
+        set(ENV{${BSP}} 1) # If the config option is set then set corresponding environment variable to 1
     endif()
 endfunction()
 
-# 1. Define all variables used in main/idf_component.yml
-set(ENV{TFLITE_USE_BSP_S3_EYE} "false")
-set(ENV{TFLITE_USE_BSP_KORVO_2} "false")
-set(ENV{TFLITE_USE_BSP_KALUGA} "false")
+# 1. Define all variables used in main/idf_component.yml and set them to zero
+set(ENV{TFLITE_USE_BSP_S3_EYE} 0)
+set(ENV{TFLITE_USE_BSP_KORVO_2} 0)
+set(ENV{TFLITE_USE_BSP_KALUGA} 0)
 
-# 2. Set correct var to 'target'
+# 2. Set correct var to 1
 # This is a workaround idf-component-manager limitation, where only
-# target and idf_version can be in the if-clause
+# target, idf_version and environment variables can be in the if-clause
 if(EXISTS ${CMAKE_CURRENT_LIST_DIR}/sdkconfig)
     file(READ ${CMAKE_CURRENT_LIST_DIR}/sdkconfig SDKCONFIG_RULE)
 
-    add_bsp("${SDKCONFIG_RULE}" "TFLITE_USE_BSP_S3_EYE" "esp32s3")
-    add_bsp("${SDKCONFIG_RULE}" "TFLITE_USE_BSP_KORVO_2" "esp32s3")
-    add_bsp("${SDKCONFIG_RULE}" "TFLITE_USE_BSP_KALUGA" "esp32s2")
+    add_bsp("${SDKCONFIG_RULE}" "TFLITE_USE_BSP_S3_EYE") # Check for CONFIG_TFLITE_USE_BSP_S3_EYE option in sdkconfig
+    add_bsp("${SDKCONFIG_RULE}" "TFLITE_USE_BSP_KORVO_2") # Check for CONFIG_TFLITE_USE_BSP_KORVO_2 option in sdkconfig
+    add_bsp("${SDKCONFIG_RULE}" "TFLITE_USE_BSP_KALUGA") # Check for CONFIG_TFLITE_USE_BSP_KALUGA option in sdkconfig
 endif()
 
 project(person_detection)

examples/person_detection/main/app_camera_esp.c

@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "app_camera_esp.h"
+#include "esp_log.h"
 #include "sdkconfig.h"
 
 #if (CONFIG_TFLITE_USE_BSP)

examples/person_detection/main/detection_responder.cc

@@ -36,10 +36,26 @@ static lv_obj_t *camera_canvas = NULL;
 static lv_obj_t *person_indicator = NULL;
 static lv_obj_t *label = NULL;
 
-static void create_gui(void)
+void create_gui(void)
 {
-  bsp_display_start();
-  bsp_display_backlight_on(); // Set display brightness to 100%
+  bsp_display_cfg_t cfg = {
+    .lvgl_port_cfg = {
+      .task_priority = CONFIG_BSP_DISPLAY_LVGL_TASK_PRIORITY,
+      .task_stack = 6144,
+      .task_affinity = 1,
+      .task_max_sleep_ms = CONFIG_BSP_DISPLAY_LVGL_MAX_SLEEP,
+      .timer_period_ms = CONFIG_BSP_DISPLAY_LVGL_TICK,
+    },
+    .buffer_size = BSP_LCD_DRAW_BUFF_SIZE,
+    .double_buffer = BSP_LCD_DRAW_BUFF_DOUBLE,
+    .flags = {
+      .buff_dma = false,
+      .buff_spiram = true,
+    }
+  };
+  bsp_display_start_with_config(&cfg);
+  bsp_display_backlight_on();
+
   bsp_display_lock(0);
   camera_canvas = lv_canvas_create(lv_scr_act());
   assert(camera_canvas);
@@ -62,20 +78,21 @@ void RespondToDetection(float person_score, float no_person_score) {
   int person_score_int = (person_score) * 100 + 0.5;
   (void) no_person_score; // unused
 #if DISPLAY_SUPPORT
-    if (!camera_canvas) {
-      create_gui();
-    }
+  if (!camera_canvas) {
+    create_gui();
+  }
 
-    uint16_t *buf = (uint16_t *) image_provider_get_display_buf();
+  uint16_t *buf = (uint16_t *) image_provider_get_display_buf();
 
-    bsp_display_lock(0);
-    if (person_score_int < 60) { // treat score less than 60% as no person
-      lv_led_off(person_indicator);
-    } else {
-      lv_led_on(person_indicator);
-    }
-    lv_canvas_set_buffer(camera_canvas, buf, IMG_WD, IMG_HT, LV_IMG_CF_TRUE_COLOR);
-    bsp_display_unlock();
+  bsp_display_lock(0);
+  if (person_score_int < 60) { // treat score less than 60% as no person
+    lv_led_off(person_indicator);
+  } else {
+    lv_led_on(person_indicator);
+  }
+
+  lv_canvas_set_buffer(camera_canvas, buf, IMG_WD, IMG_HT, LV_COLOR_FORMAT_RGB565);
+  bsp_display_unlock();
 #endif // DISPLAY_SUPPORT
   MicroPrintf("person score:%d%%, no person score %d%%",
               person_score_int, 100 - person_score_int);

examples/person_detection/main/detection_responder.h

@@ -29,4 +29,7 @@ limitations under the License.
 // particular applications.
 void RespondToDetection(float person_score, float no_person_score);
 
+// Initialize GUI components
+void create_gui();
+
 #endif  // TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_DETECTION_RESPONDER_H_

examples/person_detection/main/esp_cli.c

@@ -24,6 +24,7 @@
 #include "esp_cli.h"
 #include "esp_timer.h"
 
+#if CLI_ONLY_INFERENCE
 #define IMAGE_COUNT 10
 static uint8_t *image_database[IMAGE_COUNT];
 
@@ -38,6 +39,7 @@ extern const uint8_t image6_start[]   asm("_binary_image6_start");
 extern const uint8_t image7_start[]   asm("_binary_image7_start");
 extern const uint8_t image8_start[]   asm("_binary_image8_start");
 extern const uint8_t image9_start[]   asm("_binary_image9_start");
+#endif
 
 static const char *TAG = "[esp_cli]";
 
@@ -96,6 +98,7 @@ static int mem_dump_cli_handler(int argc, char *argv[])
     return 0;
 }
 
+#if CLI_ONLY_INFERENCE
 static int inference_cli_handler(int argc, char *argv[])
 {
     /* Just to go to the next line */
@@ -121,6 +124,18 @@ static int inference_cli_handler(int argc, char *argv[])
     return 0;
 }
 
+int esp_cli_register_inference_command() {
+    esp_console_cmd_t command = {
+        .command = "detect_image",
+        .help = "detect_image <image_number>"
+                "Note: image numbers ranging from 0 - 9 only are valid",
+        .func = inference_cli_handler,
+    };
+    esp_console_cmd_register(&command);
+    return 0;
+}
+#endif
+
 static esp_console_cmd_t diag_cmds[] = {
     {
         .command = "mem-dump",
@@ -137,12 +152,6 @@ static esp_console_cmd_t diag_cmds[] = {
         .help = "",
         .func = cpu_dump_cli_handler,
     },
-    {
-        .command = "detect_image",
-        .help = "detect_image <image_number>"
-                "Note: image numbers ranging from 0 - 9 only are valid",
-        .func = inference_cli_handler,
-    },
 };
 
 int esp_cli_register_cmds()
@@ -158,6 +167,7 @@ int esp_cli_register_cmds()
 
 static void image_database_init()
 {
+#if CLI_ONLY_INFERENCE
     image_database[0] = (uint8_t *) image0_start;
     image_database[1] = (uint8_t *) image1_start;
     image_database[2] = (uint8_t *) image2_start;
@@ -168,7 +178,7 @@ static void image_database_init()
     image_database[7] = (uint8_t *) image7_start;
     image_database[8] = (uint8_t *) image8_start;
     image_database[9] = (uint8_t *) image9_start;
-
+#endif
 }
 
 int esp_cli_start()

examples/person_detection/main/esp_cli.h

@@ -19,6 +19,7 @@ extern "C" {
 #endif
 
 int esp_cli_start();
+int esp_cli_register_inference_command();
 
 #ifdef __cplusplus
 }

examples/person_detection/main/idf_component.yml

@@ -2,20 +2,15 @@ dependencies:
   espressif/esp-tflite-micro:
     version: "*"
     override_path: "../../../"
-
-  espressif/esp32-camera: "~2.0.5"
-
+  espressif/esp32_s2_kaluga_kit:
+    rules:
+      - if: "$TFLITE_USE_BSP_KALUGA == 1"
+    version: 3.*
   espressif/esp32_s3_eye:
-    version: "3.*"
     rules:
-      - if: "target == $TFLITE_USE_BSP_S3_EYE"
-
+      - if: "$TFLITE_USE_BSP_S3_EYE == 1"
+    version: 4.*
   espressif/esp32_s3_korvo_2:
-    version: "2.*"
-    rules:
-      - if: "target == $TFLITE_USE_BSP_KORVO_2"
-
-  espressif/esp32_s2_kaluga_kit:
-    version: "3.*"
     rules:
-      - if: "target == $TFLITE_USE_BSP_KALUGA"
+      - if: "$TFLITE_USE_BSP_KORVO_2 == 1"
+    version: 2.*

examples/person_detection/main/image_provider.cc

@@ -1,4 +1,3 @@
-
 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,16 +12,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include <cstdlib>
-#include <cstring>
-#include <iostream>
 
+#include "string.h"
 #include "freertos/FreeRTOS.h"
 #include "freertos/task.h"
+
+#include "esp_heap_caps.h"
 #include "esp_log.h"
-#include "esp_spi_flash.h"
-#include "esp_system.h"
-#include "esp_timer.h"
+#include "bsp/esp-bsp.h"
 
 #include "app_camera_esp.h"
 #include "esp_camera.h"
@@ -31,8 +28,7 @@ limitations under the License.
 #include "esp_main.h"
 
 static const char* TAG = "app_camera";
-
-static uint16_t *display_buf; // buffer to hold data to be sent to display
+static uint16_t* display_buf;
 
 // Get the camera module ready
 TfLiteStatus InitCamera() {
@@ -84,13 +80,19 @@ TfLiteStatus GetImage(int image_width, int image_height, int channels, int8_t* i
   // In case if display support is enabled, we initialise camera in rgb mode
   // Hence, we need to convert this data to grayscale to send it to tf model
   // For display we extra-polate the data to 192X192
+
+  // point to the last quarter of buffer
+  uint16_t* cam_buf = display_buf + (96 * 96 * 3);
+  memcpy((uint8_t*)cam_buf, fb->buf, fb->len);
+  esp_camera_fb_return(fb);
+
   for (int i = 0; i < kNumRows; i++) {
     for (int j = 0; j < kNumCols; j++) {
-      uint16_t pixel = ((uint16_t *) (fb->buf))[i * kNumCols + j];
+      uint16_t inference_pixel = cam_buf[i * kNumCols + j];
 
       // for inference
-      uint8_t hb = pixel & 0xFF;
-      uint8_t lb = pixel >> 8;
+      uint8_t hb = inference_pixel & 0xFF;
+      uint8_t lb = inference_pixel >> 8;
       uint8_t r = (lb & 0x1F) << 3;
       uint8_t g = ((hb & 0x07) << 5) | ((lb & 0xE0) >> 3);
       uint8_t b = (hb & 0xF8);
@@ -102,8 +104,14 @@ TfLiteStatus GetImage(int image_width, int image_height, int channels, int8_t* i
       int8_t grey_pixel = ((305 * r + 600 * g + 119 * b) >> 10) - 128;
 
       image_data[i * kNumCols + j] = grey_pixel;
+    }
+  }
 
-      // to display
+  // for display
+  lv_draw_sw_rgb565_swap(cam_buf, 96 * 96);
+  for (int i = 0; i < kNumRows; i++) {
+    for (int j = 0; j < kNumCols; j++) {
+      uint16_t pixel = cam_buf[i * kNumCols + j];
       display_buf[2 * i * kNumCols * 2 + 2 * j] = pixel;
       display_buf[2 * i * kNumCols * 2 + 2 * j + 1] = pixel;
       display_buf[(2 * i + 1) * kNumCols * 2 + 2 * j] = pixel;
@@ -118,8 +126,6 @@ TfLiteStatus GetImage(int image_width, int image_height, int channels, int8_t* i
     image_data[i] = ((uint8_t *) fb->buf)[i] ^ 0x80;
   }
 #endif // DISPLAY_SUPPORT
-
-  esp_camera_fb_return(fb);
   /* here the esp camera can give you grayscale image directly */
   return kTfLiteOk;
 #else

examples/person_detection/main/main.cc

@@ -20,15 +20,13 @@ limitations under the License.
 #include "freertos/task.h"
 
 #include "esp_main.h"
-
-#if CLI_ONLY_INFERENCE
 #include "esp_cli.h"
-#endif
 
 void tf_main(void) {
   setup();
-#if CLI_ONLY_INFERENCE
   esp_cli_start();
+#if CLI_ONLY_INFERENCE
+  esp_cli_register_inference_command();
   vTaskDelay(portMAX_DELAY);
 #else
   while (true) {

examples/person_detection/main/main_functions.cc

@@ -32,7 +32,6 @@ limitations under the License.
 #include <esp_log.h>
 #include "esp_main.h"
 
-// Globals, used for compatibility with Arduino-style sketches.
 namespace {
 const tflite::Model* model = nullptr;
 tflite::MicroInterpreter* interpreter = nullptr;
@@ -56,7 +55,6 @@ constexpr int kTensorArenaSize = 100 * 1024 + scratchBufSize;
 static uint8_t *tensor_arena;//[kTensorArenaSize]; // Maybe we should move this to external
 }  // namespace
 
-// The name of this function is important for Arduino compatibility.
 void setup() {
   // Map the model into a usable data structure. This doesn't involve any
   // copying or parsing, it's a very lightweight operation.
@@ -68,7 +66,7 @@ void setup() {
   }
 
   if (tensor_arena == NULL) {
-    tensor_arena = (uint8_t *) heap_caps_malloc(kTensorArenaSize, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
+    tensor_arena = (uint8_t *) heap_caps_malloc(kTensorArenaSize, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
   }
   if (tensor_arena == NULL) {
     printf("Couldn't allocate memory of %d bytes\n", kTensorArenaSize);
@@ -113,11 +111,12 @@ void setup() {
     MicroPrintf("InitCamera failed\n");
     return;
   }
+
+  create_gui();
 #endif
 }
 
 #ifndef CLI_ONLY_INFERENCE
-// The name of this function is important for Arduino compatibility.
 void loop() {
   // Get image from provider.
   if (kTfLiteOk != GetImage(kNumCols, kNumRows, kNumChannels, input->data.int8)) {