diff --git a/examples/person_detection/CMakeLists.txt b/examples/person_detection/CMakeLists.txt index 67b05f1..ed1b8f9 100644 --- a/examples/person_detection/CMakeLists.txt +++ b/examples/person_detection/CMakeLists.txt @@ -3,27 +3,28 @@ cmake_minimum_required(VERSION 3.5) set(EXTRA_COMPONENT_DIRS static_images) include($ENV{IDF_PATH}/tools/cmake/project.cmake) -function(add_bsp SDKCONFIG BSP TARGET) +# Helper function to set required environment variables as per configuration set by user +function(add_bsp SDKCONFIG BSP) string(REGEX MATCH "CONFIG_${BSP}=y" REGEX_RESULT ${SDKCONFIG}) if (REGEX_RESULT) - set(ENV{${BSP}} "${TARGET}") + set(ENV{${BSP}} 1) # If the config option is set then set corresponding environment variable to 1 endif() endfunction() -# 1. Define all variables used in main/idf_component.yml -set(ENV{TFLITE_USE_BSP_S3_EYE} "false") -set(ENV{TFLITE_USE_BSP_KORVO_2} "false") -set(ENV{TFLITE_USE_BSP_KALUGA} "false") +# 1. Define all variables used in main/idf_component.yml and set them to zero +set(ENV{TFLITE_USE_BSP_S3_EYE} 0) +set(ENV{TFLITE_USE_BSP_KORVO_2} 0) +set(ENV{TFLITE_USE_BSP_KALUGA} 0) -# 2. Set correct var to 'target' +# 2. Set correct var to 1 # This is a workaround idf-component-manager limitation, where only -# target and idf_version can be in the if-clause +# target, idf_version and environment variables can be in the if-clause if(EXISTS ${CMAKE_CURRENT_LIST_DIR}/sdkconfig) file(READ ${CMAKE_CURRENT_LIST_DIR}/sdkconfig SDKCONFIG_RULE) - add_bsp("${SDKCONFIG_RULE}" "TFLITE_USE_BSP_S3_EYE" "esp32s3") - add_bsp("${SDKCONFIG_RULE}" "TFLITE_USE_BSP_KORVO_2" "esp32s3") - add_bsp("${SDKCONFIG_RULE}" "TFLITE_USE_BSP_KALUGA" "esp32s2") + add_bsp("${SDKCONFIG_RULE}" "TFLITE_USE_BSP_S3_EYE") # Check for CONFIG_TFLITE_USE_BSP_S3_EYE option in sdkconfig + add_bsp("${SDKCONFIG_RULE}" "TFLITE_USE_BSP_KORVO_2") # Check for CONFIG_TFLITE_USE_BSP_KORVO_2 option in sdkconfig + add_bsp("${SDKCONFIG_RULE}" "TFLITE_USE_BSP_KALUGA") # Check for CONFIG_TFLITE_USE_BSP_KALUGA option in sdkconfig endif() project(person_detection) diff --git a/examples/person_detection/main/app_camera_esp.c b/examples/person_detection/main/app_camera_esp.c index bc8440e..0be6c9f 100644 --- a/examples/person_detection/main/app_camera_esp.c +++ b/examples/person_detection/main/app_camera_esp.c @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include "app_camera_esp.h" +#include "esp_log.h" #include "sdkconfig.h" #if (CONFIG_TFLITE_USE_BSP) diff --git a/examples/person_detection/main/detection_responder.cc b/examples/person_detection/main/detection_responder.cc index 097e1c8..acb17b1 100644 --- a/examples/person_detection/main/detection_responder.cc +++ b/examples/person_detection/main/detection_responder.cc @@ -36,10 +36,26 @@ static lv_obj_t *camera_canvas = NULL; static lv_obj_t *person_indicator = NULL; static lv_obj_t *label = NULL; -static void create_gui(void) +void create_gui(void) { - bsp_display_start(); - bsp_display_backlight_on(); // Set display brightness to 100% + bsp_display_cfg_t cfg = { + .lvgl_port_cfg = { + .task_priority = CONFIG_BSP_DISPLAY_LVGL_TASK_PRIORITY, + .task_stack = 6144, + .task_affinity = 1, + .task_max_sleep_ms = CONFIG_BSP_DISPLAY_LVGL_MAX_SLEEP, + .timer_period_ms = CONFIG_BSP_DISPLAY_LVGL_TICK, + }, + .buffer_size = BSP_LCD_DRAW_BUFF_SIZE, + .double_buffer = BSP_LCD_DRAW_BUFF_DOUBLE, + .flags = { + .buff_dma = false, + .buff_spiram = true, + } + }; + bsp_display_start_with_config(&cfg); + bsp_display_backlight_on(); + bsp_display_lock(0); camera_canvas = lv_canvas_create(lv_scr_act()); assert(camera_canvas); @@ -62,20 +78,21 @@ void RespondToDetection(float person_score, float no_person_score) { int person_score_int = (person_score) * 100 + 0.5; (void) no_person_score; // unused #if DISPLAY_SUPPORT - if (!camera_canvas) { - create_gui(); - } + if (!camera_canvas) { + create_gui(); + } - uint16_t *buf = (uint16_t *) image_provider_get_display_buf(); + uint16_t *buf = (uint16_t *) image_provider_get_display_buf(); - bsp_display_lock(0); - if (person_score_int < 60) { // treat score less than 60% as no person - lv_led_off(person_indicator); - } else { - lv_led_on(person_indicator); - } - lv_canvas_set_buffer(camera_canvas, buf, IMG_WD, IMG_HT, LV_IMG_CF_TRUE_COLOR); - bsp_display_unlock(); + bsp_display_lock(0); + if (person_score_int < 60) { // treat score less than 60% as no person + lv_led_off(person_indicator); + } else { + lv_led_on(person_indicator); + } + + lv_canvas_set_buffer(camera_canvas, buf, IMG_WD, IMG_HT, LV_COLOR_FORMAT_RGB565); + bsp_display_unlock(); #endif // DISPLAY_SUPPORT MicroPrintf("person score:%d%%, no person score %d%%", person_score_int, 100 - person_score_int); diff --git a/examples/person_detection/main/detection_responder.h b/examples/person_detection/main/detection_responder.h index b3f9104..b2e3f08 100644 --- a/examples/person_detection/main/detection_responder.h +++ b/examples/person_detection/main/detection_responder.h @@ -29,4 +29,7 @@ limitations under the License. // particular applications. void RespondToDetection(float person_score, float no_person_score); +// Initialize GUI components +void create_gui(); + #endif // TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_DETECTION_RESPONDER_H_ diff --git a/examples/person_detection/main/esp_cli.c b/examples/person_detection/main/esp_cli.c index 5296fa4..82f171b 100644 --- a/examples/person_detection/main/esp_cli.c +++ b/examples/person_detection/main/esp_cli.c @@ -24,6 +24,7 @@ #include "esp_cli.h" #include "esp_timer.h" +#if CLI_ONLY_INFERENCE #define IMAGE_COUNT 10 static uint8_t *image_database[IMAGE_COUNT]; @@ -38,6 +39,7 @@ extern const uint8_t image6_start[] asm("_binary_image6_start"); extern const uint8_t image7_start[] asm("_binary_image7_start"); extern const uint8_t image8_start[] asm("_binary_image8_start"); extern const uint8_t image9_start[] asm("_binary_image9_start"); +#endif static const char *TAG = "[esp_cli]"; @@ -96,6 +98,7 @@ static int mem_dump_cli_handler(int argc, char *argv[]) return 0; } +#if CLI_ONLY_INFERENCE static int inference_cli_handler(int argc, char *argv[]) { /* Just to go to the next line */ @@ -121,6 +124,18 @@ static int inference_cli_handler(int argc, char *argv[]) return 0; } +int esp_cli_register_inference_command() { + esp_console_cmd_t command = { + .command = "detect_image", + .help = "detect_image " + "Note: image numbers ranging from 0 - 9 only are valid", + .func = inference_cli_handler, + }; + esp_console_cmd_register(&command); + return 0; +} +#endif + static esp_console_cmd_t diag_cmds[] = { { .command = "mem-dump", @@ -137,12 +152,6 @@ static esp_console_cmd_t diag_cmds[] = { .help = "", .func = cpu_dump_cli_handler, }, - { - .command = "detect_image", - .help = "detect_image " - "Note: image numbers ranging from 0 - 9 only are valid", - .func = inference_cli_handler, - }, }; int esp_cli_register_cmds() @@ -158,6 +167,7 @@ int esp_cli_register_cmds() static void image_database_init() { +#if CLI_ONLY_INFERENCE image_database[0] = (uint8_t *) image0_start; image_database[1] = (uint8_t *) image1_start; image_database[2] = (uint8_t *) image2_start; @@ -168,7 +178,7 @@ static void image_database_init() image_database[7] = (uint8_t *) image7_start; image_database[8] = (uint8_t *) image8_start; image_database[9] = (uint8_t *) image9_start; - +#endif } int esp_cli_start() diff --git a/examples/person_detection/main/esp_cli.h b/examples/person_detection/main/esp_cli.h index d9e8f85..aaf214b 100644 --- a/examples/person_detection/main/esp_cli.h +++ b/examples/person_detection/main/esp_cli.h @@ -19,6 +19,7 @@ extern "C" { #endif int esp_cli_start(); +int esp_cli_register_inference_command(); #ifdef __cplusplus } diff --git a/examples/person_detection/main/idf_component.yml b/examples/person_detection/main/idf_component.yml index 55503f1..3b36281 100644 --- a/examples/person_detection/main/idf_component.yml +++ b/examples/person_detection/main/idf_component.yml @@ -2,20 +2,16 @@ dependencies: espressif/esp-tflite-micro: version: "*" override_path: "../../../" - - espressif/esp32-camera: "~2.0.5" - + espressif/esp32-camera: ~2.0.5 + espressif/esp32_s2_kaluga_kit: + rules: + - if: "$TFLITE_USE_BSP_KALUGA == 1" + version: 3.* espressif/esp32_s3_eye: - version: "3.*" rules: - - if: "target == $TFLITE_USE_BSP_S3_EYE" - + - if: "$TFLITE_USE_BSP_S3_EYE == 1" + version: 4.* espressif/esp32_s3_korvo_2: - version: "2.*" - rules: - - if: "target == $TFLITE_USE_BSP_KORVO_2" - - espressif/esp32_s2_kaluga_kit: - version: "3.*" rules: - - if: "target == $TFLITE_USE_BSP_KALUGA" + - if: "$TFLITE_USE_BSP_KORVO_2 == 1" + version: 2.* diff --git a/examples/person_detection/main/image_provider.cc b/examples/person_detection/main/image_provider.cc index 0231d33..4dca6bf 100644 --- a/examples/person_detection/main/image_provider.cc +++ b/examples/person_detection/main/image_provider.cc @@ -1,4 +1,3 @@ - /* Copyright 2019 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,16 +12,17 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include -#include -#include +#include "string.h" #include "freertos/FreeRTOS.h" #include "freertos/task.h" + +#if (CONFIG_TFLITE_USE_BSP) +#include "bsp/esp-bsp.h" +#endif + +#include "esp_heap_caps.h" #include "esp_log.h" -#include "esp_spi_flash.h" -#include "esp_system.h" -#include "esp_timer.h" #include "app_camera_esp.h" #include "esp_camera.h" @@ -31,8 +31,7 @@ limitations under the License. #include "esp_main.h" static const char* TAG = "app_camera"; - -static uint16_t *display_buf; // buffer to hold data to be sent to display +static uint16_t* display_buf; // Get the camera module ready TfLiteStatus InitCamera() { @@ -84,13 +83,19 @@ TfLiteStatus GetImage(int image_width, int image_height, int channels, int8_t* i // In case if display support is enabled, we initialise camera in rgb mode // Hence, we need to convert this data to grayscale to send it to tf model // For display we extra-polate the data to 192X192 + + // point to the last quarter of buffer + uint16_t* cam_buf = display_buf + (96 * 96 * 3); + memcpy((uint8_t*)cam_buf, fb->buf, fb->len); + esp_camera_fb_return(fb); + for (int i = 0; i < kNumRows; i++) { for (int j = 0; j < kNumCols; j++) { - uint16_t pixel = ((uint16_t *) (fb->buf))[i * kNumCols + j]; + uint16_t inference_pixel = cam_buf[i * kNumCols + j]; // for inference - uint8_t hb = pixel & 0xFF; - uint8_t lb = pixel >> 8; + uint8_t hb = inference_pixel & 0xFF; + uint8_t lb = inference_pixel >> 8; uint8_t r = (lb & 0x1F) << 3; uint8_t g = ((hb & 0x07) << 5) | ((lb & 0xE0) >> 3); uint8_t b = (hb & 0xF8); @@ -102,8 +107,14 @@ TfLiteStatus GetImage(int image_width, int image_height, int channels, int8_t* i int8_t grey_pixel = ((305 * r + 600 * g + 119 * b) >> 10) - 128; image_data[i * kNumCols + j] = grey_pixel; + } + } - // to display + // for display + lv_draw_sw_rgb565_swap(cam_buf, 96 * 96); + for (int i = 0; i < kNumRows; i++) { + for (int j = 0; j < kNumCols; j++) { + uint16_t pixel = cam_buf[i * kNumCols + j]; display_buf[2 * i * kNumCols * 2 + 2 * j] = pixel; display_buf[2 * i * kNumCols * 2 + 2 * j + 1] = pixel; display_buf[(2 * i + 1) * kNumCols * 2 + 2 * j] = pixel; @@ -117,9 +128,9 @@ TfLiteStatus GetImage(int image_width, int image_height, int channels, int8_t* i for (int i = 0; i < image_width * image_height; i++) { image_data[i] = ((uint8_t *) fb->buf)[i] ^ 0x80; } -#endif // DISPLAY_SUPPORT esp_camera_fb_return(fb); +#endif // DISPLAY_SUPPORT /* here the esp camera can give you grayscale image directly */ return kTfLiteOk; #else diff --git a/examples/person_detection/main/main.cc b/examples/person_detection/main/main.cc index 9dbb301..9a9c327 100644 --- a/examples/person_detection/main/main.cc +++ b/examples/person_detection/main/main.cc @@ -20,15 +20,13 @@ limitations under the License. #include "freertos/task.h" #include "esp_main.h" - -#if CLI_ONLY_INFERENCE #include "esp_cli.h" -#endif void tf_main(void) { setup(); -#if CLI_ONLY_INFERENCE esp_cli_start(); +#if CLI_ONLY_INFERENCE + esp_cli_register_inference_command(); vTaskDelay(portMAX_DELAY); #else while (true) { diff --git a/examples/person_detection/main/main_functions.cc b/examples/person_detection/main/main_functions.cc index 32a257e..f70fb31 100644 --- a/examples/person_detection/main/main_functions.cc +++ b/examples/person_detection/main/main_functions.cc @@ -32,7 +32,6 @@ limitations under the License. #include #include "esp_main.h" -// Globals, used for compatibility with Arduino-style sketches. namespace { const tflite::Model* model = nullptr; tflite::MicroInterpreter* interpreter = nullptr; @@ -56,7 +55,6 @@ constexpr int kTensorArenaSize = 100 * 1024 + scratchBufSize; static uint8_t *tensor_arena;//[kTensorArenaSize]; // Maybe we should move this to external } // namespace -// The name of this function is important for Arduino compatibility. void setup() { // Map the model into a usable data structure. This doesn't involve any // copying or parsing, it's a very lightweight operation. @@ -68,7 +66,7 @@ void setup() { } if (tensor_arena == NULL) { - tensor_arena = (uint8_t *) heap_caps_malloc(kTensorArenaSize, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); + tensor_arena = (uint8_t *) heap_caps_malloc(kTensorArenaSize, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); } if (tensor_arena == NULL) { printf("Couldn't allocate memory of %d bytes\n", kTensorArenaSize); @@ -113,11 +111,14 @@ void setup() { MicroPrintf("InitCamera failed\n"); return; } -#endif + +#if DISPLAY_SUPPORT + create_gui(); +#endif // DISPLAY_SUPPORT +#endif // CLI_ONLY_INFERENCE } #ifndef CLI_ONLY_INFERENCE -// The name of this function is important for Arduino compatibility. void loop() { // Get image from provider. if (kTfLiteOk != GetImage(kNumCols, kNumRows, kNumChannels, input->data.int8)) { @@ -144,7 +145,7 @@ void loop() { RespondToDetection(person_score_f, no_person_score_f); vTaskDelay(1); // to avoid watchdog trigger } -#endif +#endif // CLI_ONLY_INFERENCE #if defined(COLLECT_CPU_STATS) long long total_time = 0;