change: Stub support overhaul

This overhauls the stub support added by @higaski, and covers:

1. Switching to the esp-flasher-stub project for stub sources, while
   adding the ability to override the source and version of the stubs.
   This ensures licensing compatibility with esp-serial-flasher and any
   project that depends on it, as esptool stubs were licensed under GPL.
   It also ensures easier future support for new features.
2. Adding an example and documentation
3. Overhauling the stub source file generation process, which is now done
   at the CMake configure stage instead of being a separate build
   target, and uses .template files instead of in-source strings.
4. Adding support for the Zephyr port use
5. Edge-case, code smell and timing fixes

Author: DNedic

.github/workflows/stub_sources_correctness.yml

@@ -0,0 +1,23 @@
+name: Stub sources correctness checks
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+  pre_commit:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Configure the stub example with the stub version argument
+        run: idf.py reconfigure -DSERIAL_FLASHER_STUB_PULL_VERSION='0.3.0'
+        working-directory: ${{github.workspace}}/examples/esp32_stub_example
+
+      - name: Run git diff to check if the sources in the repo match the generated ones
+        run: git diff --exit-code

.gitignore

@@ -1,4 +1,3 @@
-.vscode
 build
 sdkconfig
 sdkconfig.old

.gitlab-ci.yml

@@ -36,6 +36,8 @@ variables:
     - cd $CI_PROJECT_DIR/examples/esp32_example
     - idf.py build -DMD5_ENABLED=1
     - idf.py build -DMD5_ENABLED=0
+    - cd $CI_PROJECT_DIR/examples/esp32_stub_example
+    - idf.py build
     - cd $CI_PROJECT_DIR/examples/esp32_load_ram_example
     - idf.py build
     - cd $CI_PROJECT_DIR/examples/esp32_spi_load_ram_example
@@ -55,6 +57,16 @@ run_pre_commit:
     - pip install pre-commit
     - pre-commit run --show-diff-on-failure --from-ref origin/master --to-ref HEAD
 
+check_stub_source_correctness:
+  stage: pre_check
+  image: espressif/idf:latest
+  tags:
+    - internet
+  script:
+    - cd $CI_PROJECT_DIR/examples/esp32_stub_example
+    - idf.py reconfigure -DSERIAL_FLASHER_STUB_PULL_VERSION='0.3.0'
+    - git diff --exit-code
+
 # Special case as ESP-IDF v4.3 is not supported by the usb_host_cdc_acm component
 # required by the USB CDC ACM interface port and its example
 build_idf_v4.3:
@@ -71,6 +83,8 @@ build_idf_v4.3:
     - cd $CI_PROJECT_DIR/examples/esp32_example
     - idf.py build -DMD5_ENABLED=1
     - idf.py build -DMD5_ENABLED=0
+    - cd $CI_PROJECT_DIR/examples/esp32_stub_example
+    - idf.py build
     - cd $CI_PROJECT_DIR/examples/esp32_load_ram_example
     - idf.py build
     - cd $CI_PROJECT_DIR/examples/esp32_spi_load_ram_example

CMakeLists.txt

@@ -144,6 +144,15 @@ endif()
 
 target_compile_definitions(${target} PUBLIC ${defs})
 
-# esp_stubs.[ch] codegen
-include(cmake/add_gen_esp_stubs_target.cmake)
-add_gen_esp_stubs_target(VERSION 4.7.0)
+# This segment pulls the flasher stubs at a specified version at wish, and generates the
+# esp_stubs.c/h files, overwriting the library provided ones.
+# It is also possible to override stub generation to use a custom url or a local folder.
+# Please check if the license under which the custom stub sources are released fits your usecase.
+if (DEFINED SERIAL_FLASHER_STUB_PULL_VERSION OR DEFINED SERIAL_FLASHER_STUB_PULL_OVERRIDE_PATH)
+    include(cmake/serial_flasher_pull_stubs.cmake)
+    serial_flasher_pull_stubs(
+        VERSION ${SERIAL_FLASHER_STUB_PULL_VERSION}
+        SOURCE ${SERIAL_FLASHER_STUB_PULL_SOURCE}
+        PATH_OVERRIDE ${SERIAL_FLASHER_STUB_PULL_OVERRIDE_PATH}
+    )
+endif()

cmake/add_gen_esp_stubs_target.cmake

@@ -0,0 +1,47 @@
+/* Copyright 2020-{current_year} Espressif Systems (Shanghai) CO LTD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// auto-generated stubs from esp-flasher-stub v{stub_version}
+
+#include "esp_stubs.h"
+
+static bool stub_running = false;
+
+bool esp_stub_get_running(void)
+{{
+    return stub_running;
+}}
+
+void esp_stub_set_running(bool stub_status)
+{{
+    stub_running = stub_status;
+}}
+
+#if (defined SERIAL_FLASHER_INTERFACE_UART) || (defined SERIAL_FLASHER_INTERFACE_USB)
+
+#if __STDC_VERSION__ >= 201112L
+_Static_assert(ESP8266_CHIP == 0, "Stub order matches target_chip_t enumeration");
+_Static_assert(ESP32_CHIP == 1, "Stub order matches target_chip_t enumeration");
+_Static_assert(ESP32S2_CHIP == 2, "Stub order matches target_chip_t enumeration");
+_Static_assert(ESP32C3_CHIP == 3, "Stub order matches target_chip_t enumeration");
+_Static_assert(ESP32S3_CHIP == 4, "Stub order matches target_chip_t enumeration");
+_Static_assert(ESP32C2_CHIP == 5, "Stub order matches target_chip_t enumeration");
+_Static_assert(ESP32_RESERVED0_CHIP == 6, "Stub order matches target_chip_t enumeration");
+_Static_assert(ESP32H2_CHIP == 7, "Stub order matches target_chip_t enumeration");
+_Static_assert(ESP32C6_CHIP == 8, "Stub order matches target_chip_t enumeration");
+_Static_assert(ESP_MAX_CHIP == {max_chip_number}, "Stub order matches target_chip_t enumeration");
+#endif
+
+const esp_stub_t esp_stub[ESP_MAX_CHIP] = {{

cmake/esp_stubs.c.template

@@ -0,0 +1,44 @@
+/* Copyright 2020-{current_year} Espressif Systems (Shanghai) CO LTD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// auto-generated stubs from esp-flasher-stub v{stub_version}
+
+#pragma once
+
+#include <stdint.h>
+#include <stdbool.h>
+#include "esp_loader.h"
+
+#ifdef __cplusplus
+extern "C" {{
+#endif
+
+bool esp_stub_get_running(void);
+void esp_stub_set_running(bool stub_status);
+
+#if (defined SERIAL_FLASHER_INTERFACE_UART) || (defined SERIAL_FLASHER_INTERFACE_USB)
+
+typedef struct {{
+    esp_loader_bin_header_t header;
+    esp_loader_bin_segment_t segments[2];
+}} esp_stub_t;
+
+extern const esp_stub_t esp_stub[ESP_MAX_CHIP];
+
+#endif
+
+#ifdef __cplusplus
+}}
+#endif