Merge branch 'feature/enable_CI_target_tests' into 'master'

feat: Add pytest target tests

Closes RDT-473, ESF-54, and ESF-53

See merge request espressif/esp-serial-flasher!116

Author: DNedic

.gitlab-ci.yml

@@ -22,6 +22,12 @@ variables:
   ZEPHYR_SDK: https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v0.16.3/zephyr-sdk-0.16.3_linux-aarch64_minimal.tar.xz
   PI_PICO_SDK_REV: 1.5.1
   PI_PICO_SDK: https://github.com/raspberrypi/pico-sdk
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+  PIP_EXTRA_INDEX_URL: "https://dl.espressif.com/pypi"
+
+cache:
+  paths:
+    - "$CI_PROJECT_DIR/.cache/pip"
 
 run_pre_commit:
   stage: pre_check
@@ -50,19 +56,32 @@ check_stub_source_correctness:
     - build
     - internet
   before_script:
-    - pip install -U idf-build-apps
+    - apt install ccache
+    - pip install -U idf-build-apps --prefer-binary
   variables:
+    IDF_CCACHE_ENABLE: 1
+    CCACHE_DIR: "/cache/esf_ccache"
+    CCACHE_SLOPPINESS: "time_macros"
     PEDANTIC_FLAGS: "-Werror -Wall -Wextra"
     EXTRA_CFLAGS: "${PEDANTIC_FLAGS}"
     EXTRA_CXXFLAGS: "${PEDANTIC_FLAGS}"
     BUILD_DIR: "build"
   script:
+    # CCACHE_RECACHE Used when invalidating the current cache.
+    # could be enabled by MR label "ccache:recache"
+    - |
+      if [ -n "${CI_MERGE_REQUEST_LABELS}" ] && echo "${CI_MERGE_REQUEST_LABELS}" | grep -q "ccache::recache"; then
+        export CCACHE_RECACHE="1"
+        echo "INFO: ccache recache enabled"
+      fi
     - python -m idf_build_apps build -v -p .
       --recursive
       --exclude ./examples/binaries
       --config "sdkconfig.defaults*"
       --build-dir ${BUILD_DIR}
       --check-warnings
+    # Show ccache statistics if enabled globally
+    - test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats -vv || true
 
 build_idf_v4.3:
   image: espressif/idf:release-v4.3
@@ -71,20 +90,33 @@ build_idf_v4.3:
     - build
     - internet
   before_script:
-    - pip install -U idf-build-apps
+    - apt install ccache
+    - pip install -U idf-build-apps --prefer-binary
   variables:
+    IDF_CCACHE_ENABLE: 1
+    CCACHE_DIR: "/cache/esf_ccache"
+    CCACHE_SLOPPINESS: "time_macros"
     PEDANTIC_FLAGS: "-Werror -Wall -Wextra"
     EXTRA_CFLAGS: "${PEDANTIC_FLAGS}"
     EXTRA_CXXFLAGS: "${PEDANTIC_FLAGS}"
     BUILD_DIR: "build"
   script:
+    # CCACHE_RECACHE Used when invalidating the current cache.
+    # could be enabled by MR label "ccache:recache"
+    - |
+      if [ -n "${CI_MERGE_REQUEST_LABELS}" ] && echo "${CI_MERGE_REQUEST_LABELS}" | grep -q "ccache::recache"; then
+        export CCACHE_RECACHE="1"
+        echo "INFO: ccache recache enabled"
+      fi
     - python -m idf_build_apps build -v -p .
       --recursive
       --exclude ./examples/binaries
       --target "esp32"
       --config "sdkconfig.defaults*"
       --build-dir ${BUILD_DIR}
       --check-warnings
+    # Show ccache statistics if enabled globally
+    - test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats -vv || true
 
 build_idf_v4.4:
   image: espressif/idf:release-v4.4
@@ -143,6 +175,11 @@ build_stm32:
   # IDF is not necessary for STM32 build, but this image is already used in another job
   # and it comes with a recent enough CMake version.
   image: espressif/idf:latest
+  artifacts:
+    paths:
+      - "**/build*/*.bin"
+    when: always
+    expire_in: 3 days
   tags:
     - build
     - internet
@@ -194,8 +231,15 @@ build_pi_pico:
   tags:
     - build
     - internet
+  artifacts:
+    paths:
+      - "**/build*/*.bin"
+      - "**/build*/*.elf"
+      - "**/build*/*.uf2"
+    when: always
+    expire_in: 3 days
   script:
-    - git clone --single-branch --depth=1 -b ${PI_PICO_SDK_REV} ${PI_PICO_SDK}
+    - git clone --recursive --single-branch --depth=1 -b ${PI_PICO_SDK_REV} ${PI_PICO_SDK}
     - export PICO_SDK_PATH=$(pwd)/pico-sdk
     - wget --no-verbose -O gcc-arm-none-eabi.tar.xz ${ARM_TOOLCHAIN_URL}
     - tar xf gcc-arm-none-eabi.tar.xz
@@ -207,7 +251,25 @@ build_pi_pico:
     - cmake ..
     - cmake --build .
 
-run_tests:
+build_pi:
+  stage: build
+  image: dtcooper/raspberrypi-os:latest
+  tags:
+    - ESF-RPI-01
+  before_script:
+    - apt-get update
+    - apt-get install -y cmake gcc g++ make pigpio
+  script:
+    - cd $CI_PROJECT_DIR/examples/raspberry_example
+    - mkdir build && cd build
+    - cmake .. && cmake --build .
+  artifacts:
+    paths:
+      - "${CI_PROJECT_DIR}/examples/raspberry_example/build/raspberry_flasher"
+    when: always
+    expire_in: 3 days
+
+test_qemu:
   stage: test
   image: ${CI_DOCKER_REGISTRY}/qemu:esp-develop-20191124
   tags:
@@ -218,6 +280,73 @@ run_tests:
     - export QEMU_PATH=/opt/qemu/bin/qemu-system-xtensa
     - ./run_qemu_test.sh
 
+.test_template:
+  stage: test
+  image: debian:latest
+  before_script:
+    - apt-get update
+    - apt-get install -y python3 python3-pip libffi-dev
+    - pip install -r $CI_PROJECT_DIR/test/requirements_test.txt --break-system-packages --prefer-binary
+    - pip install esptool --break-system-packages --prefer-binary
+  artifacts:
+    paths:
+      - "${CI_PROJECT_DIR}/examples/pytest_embedded_log/"
+    when: always
+    expire_in: 1 week
+
+test_esp1:
+  extends: .test_template
+  tags:
+    - ESF-RPI-02
+  parallel:
+    matrix:
+      - CI_BUILD_FOLDER:
+          - "build_stable"
+          - "build_master"
+  script:
+    - cd $CI_PROJECT_DIR
+    - pytest --target=esp32s3 --port=/dev/serial_ports/ESP32S3_ESP32C3 -k 'not test_esp32_spi_load_ram_example'
+
+test_esp2:
+  extends: .test_template
+  tags:
+    - ESF-RPI-01
+  parallel:
+    matrix:
+      - CI_BUILD_FOLDER:
+          - "build_stable"
+          - "build_master"
+  script:
+    - cd $CI_PROJECT_DIR
+    - pytest --target=esp32 --port=/dev/serial_ports/ESP32_ESP32
+    - pytest --target=esp32s3 --port=/dev/serial_ports/ESP32S3_ESP32C3 -k 'not test_esp32_usb_cdc_acm_example'
+
+test_stm32:
+  extends: .test_template
+  tags:
+    - ESF-RPI-03
+  script:
+    - pytest --target=stm32 --port=/dev/serial_ports/STM32_ESP32
+
+test_pi_pico:
+  extends: .test_template
+  tags:
+    - ESF-RPI-03
+  script:
+    - pytest --target=pi_pico --port=/dev/serial_ports/pico_ESP32
+
+test_rpi:
+  stage: test
+  image: dtcooper/raspberrypi-os:latest
+  tags:
+    - ESF-RPI-01
+  before_script:
+    - apt-get update
+    - apt-get install -y pigpio python3 python3-pip libffi-dev
+    - pip install -r $CI_PROJECT_DIR/test/requirements_test.txt --break-system-packages --prefer-binary
+  script:
+    - pytest --target=raspberry
+
 push_to_the_components_registry:
   stage: deploy
   image: python:3.11-bookworm

.pre-commit-config.yaml

@@ -24,3 +24,11 @@ repos:
     hooks:
       - id: conventional-precommit-linter
         stages: [commit-msg]
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.14
+    hooks:
+      - id: ruff # Runs ruff linter
+        args: [--fix, --exit-non-zero-on-fix] # --fix for fixing errors
+        types: [python]
+      - id: ruff-format
+        types: [python]

cmake/gen_stub_sources.py

@@ -40,6 +40,7 @@
     "esp32c6.json",  # ESP32C6_CHIP
 ]
 
+
 def read_stub_json(json_file):
     stub = json.load(json_file)
     entry = stub["entry"]
@@ -83,7 +84,10 @@ def read_stub_json(json_file):
 
 if __name__ == "__main__":
     # .h and .c file templates
-    with open(h_template_path, "r") as h_template_file, open(c_template_path, "r") as c_template_file:
+    with (
+        open(h_template_path, "r") as h_template_file,
+        open(c_template_path, "r") as c_template_file,
+    ):
         h_template = h_template_file.read()
         c_template = c_template_file.read()
 
@@ -112,7 +116,9 @@ def read_stub_json(json_file):
                     with open(f"{stub_override_path}/{file_to_download}") as file_path:
                         cfile.write(read_stub_json(file_path))
                 else:
-                    with urllib.request.urlopen(f"{stub_download_url}/v{stub_version}/{file_to_download}") as url:
+                    with urllib.request.urlopen(
+                        f"{stub_download_url}/v{stub_version}/{file_to_download}"
+                    ) as url:
                         cfile.write(read_stub_json(url))
 
         cfile.write("};\n" "\n" "#endif\n")

examples/conftest.py

@@ -0,0 +1,81 @@
+import logging
+import os
+import sys
+from datetime import datetime
+from typing import List, Optional
+
+import pytest
+from pytest_embedded.plugin import multi_dut_fixture
+
+
+def pytest_configure(config):
+    target = config.getoption("--target")
+
+    if "stm32" in target:
+        config.option.embedded_services = "serial"
+    elif "raspberry" in target:
+        pass
+    elif "pi_pico" in target:
+        config.option.embedded_services = "serial"
+    else:
+        config.option.embedded_services = "esp,idf"
+
+
+def pytest_collection_modifyitems(
+    session: pytest.Session, config: pytest.Config, items: List[pytest.Item]
+):
+    """Modify test collection based on selected target"""
+    target = config.getoption("--target")
+
+    # Filter the test cases
+    filtered_items = []
+    for item in items:
+        # filter by target
+        all_markers = [marker.name for marker in item.iter_markers()]
+        if target not in all_markers:
+            continue
+
+        filtered_items.append(item)
+    items[:] = filtered_items
+
+
+@pytest.fixture(scope="session", autouse=True)
+def session_tempdir() -> str:
+    _tmpdir = os.path.join(
+        os.path.dirname(__file__),
+        "pytest_embedded_log",
+        datetime.now().strftime("%Y-%m-%d_%H-%M-%S"),
+    )
+    os.makedirs(_tmpdir, exist_ok=True)
+    return _tmpdir
+
+
+@pytest.fixture
+@multi_dut_fixture
+def build_dir(
+    request: pytest.FixtureRequest, app_path: str, target: Optional[str]
+) -> str:
+    """
+    Check local build dir and return the valid one
+
+    Returns:
+        valid build directory
+    """
+    check_dirs = []
+    build_folder = os.getenv("CI_BUILD_FOLDER")
+    if build_folder is not None:
+        check_dirs.append(build_folder)
+    check_dirs.append("build")
+
+    for check_dir in check_dirs:
+        binary_path = os.path.join(app_path, check_dir)
+        if os.path.isdir(binary_path):
+            logging.info(f"find valid binary path: {binary_path}")
+            return check_dir
+
+        logging.warning(
+            "checking binary path: %s... missing... try another place", binary_path
+        )
+
+    logging.error("no build dir valid. Please build the binary and run pytest again.")
+    sys.exit(1)

examples/esp32_example/pytest_esp32_example.py

@@ -0,0 +1,9 @@
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.esp32
+def test_esp32_example(dut: Dut) -> None:
+    for i in range(3):
+        dut.expect("Finished programming")
+    dut.expect("Hello world!")

examples/esp32_load_ram_example/pytest_esp32_load_ram_example.py

@@ -0,0 +1,8 @@
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.esp32
+def test_esp32_load_ram_example(dut: Dut) -> None:
+    dut.expect("Finished loading")
+    dut.expect("Hello world!")

examples/esp32_spi_load_ram_example/pytest_esp32_spi_load_ram_example.py

@@ -0,0 +1,8 @@
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.esp32s3
+def test_esp32_spi_load_ram_example(dut: Dut) -> None:
+    dut.expect("Finished loading")
+    dut.expect("Hello world!")

examples/esp32_stub_example/pytest_esp32_stub_example.py

@@ -0,0 +1,10 @@
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.esp32
+def test_esp32_stub_example(dut: Dut) -> None:
+    dut.expect("Finished programming")
+    for i in range(3):
+        dut.expect("Flash verified")
+    dut.expect("Hello world!")

examples/esp32_usb_cdc_acm_example/main/main.c

@@ -96,6 +96,9 @@ void app_main(void)
         device_disconnected_sem = xSemaphoreCreateBinary();
         assert(device_disconnected_sem);
 
+        // Delay as target may not be ready to accept commands immediately after connection
+        loader_port_delay_ms(100);
+
         /* The ESP32-S3 ignores the line coding set commands,
            so we don't set the higher baudrate argument */
         if (connect_to_target(0) == ESP_LOADER_SUCCESS) {

examples/esp32_usb_cdc_acm_example/pytest_esp32_usb_cdc_acm_example.py

@@ -0,0 +1,9 @@
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.esp32s3
+def test_esp32_usb_cdc_acm_example(dut: Dut) -> None:
+    dut.expect("Finished programming")
+    for i in range(3):
+        dut.expect("Flash verified")