Merge branch 'espressif:master' into patch-ble-notify-range

Author: philipp-simon

.github/ISSUE_TEMPLATE/02_runtime_bug.yml

@@ -26,7 +26,7 @@ body:
     id: chip_revision
     attributes:
       label: Espressif SoC revision.
-      description: On which Espressif SoC revision does your application run on? Run `esptool chip_id` to find it.
+      description: On which Espressif SoC revision does your application run on? Run `esptool chip-id` to find it.
       placeholder: ex. ESP32-C3 (QFN32) (revision v0.3)
     validations:
       required: true

.gitignore

@@ -66,8 +66,6 @@ build_summary_*.xml
 coverage.info
 coverage_report/
 
-test_multi_heap_host
-
 # VS Code Settings
 .vscode/
 

.gitlab-ci.yml

@@ -3,12 +3,12 @@ workflow:
     # Disable those non-protected push triggered pipelines
     - if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && $CI_PIPELINE_SOURCE == "push"'
       when: never
-    # when running merged result pipelines, CI_COMMIT_SHA represents the temp commit it created.
-    # Please use PIPELINE_COMMIT_SHA at all places that require a commit sha of the original commit.
-    - if: $CI_OPEN_MERGE_REQUESTS != null
+    # merged result pipelines
+    - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
       variables:
         PIPELINE_COMMIT_SHA: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
-    - if: $CI_OPEN_MERGE_REQUESTS == null
+    # else
+    - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA == null || $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA == ""
       variables:
         PIPELINE_COMMIT_SHA: $CI_COMMIT_SHA
     - when: always

.gitlab/CODEOWNERS

@@ -54,7 +54,6 @@
 /.idf_build_apps.toml                 @esp-idf-codeowners/ci
 /.idf_ci.toml                         @esp-idf-codeowners/ci
 /.pre-commit-config.yaml              @esp-idf-codeowners/ci
-/.readthedocs.yml                     @esp-idf-codeowners/docs
 /.vale.ini                            @esp-idf-codeowners/docs
 /CMakeLists.txt                       @esp-idf-codeowners/build-config
 /COMPATIBILITY*.md                    @esp-idf-codeowners/peripherals
@@ -137,7 +136,6 @@
 /components/http_parser/              @esp-idf-codeowners/app-utilities
 /components/idf_test/                 @esp-idf-codeowners/peripherals @esp-idf-codeowners/system
 /components/ieee802154/               @esp-idf-codeowners/ieee802154
-/components/json/                     @esp-idf-codeowners/app-utilities
 /components/linux/                    @esp-idf-codeowners/system
 /components/log/                      @esp-idf-codeowners/system
 /components/lwip/                     @esp-idf-codeowners/lwip
@@ -160,10 +158,8 @@
 /components/tcp_transport/            @esp-idf-codeowners/network
 /components/ulp/                      @esp-idf-codeowners/system
 /components/unity/                    @esp-idf-codeowners/peripherals @esp-idf-codeowners/system
-/components/usb/                      @esp-idf-codeowners/peripherals/usb
 /components/vfs/                      @esp-idf-codeowners/storage
 /components/wear_levelling/           @esp-idf-codeowners/storage
-/components/wifi_provisioning/        @esp-idf-codeowners/app-utilities/provisioning
 /components/wpa_supplicant/           @esp-idf-codeowners/wifi @esp-idf-codeowners/app-utilities/mbedtls
 /components/xtensa/                   @esp-idf-codeowners/system
 
@@ -205,7 +201,6 @@
 /examples/peripherals/usb/    @esp-idf-codeowners/peripherals @esp-idf-codeowners/peripherals/usb
 /examples/phy/                @esp-idf-codeowners/bluetooth @esp-idf-codeowners/wifi @esp-idf-codeowners/ieee802154
 /examples/protocols/          @esp-idf-codeowners/network @esp-idf-codeowners/app-utilities
-/examples/provisioning/       @esp-idf-codeowners/app-utilities/provisioning
 /examples/security/           @esp-idf-codeowners/security
 /examples/storage/            @esp-idf-codeowners/storage
 /examples/system/             @esp-idf-codeowners/system
@@ -216,12 +211,10 @@
 /tools/                       @esp-idf-codeowners/tools
 /tools/ble/                   @esp-idf-codeowners/app-utilities
 /tools/bt/                    @esp-idf-codeowners/bluetooth
-/tools/catch/                 @esp-idf-codeowners/ci
 /tools/ci/                    @esp-idf-codeowners/ci
 /tools/cmake/                 @esp-idf-codeowners/build-config
 /tools/cmake/toolchain-*.cmake      @esp-idf-codeowners/toolchain
 /tools/esp_app_trace/         @esp-idf-codeowners/debugging
-/tools/esp_prov/              @esp-idf-codeowners/app-utilities
 /tools/gdb_panic_server.py    @esp-idf-codeowners/debugging
 /tools/kconfig*/              @esp-idf-codeowners/build-config
 /tools/ldgen/                 @esp-idf-codeowners/build-config
@@ -236,7 +229,6 @@
 /tools/test_apps/components/test_utils/ @esp-idf-codeowners/peripherals @esp-idf-codeowners/system
 /tools/test_apps/configs/               @esp-idf-codeowners/system
 /tools/test_apps/linux_compatible/      @esp-idf-codeowners/system
-/tools/test_apps/peripherals/           @esp-idf-codeowners/peripherals
 /tools/test_apps/phy/                   @esp-idf-codeowners/bluetooth @esp-idf-codeowners/wifi @esp-idf-codeowners/ieee802154
 /tools/test_apps/protocols/             @esp-idf-codeowners/network @esp-idf-codeowners/app-utilities
 /tools/test_apps/security/              @esp-idf-codeowners/security

.gitlab/ci/build.yml

@@ -214,27 +214,68 @@ build_clang_test_apps_esp32p4:
       --junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
 
+.test_build_system_macos_template_extension:
+  tags:
+    - macos
+  variables:
+    PYENV_VERSION: "3.10"
+    # CCACHE_DIR: "/cache/idf_ccache". On macOS, you cannot write to this folder due to insufficient permissions.
+    CCACHE_DIR: "" # ccache will use "$HOME/Library/Caches/ccache".
+    # Workaround for a bug in Parallels executor where CI_PROJECT_DIR is not an absolute path,
+    # but a relative path to the build directory (builds/espressif/esp-idf instead of ~/builds/espressif/esp-idf.
+    # GitLab sets the project dir to this template `<builds_dir>/<namespace>/<project_name>`
+    IDF_PATH: "/Users/espressif/builds/espressif/esp-idf"
+
+.test_build_system_minimal_cmake_template:
+  extends: .test_build_system_template
+  variables:
+    INSTALL_EXTRA_TOOLS: [email protected]
+  script:
+    - MINIMAL_SUPPORTED_CMAKE_VERSION=$(echo "${INSTALL_EXTRA_TOOLS}" | sed -n 's/.*cmake@\([0-9.]*\).*/\1/p')
+    - export PATH=$(echo "$PATH" | sed -E "s|/tools/cmake/[0-9.]+|/tools/cmake/${MINIMAL_SUPPORTED_CMAKE_VERSION}|")
+    - ACTUAL_CMAKE_VERSION=$(cmake --version | head -n1 | awk '{print $3}')
+    - |
+      if [ "${ACTUAL_CMAKE_VERSION}" != "${MINIMAL_SUPPORTED_CMAKE_VERSION}" ]; then
+        echo "ERROR: Wrong minimal CMake version! Detected: ${ACTUAL_CMAKE_VERSION}, but should be: ${MINIMAL_SUPPORTED_CMAKE_VERSION}"
+        exit 1
+      fi
+    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
+    - cd ${IDF_PATH}/tools/test_build_system
+    - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
+    - pytest
+      -k cmake
+      --cleanup-idf-copy
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+      --work-dir ${CI_PROJECT_DIR}/test_build_system
+      --junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml
+      --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
+
 pytest_build_system:
   extends: .test_build_system_template
   parallel: 3
 
 pytest_build_system_macos:
   extends:
     - .test_build_system_template
+    - .test_build_system_macos_template_extension
     - .before_script:build:macos
     - .after_script:build:macos:upload-failed-job-logs:ccache-show-stats
     - .rules:build:macos
-  tags:
-    - macos
   parallel: 3
+
+pytest_build_system_minimal_cmake:
+  extends: .test_build_system_minimal_cmake_template
+
+pytest_build_system_macos_minimal_cmake:
+  extends:
+    - .test_build_system_minimal_cmake_template
+    - .test_build_system_macos_template_extension
+    - .before_script:build:macos
+    - .after_script:build:macos:upload-failed-job-logs:ccache-show-stats
+    - .rules:build:macos
   variables:
-    PYENV_VERSION: "3.10"
-    # CCACHE_DIR: "/cache/idf_ccache". On macOS, you cannot write to this folder due to insufficient permissions.
-    CCACHE_DIR: "" # ccache will use "$HOME/Library/Caches/ccache".
-    # Workaround for a bug in Parallels executor where CI_PROJECT_DIR is not an absolute path,
-    # but a relative path to the build directory (builds/espressif/esp-idf instead of ~/builds/espressif/esp-idf.
-    # GitLab sets the project dir to this template `<builds_dir>/<namespace>/<project_name>`
-    IDF_PATH: "/Users/espressif/builds/espressif/esp-idf"
+    INSTALL_EXTRA_TOOLS: ninja [email protected]
 
 build_docker:
   extends:
@@ -243,17 +284,11 @@ build_docker:
   stage: host_test
   needs: []
   image: espressif/docker-builder:1
-  tags:
-    - build_docker_amd64_brno
+  tags: [shiny, dind]
   variables:
     DOCKER_TMP_IMAGE_NAME: "idf_tmp_image"
   script:
-    - export LOCAL_CI_REPOSITORY_URL=$CI_REPOSITORY_URL
-    - if [ -n "$LOCAL_GITLAB_HTTPS_HOST" ]; then export LOCAL_CI_REPOSITORY_URL="https://gitlab-ci-token:${CI_JOB_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}"; fi
-    - if [ -n "$LOCAL_GIT_MIRROR" ]; then export LOCAL_CI_REPOSITORY_URL="${LOCAL_GIT_MIRROR}/${CI_PROJECT_PATH}"; fi
-    - echo "Using repository at $LOCAL_CI_REPOSITORY_URL"
-    - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${LOCAL_CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$PIPELINE_COMMIT_SHA}"
-    # Build
+    - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} --build-arg IDF_CLONE_SHALLOW=1 --build-arg IDF_GITHUB_ASSETS=${INTERNAL_GITHUB_ASSETS}"
     - docker build --tag ${DOCKER_TMP_IMAGE_NAME} ${DOCKER_BUILD_ARGS} tools/docker/
     # We can't mount $PWD/examples/get-started/blink into the container, see https://gitlab.com/gitlab-org/gitlab-ce/issues/41227.
     # The workaround mentioned there works, but leaves around directories which need to be cleaned up manually.
@@ -329,6 +364,6 @@ generate_disabled_apps_report:
     expire_in: 1 week
     when: always
   script:
-    - pip install dominate idf-build-apps>=2.12.0
+    - pip install dominate idf-build-apps
     - run_cmd python tools/ci/gen_disabled_report.py --output disabled_report.html --verbose --enable-preview-targets
     - echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/disabled_report.html"

.gitlab/ci/common.yml

@@ -52,7 +52,7 @@ variables:
   CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
 
   # Docker images
-  ESP_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-env-v6.0:2"
+  ESP_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-env-v6.0:3"
   ESP_IDF_DOC_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-doc-env-v6.0:2-1"
   TARGET_TEST_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/target-test-env-v6.0:2"
   SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5"
@@ -97,6 +97,7 @@ variables:
   CCACHE_MAXSIZE: "50G"
 
   FF_USE_NEW_BASH_EVAL_STRATEGY: "true"
+  FORCE_COLOR: "1"  # rich print with color
 
 ################################################
 # `before_script` and `after_script` Templates #
@@ -334,9 +335,10 @@ variables:
       TEMP_FILE=$(mktemp)
       # merged results pipelines, by default
       if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then
-        git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE"
+        git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA
+        git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
+
+        git diff --name-only $CI_MERGE_REQUEST_TARGET_BRANCH_SHA...$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE"
         GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
         git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
       # merge request pipelines, when the mr got conflicts

.gitlab/ci/default-build-test-rules.yml

@@ -2,15 +2,12 @@
 # - extra_default_build_targets:
 #     besides of the SUPPORTED_TARGETS in IDF,
 #     enable build for the specified targets by default as well.
+# !!! DEPRECATED: use `additional_build_targets` in .idf_build_apps.toml instead
+#
 # - bypass_check_test_targets:
 #     suppress the check_build_test_rules check-test-script warnings for the specified targets
 #
 # This file should ONLY be used during bringup. Should be reset to empty after the bringup process
-extra_default_build_targets:
-  - esp32c5
-  - esp32c61
-  - esp32h21
-  - esp32h4
 
 bypass_check_test_targets:
   - esp32h21

.gitlab/ci/host-test.yml

@@ -77,24 +77,12 @@ test_fatfsgen_on_host:
     - ./test_wl_fatfsgen.py
     - ./test_fatfsparse.py
 
-test_multi_heap_on_host:
-  extends: .host_test_template
-  script:
-    - cd components/heap/test_multi_heap_host
-    - ./test_all_configs.sh
-
 test_certificate_bundle_on_host:
   extends: .host_test_template
   script:
     - cd components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/
     - ./test_gen_crt_bundle.py
 
-test_gdbstub_on_host:
-  extends: .host_test_template
-  script:
-    - cd components/esp_gdbstub/test_gdbstub_host
-    - make test
-
 # Test for create virtualenv. It must be invoked from Python, not from virtualenv.
 # Use docker image system python without any extra dependencies
 test_cli_installer:
@@ -200,20 +188,21 @@ test_tools:
     INSTALL_EXTRA_TOOLS: "qemu-xtensa qemu-riscv32"  # for test_idf_qemu.py
   script:
     - stat=0
+    - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
     - cd ${IDF_PATH}/tools/ci/test_autocomplete
-    - pytest --noconftest test_autocomplete.py --junitxml=${IDF_PATH}/XUNIT_AUTOCOMP.xml                  || stat=1
+    - run_cmd pytest --noconftest test_autocomplete.py --junitxml=${IDF_PATH}/XUNIT_AUTOCOMP.xml     --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}             || stat=1
     - cd ${IDF_PATH}/tools/test_idf_py
-    - pytest --noconftest test_idf_py.py       --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml                    || stat=1
-    - pytest --noconftest test_hints.py        --junitxml=${IDF_PATH}/XUNIT_HINTS.xml                     || stat=1
-    - pytest --noconftest test_idf_qemu.py     --junitxml=${IDF_PATH}/XUNIT_IDF_PY_QEMU.xml               || stat=1
+    - run_cmd pytest --noconftest test_idf_py.py       --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}             || stat=1
+    - run_cmd pytest --noconftest test_hints.py        --junitxml=${IDF_PATH}/XUNIT_HINTS.xml        --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}             || stat=1
+    - run_cmd pytest --noconftest test_idf_qemu.py     --junitxml=${IDF_PATH}/XUNIT_IDF_PY_QEMU.xml  --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}             || stat=1
     - cd ${IDF_PATH}/tools/test_bsasm
-    - pytest --noconftest test_bsasm.py        --junitxml=${IDF_PATH}/XUNIT_BSASM.xml                     || stat=1
+    - run_cmd pytest --noconftest test_bsasm.py        --junitxml=${IDF_PATH}/XUNIT_BSASM.xml        --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}             || stat=1
     - cd ${IDF_PATH}/tools/test_mkdfu
-    - pytest --noconftest test_mkdfu.py        --junitxml=${IDF_PATH}/XUNIT_MKDFU.xml                     || stat=1
+    - run_cmd pytest --noconftest test_mkdfu.py        --junitxml=${IDF_PATH}/XUNIT_MKDFU.xml        --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}             || stat=1
     - cd ${IDF_PATH}/tools/test_idf_size
-    - pytest --noconftest test_idf_size.py     --junitxml=${IDF_PATH}/XUNIT_IDF_SIZE.xml                  || stat=1
+    - run_cmd pytest --noconftest test_idf_size.py     --junitxml=${IDF_PATH}/XUNIT_IDF_SIZE.xml     --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}             || stat=1
     - cd ${IDF_PATH}/tools/test_idf_diag
-    - pytest --noconftest test_idf_diag.py     --junitxml=${IDF_PATH}/XUNIT_IDF_DIAG.xml                  || stat=1
+    - run_cmd pytest --noconftest test_idf_diag.py     --junitxml=${IDF_PATH}/XUNIT_IDF_DIAG.xml     --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}             || stat=1
     - cd ${IDF_PATH}
     - shellcheck -s sh tools/detect_python.sh                                                             || stat=1
     - shellcheck -s bash tools/detect_python.sh                                                           || stat=1
@@ -230,13 +219,6 @@ test_split_path_by_spaces:
     - cd ${IDF_PATH}/tools
     - python -m unittest split_paths_by_spaces.py
 
-test_mqtt_on_host:
-  extends: .host_test_template
-  script:
-    - cd ${IDF_PATH}/components/mqtt/esp-mqtt/host_test
-    - idf.py build
-    - LSAN_OPTIONS=verbosity=1:log_threads=1 build/host_mqtt_client_test.elf
-
 test_transport_on_host:
   extends: .host_test_template
   allow_failure: true # IDFCI-2781 [v5.5, v5.4] test_transport_on_host fails on ubuntu 24.04

.gitlab/ci/pre_check.yml

@@ -142,6 +142,10 @@ pipeline_variables:
         MR_MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | tr '\n' ';')
         echo "MR_MODIFIED_COMPONENTS=\"$MR_MODIFIED_COMPONENTS\"" >> pipeline.env
       fi
+    - |
+      if [ -n "$PIPELINE_COMMIT_SHA" ]; then
+        echo "PIPELINE_COMMIT_SHA=$PIPELINE_COMMIT_SHA" >> pipeline.env
+      fi
     - echo "OOCD_DISTRO_URL_ARMHF=$OOCD_DISTRO_URL_ARMHF" >> pipeline.env
     - echo "OOCD_DISTRO_URL_ARM64=$OOCD_DISTRO_URL_ARM64" >> pipeline.env
     - run_cmd idf-ci gitlab pipeline-variables >> pipeline.env

.gitlab/ci/rules.yml

@@ -147,7 +147,6 @@
   - "components/esp_phy/lib"
   - "components/esp_wifi/lib"
   - "components/esp_coex/lib"
-  - "components/json/cJSON"
   - "components/lwip/lwip"
   - "components/mbedtls/mbedtls"
   - "components/mqtt/esp-mqtt"