Merge branch 'master' into master

Author: DogushC

.github/CI_README.md

@@ -604,7 +604,7 @@ requires:
 Some tests require two physical devices (e.g., BLE server/client, WiFi AP/client). These are defined using the `multi_device` field in `ci.yml`, where each entry points to a sketch directory for that device.
 
 **How it works:**
-- `tests_build.sh` builds each device sketch separately and stores artifacts under `~/.arduino/tests/<target>/<test>_<sketch>/build.tmp`.
+- `tests_build.sh` builds each device sketch separately and stores artifacts under `~/.arduino/tests/<target>/<test>/<sketch>/build.tmp`.
 - `tests_run.sh` assembles multiple build directories and ports using `|` separators, then runs pytest with `--count` and `--embedded-services` for each DUT.
 - Only **2 devices** are supported currently.
 - **Hardware only**: multi-device tests are blocked for Wokwi and QEMU.

.github/scripts/generate_missing_junits.py

@@ -100,29 +100,29 @@ def expected_from_artifacts(build_root: Path) -> dict[tuple[str, str, str, str],
         test_type = m.group(2)
         print(f"[DEBUG] Artifact group target={target} type={test_type} dir={artifact_dir}", file=sys.stderr)
 
-        # Group build*.tmp directories by sketch
-        # Structure: test-bin-<target>-<type>/<sketch>/build*.tmp/
-        sketches_processed = set()
-        # Track multi-device test names to avoid counting them multiple times
-        # (e.g., wifi_ap_ap and wifi_ap_client both map to test name wifi_ap)
-        multi_device_tests_processed = set()
-
-        # Find all build*.tmp directories and process each sketch once
+        # Find all build*.tmp directories and determine test names.
+        # Directory layout:
+        #   Single-device: test-bin-<target>-<type>/<test>/build*.tmp/
+        #   Multi-device:  test-bin-<target>-<type>/<test>/<device>/build*.tmp/
+        # The test name is always the first path component after the artifact group.
+        tests_processed = set()
+
         for build_tmp in artifact_dir.rglob("build*.tmp"):
             if not build_tmp.is_dir():
                 continue
             if not re.search(r"build\d*\.tmp$", build_tmp.name):
                 continue
 
-            # Path structure is: test-bin-<target>-<type>/<sketch>/build*.tmp/
-            sketch = build_tmp.parent.name
+            # Test name is always the first directory component after the artifact group
+            rel = build_tmp.relative_to(artifact_dir)
+            effective_sketch = rel.parts[0]
 
-            # Skip if we already processed this sketch
-            if sketch in sketches_processed:
+            # Skip if we already processed this test
+            if effective_sketch in tests_processed:
                 continue
-            sketches_processed.add(sketch)
+            tests_processed.add(effective_sketch)
 
-            print(f"[DEBUG]  Processing sketch={sketch} from artifact {artifact_dir.name}", file=sys.stderr)
+            print(f"[DEBUG]  Processing test={effective_sketch} from artifact {artifact_dir.name}", file=sys.stderr)
 
             ci_path = build_tmp / "ci.yml"
             sdk_path = build_tmp / "sdkconfig"
@@ -146,26 +146,6 @@ def expected_from_artifacts(build_root: Path) -> dict[tuple[str, str, str, str],
             ci = _parse_ci_yml(ci_text)
             fqbn_counts = _fqbn_counts_from_yaml(ci)
 
-            # For multi-device tests, the build artifact sketch name follows the
-            # pattern {test_name}_{device_sketch} (e.g., wifi_ap_ap, wifi_ap_client).
-            # Derive the original test name to match the test result XML structure,
-            # which uses the test name (e.g., validation/wifi_ap/esp32/wifi_ap.xml).
-            effective_sketch = sketch
-            multi_device = ci.get("multi_device") if isinstance(ci, dict) else None
-            if isinstance(multi_device, dict):
-                device_names = [str(v) for v in multi_device.values()]
-                for dname in device_names:
-                    suffix = f"_{dname}"
-                    if sketch.endswith(suffix):
-                        effective_sketch = sketch[:-len(suffix)]
-                        break
-                # Only process each multi-device test once across all its device sketches
-                if effective_sketch in multi_device_tests_processed:
-                    print(f"[DEBUG]   Skip (multi-device test already processed): {sketch} -> {effective_sketch}", file=sys.stderr)
-                    continue
-                multi_device_tests_processed.add(effective_sketch)
-                print(f"[DEBUG]   Multi-device test detected: {sketch} -> {effective_sketch}", file=sys.stderr)
-
             # Determine allowed platforms for this test
             # Performance tests are only run on hardware
             if test_type == "performance":
@@ -198,7 +178,7 @@ def expected_from_artifacts(build_root: Path) -> dict[tuple[str, str, str, str],
                 expected[(plat, target, test_type, effective_sketch)] = exp_runs
                 print(f"[DEBUG]   Expected: plat={plat} target={target} type={test_type} sketch={effective_sketch} runs={exp_runs}", file=sys.stderr)
 
-        if len(sketches_processed) == 0:
+        if len(tests_processed) == 0:
             print(f"[DEBUG]  No sketches found in this artifact group", file=sys.stderr)
     return expected
 

.github/scripts/sketch_utils.sh

@@ -102,6 +102,10 @@ function build_sketch { # build_sketch <ide_path> <user_path> <path-to-ino> [ext
             shift
             ci_yml_dir=$1
             ;;
+        -bn )
+            shift
+            build_name=$1
+            ;;
         * )
             break
             ;;
@@ -132,13 +136,23 @@ function build_sketch { # build_sketch <ide_path> <user_path> <path-to-ino> [ext
         # precedence.  Note that the following logic also falls to the default
         # parameters if no arguments were passed and no file was found.
 
-        if [ -z "$options" ] && [ -f "$sketchdir"/ci.yml ]; then
+        # Resolve which ci.yml to use for FQBN / fqbn_append lookups.
+        # For multi-device tests (-td), the sub-sketch directory does not
+        # contain its own ci.yml; the parent test directory does.
+        local ci_yml_for_build=""
+        if [ -f "$sketchdir"/ci.yml ]; then
+            ci_yml_for_build="$sketchdir/ci.yml"
+        elif [ -n "$ci_yml_dir" ] && [ -f "$ci_yml_dir/ci.yml" ]; then
+            ci_yml_for_build="$ci_yml_dir/ci.yml"
+        fi
+
+        if [ -z "$options" ] && [ -n "$ci_yml_for_build" ]; then
             # The config file could contain multiple FQBNs for one chip.  If
             # that's the case we build one time for every FQBN.
 
-            len=$(yq eval ".fqbn.${target} | length" "$sketchdir"/ci.yml 2>/dev/null || echo 0)
+            len=$(yq eval ".fqbn.${target} | length" "$ci_yml_for_build" 2>/dev/null || echo 0)
             if [ "$len" -gt 0 ]; then
-                fqbn=$(yq eval ".fqbn.${target} | sort | @json" "$sketchdir"/ci.yml)
+                fqbn=$(yq eval ".fqbn.${target} | sort | @json" "$ci_yml_for_build")
             fi
         fi
 
@@ -148,8 +162,8 @@ function build_sketch { # build_sketch <ide_path> <user_path> <path-to-ino> [ext
 
             len=1
 
-            if [ -f "$sketchdir"/ci.yml ]; then
-                fqbn_append=$(yq eval '.fqbn_append' "$sketchdir"/ci.yml 2>/dev/null)
+            if [ -n "$ci_yml_for_build" ]; then
+                fqbn_append=$(yq eval '.fqbn_append' "$ci_yml_for_build" 2>/dev/null)
                 if [ "$fqbn_append" == "null" ]; then
                     fqbn_append=""
                 fi
@@ -228,14 +242,6 @@ function build_sketch { # build_sketch <ide_path> <user_path> <path-to-ino> [ext
         exit 1
     fi
 
-    # The directory that will hold all the artifacts (the build directory) is
-    # provided through:
-    #  1. An env variable called ARDUINO_BUILD_DIR.
-    #  2. Created at the sketch level as "build" in the case of a single
-    #     configuration test.
-    #  3. Created at the sketch level as "buildX" where X is the number
-    #     of configuration built in case of a multiconfiguration test.
-
     sketchname=$(basename "$sketchdir")
     local has_requirements
 
@@ -255,29 +261,36 @@ function build_sketch { # build_sketch <ide_path> <user_path> <path-to-ino> [ext
     fi
 
     # Install libraries from ci.yml if they exist
-    install_libs -ai "$ide_path" -s "$sketchdir"
+    local ci_yml_lib_dir="${ci_yml_dir:-$sketchdir}"
+    install_libs -ai "$ide_path" -s "$ci_yml_lib_dir"
     install_result=$?
     if [ $install_result -ne 0 ]; then
         echo "ERROR: Library installation failed for $sketchname" >&2
         exit $install_result
     fi
 
-    ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp"
-    if [ -n "$ARDUINO_BUILD_DIR" ]; then
-        build_dir="$ARDUINO_BUILD_DIR"
-    elif [ "$len" -eq 1 ]; then
-        # build_dir="$sketchdir/build"
-        build_dir="$HOME/.arduino/tests/$target/$sketchname/build.tmp"
-    fi
+    # Build directory path:
+    #   ARDUINO_BUILD_DIR env var → full override (used by on-push.sh for non-test builds)
+    #   Otherwise → ~/.arduino/tests/$target/$build_output_name/build[N].tmp
+    #     -bn sets a parent directory (test name) for multi-device builds:
+    #       e.g. -bn wifi_ap + sketchname=ap → wifi_ap/ap/build.tmp
+    #     Without -bn, uses $sketchname directly:
+    #       e.g. sketchname=gpio → gpio/build.tmp
+    local build_output_name="${build_name:+$build_name/}$sketchname"
 
+    ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp"
     output_file="$HOME/.arduino/cli_compile_output.txt"
     sizes_file="$GITHUB_WORKSPACE/cli_compile_$chunk_index.json"
 
     mkdir -p "$ARDUINO_CACHE_DIR"
     for i in $(seq 0 $((len - 1))); do
-        if [ "$len" -ne 1 ]; then
-            # build_dir="$sketchdir/build$i"
-            build_dir="$HOME/.arduino/tests/$target/$sketchname/build$i.tmp"
+        if [ -n "$ARDUINO_BUILD_DIR" ]; then
+            # Full override from env var (non-test builds like on-push.sh)
+            build_dir="$ARDUINO_BUILD_DIR"
+        elif [ "$len" -eq 1 ]; then
+            build_dir="$HOME/.arduino/tests/$target/$build_output_name/build.tmp"
+        else
+            build_dir="$HOME/.arduino/tests/$target/$build_output_name/build$i.tmp"
         fi
         rm -rf "$build_dir"
         mkdir -p "$build_dir"

.github/scripts/tests_build.sh

@@ -64,34 +64,6 @@ function extract_target_and_args {
     done
 }
 
-# Build a single sketch for multi-device tests
-function build_multi_device_sketch {
-    local test_name=$1
-    local sketch_path=$2
-    local target=$3
-    shift 3
-    local build_args=("$@")
-    local sketch_dir
-    local sketch_name
-    local test_dir
-    local build_dir
-
-    sketch_dir=$(dirname "$sketch_path")
-    sketch_name=$(basename "$sketch_dir")
-    test_dir=$(dirname "$sketch_dir")
-
-    # Override build directory to use <test_name>_<sketch_name> pattern
-    build_dir="$HOME/.arduino/tests/$target/${test_name}_${sketch_name}/build.tmp"
-
-    echo "Building sketch $sketch_name for multi-device test $test_name"
-
-    # Call sketch_utils.sh build function with custom build directory
-    # Pass -td to point to the parent test directory where ci.yml lives,
-    # since individual sub-sketch directories don't have their own ci.yml.
-    ARDUINO_BUILD_DIR="$build_dir" ${SKETCH_UTILS} build "${build_args[@]}" -s "$sketch_dir" -td "$test_dir"
-    return $?
-}
-
 # Build all sketches for a multi-device test
 function build_multi_device_test {
     local test_dir=$1
@@ -133,16 +105,23 @@ function build_multi_device_test {
     local result=0
     local sketch_name
     local sketch_path
+    local sketch_dir
     for device in $devices; do
         sketch_name=$(yq eval ".multi_device.$device" "$test_dir/ci.yml" 2>/dev/null)
         sketch_path="$test_dir/$sketch_name/$sketch_name.ino"
+        sketch_dir="$test_dir/$sketch_name"
 
         if [ ! -f "$sketch_path" ]; then
             echo "ERROR: Sketch not found: $sketch_path"
             return 1
         fi
 
-        build_multi_device_sketch "$test_name" "$sketch_path" "$target" "${build_args[@]}"
+        echo "Building sketch $sketch_name for multi-device test $test_name"
+
+        # -td: ci.yml lives in the parent test directory
+        # -bn: sets the parent build dir to the test name (nested: $test_name/$sketch_name/build.tmp)
+        ${SKETCH_UTILS} build "${build_args[@]}" -s "$sketch_dir" \
+            -td "$test_dir" -bn "$test_name"
         result=$?
         if [ $result -ne 0 ]; then
             echo "ERROR: Failed to build sketch $sketch_name for test $test_name"