Matter over WiFi (esp-idf component inside an arduino-esp32 project)

CMakeLists.txt

@@ -0,0 +1,15 @@
+cmake_minimum_required(VERSION 3.16.0)
+set(EXECUTABLE_COMPONENT_NAME "wled00")
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+
+# ── Matter / GCC 14 compatibility fixups ──────────────────────────────────────
+# The CHIP SDK's TypeTraits.h has a C++23 branch with invalid code:
+#   using to_underlying = std::to_underlying;  (type alias to function template)
+# Downgrade from gnu++2b to gnu++20 so the correct #else polyfill is used.
+# Guard: only apply when the Matter usermod is present.
+if(EXISTS "${CMAKE_SOURCE_DIR}/usermods/usermod_v2_matter")
+  idf_build_replace_option_from_property(CXX_COMPILE_OPTIONS
+    "-std=gnu++2b" "-std=gnu++20")
+endif()
+
+project(WLED)

pio-scripts/dynarray.py

@@ -22,13 +22,20 @@ def inject_before_marker(path, marker):
 
 
 if env.get("PIOPLATFORM") == "espressif32":
-    # Find sections.ld on the linker search path (LIBPATH).
+    # Find sections.ld on the linker search path (LIBPATH), or fall back to
+    # BUILD_DIR (the dual-framework espidf+arduino build generates sections.ld
+    # directly in BUILD_DIR and passes it as a bare filename in LINKFLAGS).
     sections_ld_path = None
     for ld_dir in env.get("LIBPATH", []):
         candidate = Path(str(ld_dir)) / "sections.ld"
         if candidate.exists():
             sections_ld_path = candidate
             break
+    if sections_ld_path is None:
+        build_dir = Path(env.subst("$BUILD_DIR"))
+        candidate = build_dir / "sections.ld"
+        if candidate.exists():
+            sections_ld_path = candidate
 
     if sections_ld_path is not None:
         # Inject inside the existing .flash.rodata output section, just before

pio-scripts/fix_nodelist.py

@@ -0,0 +1,125 @@
+Import("env")
+# ── fix_nodelist.py ───────────────────────────────────────────────────────────
+# Pre-build monkey-patch that prevents a clean-build crash in dual-framework
+# (arduino + espidf) PlatformIO builds.
+#
+# Root cause
+# ----------
+# platformio/builder/tools/piobuild.py::CollectBuildFiles() calls
+#   env.File(os.path.join(_var_dir, os.path.basename(item)))
+# for each matched source file.  When multiple VariantDir mappings overlap
+# (which happens with libraries that have sub-directories such as the
+# Arduino WebServer library's detail/, middleware/, uri/ sub-trees), SCons
+# can return a NodeList instead of a single Node.
+#
+# Additionally, middlewares such as arduino.py's smart_include_length_shorten
+# call env.Object(node) which also returns a NodeList (SCons Object builder
+# returns a list).  When the next middleware (_skip_prj_source_files from
+# espidf.py) receives a NodeList, it calls node.srcnode().get_path() which
+# fails with:
+#   AttributeError: 'NodeList' object has no attribute 'srcnode'
+#   (or produces a wrong result when NodeList.__getattr__ proxies the call).
+#
+# Fix
+# ---
+# We monkey-patch CollectBuildFiles to:
+#  1. Flatten NodeList items produced by env.File() BEFORE the middleware loop.
+#  2. Flatten NodeList values returned BY middleware callbacks during the loop.
+#     When a callback returns a NodeList, all elements pass the remaining
+#     middlewares individually.
+# ─────────────────────────────────────────────────────────────────────────────
+
+import fnmatch as _fnmatch
+import os as _os
+
+from platformio.builder.tools import piobuild as _piobuild
+from platformio.builder.tools.piobuild import SRC_BUILD_EXT
+
+
+def _is_nodelist(node):
+    """Return True if node is a SCons NodeList (a UserList subclass that
+    proxies attribute access but does not have get_path() directly)."""
+    return hasattr(node, 'data') and not hasattr(node, 'get_path')
+
+
+def _iter_nodes(node):
+    """Yield individual SCons nodes from node, recursively unwrapping NodeList."""
+    if _is_nodelist(node):
+        for child in node.data:
+            yield from _iter_nodes(child)
+    else:
+        yield node
+
+
+def _run_middlewares(env, node, middlewares):
+    """Run all middlewares on a single node.  Returns a list of result nodes
+    (empty if the node was filtered out, >1 if a middleware returned a NodeList).
+    """
+    current_nodes = [node]
+    for callback, pattern in middlewares:
+        next_nodes = []
+        for n in current_nodes:
+            if pattern and not _fnmatch.fnmatch(n.srcnode().get_path(), pattern):
+                next_nodes.append(n)  # pattern didn't match – pass through unchanged
+                continue
+            if callback.__code__.co_argcount == 2:
+                result = callback(env, n)
+            else:
+                result = callback(n)
+            if result is None:
+                pass  # filtered out
+            elif _is_nodelist(result):
+                next_nodes.extend(_iter_nodes(result))
+            else:
+                next_nodes.append(result)
+        current_nodes = next_nodes
+        if not current_nodes:
+            break  # all nodes filtered out
+    return current_nodes
+
+
+def _patched_CollectBuildFiles(env, variant_dir, src_dir, src_filter=None, duplicate=False):
+    """Drop-in replacement for piobuild.CollectBuildFiles that handles NodeList
+    nodes both from env.File() and from middleware callbacks."""
+    from platformio.builder.tools.piobuild import SRC_BUILD_EXT as _SRC_BUILD_EXT
+
+    sources = []
+    variants = []
+
+    src_dir = env.subst(src_dir)
+    if src_dir.endswith(_os.sep):
+        src_dir = src_dir[:-1]
+
+    for item in env.MatchSourceFiles(src_dir, src_filter, _SRC_BUILD_EXT):
+        _reldir = _os.path.dirname(item)
+        _src_dir = _os.path.join(src_dir, _reldir) if _reldir else src_dir
+        _var_dir = _os.path.join(variant_dir, _reldir) if _reldir else variant_dir
+
+        if _var_dir not in variants:
+            variants.append(_var_dir)
+            env.VariantDir(_var_dir, _src_dir, duplicate)
+
+        raw_node = env.File(_os.path.join(_var_dir, _os.path.basename(item)))
+        # Flatten: a NodeList becomes individual nodes; a plain Node stays as-is
+        for n in _iter_nodes(raw_node):
+            sources.append(n)
+
+    middlewares = env.get("__PIO_BUILD_MIDDLEWARES")
+    if not middlewares:
+        return sources
+
+    new_sources = []
+    for node in sources:
+        for result_node in _run_middlewares(env, node, middlewares):
+            new_sources.append(result_node)
+
+    return new_sources
+
+
+# Replace the function in the piobuild module so all subsequent callers
+# (espidf.py, etc.) pick up the patched version.
+_piobuild.CollectBuildFiles = _patched_CollectBuildFiles
+# Also replace the env-bound method so env.CollectBuildFiles() uses our version.
+env.AddMethod(_patched_CollectBuildFiles, "CollectBuildFiles")
+
+print("[fix_nodelist] CollectBuildFiles patched – NodeList flattening enabled")

pio-scripts/generate_embed_files.py

@@ -0,0 +1,96 @@
+Import("env")
+from pathlib import Path
+
+# ── generate_embed_files.py ───────────────────────────────────────────────────
+# Pre-build script for the esp32s3_matter_wifi environment.
+#
+# Generates .S assembly files for binary data that ESP-IDF components
+# expect to be embedded via CMake's target_add_binary_data().
+# PlatformIO's SCons build doesn't execute those CMake commands,
+# so we replicate the output here.
+#
+# The .S files are generated at BUILD_DIR/<name>.S – exactly the path that
+# CMake's project_description.json records as the source path for these
+# generated files.  espidf.py's compile_source_files() will then find and
+# compile them correctly.
+#
+# DO NOT place them in a sub-directory: the CMake code model records relative
+# paths (e.g. ".pio/build/esp32s3_matter_wifi/https_server.crt.S") and
+# espidf.py resolves those relative to BUILD_DIR.
+# ─────────────────────────────────────────────────────────────────────────────
+
+project_dir = Path(env["PROJECT_DIR"]).resolve()
+build_dir   = Path(env.subst("$BUILD_DIR")).resolve()
+
+# Map of (source cert path, symbol name) pairs that need embedding.
+# These correspond to all target_add_binary_data() calls in the managed
+# components' CMakeLists.txt files (which PlatformIO's SCons build skips).
+EMBED_FILES = [
+    (
+        project_dir / "managed_components" / "espressif__esp_insights" / "server_certs" / "https_server.crt",
+        "https_server_crt",
+    ),
+    (
+        project_dir / "managed_components" / "espressif__esp_insights" / "server_certs" / "mqtt_server.crt",
+        "mqtt_server_crt",
+    ),
+    (
+        project_dir / "managed_components" / "espressif__esp_rainmaker" / "server_certs" / "rmaker_mqtt_server.crt",
+        "rmaker_mqtt_server_crt",
+    ),
+    (
+        project_dir / "managed_components" / "espressif__esp_rainmaker" / "server_certs" / "rmaker_claim_service_server.crt",
+        "rmaker_claim_service_server_crt",
+    ),
+    (
+        project_dir / "managed_components" / "espressif__esp_rainmaker" / "server_certs" / "rmaker_ota_server.crt",
+        "rmaker_ota_server_crt",
+    ),
+]
+
+
+def generate_asm(src_path: Path, symbol: str, out_dir: Path):
+    """Generate an assembly .S file that embeds binary data, matching the
+    format produced by ESP-IDF's ``target_add_binary_data()``."""
+    if not src_path.exists():
+        print(f"  [embed] WARNING: {src_path} not found – skipping")
+        return None
+
+    out_file = out_dir / f"{src_path.name}.S"
+    data = src_path.read_bytes()
+
+    lines = [
+        f"/* Data converted from {src_path} */",
+        ".data",
+        "#if !defined (__APPLE__) && !defined (__linux__)",
+        ".section .rodata.embedded",
+        "#endif",
+        "",
+        f".global {symbol}",
+        f"{symbol}:",
+        "",
+        f".global _binary_{symbol}_start",
+        f"_binary_{symbol}_start: /* for objcopy compatibility */",
+    ]
+
+    for i in range(0, len(data), 16):
+        chunk = data[i : i + 16]
+        hexvals = ", ".join(f"0x{b:02x}" for b in chunk)
+        lines.append(f".byte {hexvals}")
+
+    lines += [
+        "",
+        f".global _binary_{symbol}_end",
+        f"_binary_{symbol}_end: /* for objcopy compatibility */",
+        ".byte 0x00",  # null terminator for TEXT mode
+    ]
+
+    out_dir.mkdir(parents=True, exist_ok=True)
+    out_file.write_text("\n".join(lines) + "\n")
+    print(f"  [embed] Generated {out_file} ({len(data)} bytes)")
+    return str(out_file)
+
+
+for src_path, symbol in EMBED_FILES:
+    generate_asm(src_path, symbol, build_dir)
+

pio-scripts/setup_matter_component.py

@@ -0,0 +1,34 @@
+Import("env")
+import shutil
+from pathlib import Path
+from SCons.Script import Exit
+
+# ── setup_matter_component.py ─────────────────────────────────────────────────
+# Pre-build script for the esp32s3_matter_wifi environment.
+#
+# Copies the idf_component.yml manifest from the Matter usermod into wled00/
+# so that pioarduino's built-in ComponentManager (and the IDF Component Manager)
+# can resolve the espressif/esp_matter dependency.
+#
+# The destination file is listed in .gitignore and must NOT be committed.
+# ─────────────────────────────────────────────────────────────────────────────
+
+project_dir = Path(env["PROJECT_DIR"]).resolve()
+src_yml     = project_dir / "usermods" / "usermod_v2_matter" / "idf_component.yml"
+dst_yml     = project_dir / "wled00"   / "idf_component.yml"
+
+if not src_yml.exists():
+    print(
+        "\033[0;31;43m"
+        f"Matter: idf_component.yml not found at {src_yml} – "
+        "cannot resolve the esp_matter component."
+        "\033[0m"
+    )
+    Exit(1)
+
+shutil.copy2(str(src_yml), str(dst_yml))
+print(
+    "\033[6;33;42m"
+    "Matter: copied idf_component.yml → wled00/idf_component.yml"
+    "\033[0m"
+)