ESP32: Modernize Build System Infrastructure (Step 1)

This PR introduces the core infrastructure for a data-driven build system for ESP32, aligning it further with the standard ArduPilot hwdef.dat approach used by ChibiOS.

Key Infrastructure Changes:
- Generalized esp32_hwdef.py to support both V1 (Current Master) and V2 (Standard-aligned) syntax via the ESP32_HWDEF_V2 flag.
- Integrated auto-discovery in boards.py to support legacy headers and hwdef subdirectories in a unified 'boards/' location.
- Automated sdkconfig generation and partition handling in esp32.py for V2-enabled boards.
- Introduced feature-based capability defines (e.g., HAL_ESP32_HAS_MCPWM) to replace target-specific ifdefs.
- Restored legacy board headers to provide a risk-free, staged migration path.
- Added MIGRATION.md detailing the transition path from V1 to V2 syntax.

This is Step 1 of a multi-stage refactor. Existing boards in master remain functional through the compatibility parser.

Relates to #30709

Author: Bwooce

Tools/ardupilotwaf/boards.py

@@ -681,18 +681,28 @@ def add_dynamic_boards_from_hwdef_dir(base_type, hwdef_dir):
             newclass = type(d, (base_type,), {'name': d})
 
 def add_dynamic_boards_esp32():
-    '''add boards based on existence of hwdef.dat in subdirectories for ESP32'''
-    dirname, dirlist, filenames = next(os.walk('libraries/AP_HAL_ESP32/hwdef'))
-    for d in dirlist:
-        if d in _board_classes.keys():
-            continue
-        hwdef = os.path.join(dirname, d, 'hwdef.dat')
-        if os.path.exists(hwdef):
-            mcu_esp32s3 = True if (d[0:7] == "esp32s3") else False
-            if mcu_esp32s3:
-                newclass = type(d, (esp32s3,), {'name': d})
-            else:
-                newclass = type(d, (esp32,), {'name': d})
+    '''add boards based on existence of hwdef.dat in hwdef subdirectories or legacy headers in boards directory'''
+    hwdef_base = 'libraries/AP_HAL_ESP32/hwdef'
+    if os.path.exists(hwdef_base):
+        dirname, dirlist, filenames = next(os.walk(hwdef_base))
+        for d in dirlist:
+            if d in _board_classes.keys() or d == 'scripts':
+                continue
+            hwdef_path = os.path.join(dirname, d, 'hwdef.dat')
+            if os.path.exists(hwdef_path):
+                mcu_esp32s3 = True if (d[0:7] == "esp32s3") else False
+                type(d, (esp32s3 if mcu_esp32s3 else esp32,), {'name': d, 'hwdef': hwdef_path})
+    
+    # Check for legacy headers in boards directory
+    legacy_base = 'libraries/AP_HAL_ESP32/boards'
+    if os.path.exists(legacy_base):
+        dirname, dirlist, filenames = next(os.walk(legacy_base))
+        for f in filenames:
+            if f.endswith('.h'):
+                board_name = f[:-2]
+                if board_name not in _board_classes:
+                    mcu_esp32s3 = True if (board_name[0:7] == "esp32s3") else False
+                    type(board_name, (esp32s3 if mcu_esp32s3 else esp32,), {'name': board_name})
 
 def get_boards_names():
     add_dynamic_boards_chibios()
@@ -995,13 +1005,44 @@ class esp32(Board):
     abstract = True
     toolchain = 'xtensa-esp32-elf'
 
+    def __init__(self):
+        super().__init__()
+        self.with_can = True
+
     def configure(self, cfg):
+        # run hwdef to get MCU if this is an hwdef board
+        hwdef_dat = getattr(self, 'hwdef', None)
+        if hwdef_dat:
+            sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../libraries/AP_HAL_ESP32/hwdef/scripts'))
+            import esp32_hwdef
+            if not os.path.exists(cfg.bldnode.abspath()):
+                os.makedirs(cfg.bldnode.abspath())
+            eh = esp32_hwdef.ESP32HWDef(
+                outdir=cfg.bldnode.abspath(),
+                hwdef=[hwdef_dat],
+                quiet=True,
+            )
+            eh.run()
+            mcu = eh.mcu
+            cfg.env.MCU = mcu
+            cfg.env.append_value('DEFINES', 'HAL_ESP32_HWDEF_BUILD=1')
+        else:
+            # Fallback for legacy boards
+            mcu = 'esp32s3' if 'esp32s3' in self.name else 'esp32'
+            cfg.env.MCU = mcu
+            cfg.env.append_value('DEFINES', 'HAL_ESP32_HWDEF_BUILD=0')
+            cfg.env.append_value('DEFINES', f'HAL_BOARD_NAME_{self.name.upper()}=1')
+        
+        # Select toolchain based on MCU architecture (Xtensa vs RISC-V)
+        mcu_lower = mcu.lower()
+        if mcu_lower in ['esp32c3', 'esp32c6', 'esp32h2', 'esp32p4']:
+            self.toolchain = 'riscv32-esp-elf'
+        else:
+            self.toolchain = 'xtensa-%s-elf' % mcu_lower
+
         super(esp32, self).configure(cfg)
         if cfg.env.TOOLCHAIN:
             self.toolchain = cfg.env.TOOLCHAIN
-        else:
-            # default tool-chain for esp32-based boards:
-            self.toolchain = 'xtensa-esp32-elf'
 
     def configure_env(self, cfg, env):
         env.BOARD_CLASS = "ESP32"
@@ -1057,13 +1098,33 @@ def expand_path(p):
                          '-fdata-sections',
                          '-fno-exceptions',
                          '-fno-rtti',
-                         '-nostdlib',
                          '-fstrict-volatile-bitfields',
                          '-Wno-sign-compare',
                          '-fno-inline-functions',
                          '-mlongcalls',
-                         '-fsingle-precision-constant', # force const vals to be float , not double. so 100.0 means 100.0f 
+                         '-fsingle-precision-constant', # force const vals to be float , not double. so 100.0 means 100.0f
                          '-fno-threadsafe-statics']
+        # Detect IDF version from the IDF_PATH set during configure()
+        idf_major = 5
+        idf_version_h = os.path.join(os.environ.get('IDF_PATH', ''),
+                                     'components/esp_common/include/esp_idf_version.h')
+        if os.path.exists(idf_version_h):
+            with open(idf_version_h, 'r') as f:
+                for line in f:
+                    m = re.match(r'#define\s+ESP_IDF_VERSION_MAJOR\s+(\d+)', line)
+                    if m:
+                        idf_major = int(m.group(1))
+                        break
+        if idf_major >= 6:
+            # IDF 6.0 uses PicoLibC which is incompatible with -nostdlib.
+            # The -specs=picolibc.specs flag is needed so the compiler finds
+            # PicoLibC headers. The cmath std:: namespace fix is handled in
+            # libraries/AP_Common/missing/cmath via ESP_PLATFORM guard.
+            env.CFLAGS += ['-specs=picolibc.specs']
+            env.CXXFLAGS += ['-specs=picolibc.specs']
+        else:
+            # IDF 5.x uses Newlib which works with -nostdlib
+            env.CXXFLAGS += ['-nostdlib']
         env.CXXFLAGS.remove('-Werror=undef')
         env.CXXFLAGS.remove('-Werror=shadow')
 
@@ -1079,6 +1140,29 @@ def expand_path(p):
                 HAL_PARAM_DEFAULTS_PATH='"@ROMFS/defaults.parm"',
             )
 
+        # Use generated board headers instead of static ones
+        env.INCLUDES += [
+                cfg.bldnode.find_or_declare('boards').abspath(),
+            ]
+
+        # Include hwdef.h in compilation for ESP32 builds
+        hwdef_h = cfg.bldnode.find_or_declare('hwdef.h').abspath()
+        env.CFLAGS += ['-include', hwdef_h]
+        env.CXXFLAGS += ['-include', hwdef_h]
+
+        # Parse hwdef.dat for APA102 pins to set environment variables
+        hwdef_path = 'libraries/AP_HAL_ESP32/hwdef/%s/hwdef.dat' % self.get_name()
+        if os.path.exists(hwdef_path):
+            with open(hwdef_path, 'r') as f:
+                for line in f:
+                    line = line.strip()
+                    if line.startswith('APA102_DATA_PIN='):
+                        pin_num = line.split('=')[1]
+                        env.APA102_DATA = pin_num
+                    elif line.startswith('APA102_CLOCK_PIN='):
+                        pin_num = line.split('=')[1]
+                        env.APA102_CLOCK = pin_num
+
         env.AP_PROGRAM_AS_STLIB = True
         #if cfg.options.enable_profile:
         #    env.CXXFLAGS += ['-pg',