Merge branch 'fix/kconfig_retain_local_component_list_file_during_second_cmake_run' into 'main'

fix(kconfig): transitive dependencies incorrect source on second CMake run

Closes PACMAN-1190

See merge request espressif/idf-component-manager!549

Author: kumekay

idf_component_manager/local_component_list.py

@@ -20,10 +20,16 @@ class LocalComponentList(BaseModel):
     components: t.List[LocalComponent]
 
 
-def parse_component_list(path: str) -> t.List[t.Dict[str, str]]:
-    with open(path, encoding='utf-8') as f:
-        try:
-            components = LocalComponentList.fromdict(YAML(typ='safe').load(f))
-            return [c for c in components.components]  # type: ignore
-        except (YAMLError, ValidationError):
-            raise ProcessingError('Cannot parse components list file.')
+def parse_component_list(path: str) -> t.List[LocalComponent]:
+    """
+    Parse component list from YAML file.
+    """
+
+    try:
+        with open(path, encoding='utf-8') as f:
+            data = YAML(typ='safe').load(f)
+        components = LocalComponentList.fromdict(data)
+    except (YAMLError, ValidationError):
+        raise ProcessingError('Cannot parse components list file.')
+
+    return components.components

idf_component_manager/prepare_components/prepare.py

@@ -8,8 +8,10 @@
 
 import argparse
 import os
+import shutil
 import sys
 import typing as t
+from pathlib import Path
 
 from idf_component_manager.core import ComponentManager
 from idf_component_tools import error, setup_logging, warn
@@ -23,19 +25,50 @@ def _component_list_file(build_dir):
     return os.path.join(build_dir, 'components_with_manifests_list.temp')
 
 
+def _get_ppid_file_path(local_component_list_file: t.Optional[str]) -> Path:
+    return Path(f'{local_component_list_file}.{os.getppid()}')
+
+
+def _get_component_list_file(local_components_list_file):
+    """
+    Get the appropriate component list file, preferring the PPID version
+    if it exists from a parent CMake run.
+
+    Args:
+        args: Command line arguments containing local_components_list_file
+
+    Returns:
+        Path to the component list file to use, or None if not configured
+    """
+    if not local_components_list_file:
+        return None
+
+    component_list_parent_pid = _get_ppid_file_path(local_components_list_file)
+    # Always use local component list file from the first execution of the component manager
+    # (component_list_parent_pid) during one CMake run,
+    # to be sure that it doesn't contain components introduced by the component manager.
+    if component_list_parent_pid.exists():
+        return component_list_parent_pid
+    else:
+        return local_components_list_file
+
+
 def prepare_dep_dirs(args):
     if args.sdkconfig_json_file:
         KCONFIG_CONTEXT.get().update_from_file(args.sdkconfig_json_file)
 
     build_dir = args.build_dir or os.path.dirname(args.managed_components_list_file)
+
+    local_components_list_file = _get_component_list_file(args.local_components_list_file)
+
     ComponentManager(
         args.project_dir,
         lock_path=args.lock_path,
         interface_version=args.interface_version,
     ).prepare_dep_dirs(
         managed_components_list_file=args.managed_components_list_file,
         component_list_file=_component_list_file(build_dir),
-        local_components_list_file=args.local_components_list_file,
+        local_components_list_file=local_components_list_file,
     )
 
     kconfig_ctx = KCONFIG_CONTEXT.get()
@@ -67,7 +100,26 @@ def debug_message(req: ComponentRequirement) -> str:
             f'but not found in any Kconfig file:\n'
             f'{_nl.join(sorted(debug_strs))}\n'
         )
-        exit(10)
+
+        # Copy local component list file for next run of CMake before exiting
+        if args.local_components_list_file:
+            ppid_file = _get_ppid_file_path(args.local_components_list_file)
+
+            if not Path(ppid_file).exists():
+                try:
+                    shutil.copyfile(args.local_components_list_file, ppid_file)
+                except (OSError, IOError) as e:
+                    raise FatalError(
+                        f"Failed to copy '{args.local_components_list_file}' → '{ppid_file}': {e}"
+                    ) from e
+        # Exiting with code 10 to signal CMake to re-run component discovery due to missing KConfig options
+        sys.exit(10)
+
+    # Clean up PPID file on successful completion
+    if args.local_components_list_file:
+        ppid_file_path = Path(_get_ppid_file_path(args.local_components_list_file))
+        if ppid_file_path.exists():
+            ppid_file_path.unlink()
 
 
 def inject_requirements(args):

integration_tests/test_integration.py

@@ -119,15 +119,15 @@ def test_set_component_version(project):
                 'main': {
                     'dependencies': {
                         'idf': {
-                            'version': '^6.1',
+                            'version': '^4.9',
                         }
                     }
                 },
                 'component_foo': {
                     'version': '1.0.0',
                     'dependencies': {
                         'idf': {
-                            'version': '^6.1',
+                            'version': '^4.9',
                         }
                     },
                 },
@@ -138,7 +138,7 @@ def test_set_component_version(project):
 )
 def test_root_dep_failed(project):
     res = project_action(project, 'reconfigure')
-    assert "Result: Because project depends on idf (^6.1) which doesn't match any" in res
+    assert "Result: Because project depends on idf (^4.9) which doesn't match any" in res
     assert 'versions, version solving failed.' in res
     assert 'Please check manifest file of the following component(s): main,' in res
     assert 'component_foo' in res

integration_tests/test_prepare_dep_dirs_kconfig.py

@@ -0,0 +1,55 @@
+# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import os
+from pathlib import Path
+
+import pytest
+from ruamel.yaml import YAML
+
+from idf_component_tools.semver.base import Version
+from integration_tests.integration_test_helpers import fixtures_path, project_action
+
+idf_version = Version.coerce(os.getenv('ESP_IDF_VERSION'))
+
+
+@pytest.mark.skipif(
+    idf_version < Version.coerce('5.3'),
+    reason='KConfig variables in the manifest are not supported in ESP-IDF < 5.3',
+)
+@pytest.mark.parametrize(
+    'project',
+    [
+        {
+            'components': {
+                'main': {
+                    'dependencies': {
+                        'cmp': {
+                            'override_path': fixtures_path(
+                                'components', 'cmp_with_kconfig_var', 'cmp'
+                            ),
+                        },
+                    },
+                },
+            },
+        },
+    ],
+    indirect=True,
+)
+def test_prepare_dep_dirs_with_kconfig(project):
+    # Create a mock sdkconfig file
+    (Path(project) / 'sdkconfig.default').write_text('CONFIG_ESP_BOARD_DEV_AUDIO_CODEC_SUPPORT=y')
+
+    project_action(
+        project,
+        'reconfigure',
+    )
+
+    # Verify that valid Kconfig options are resolved correctly
+    lock = YAML().load(Path(project) / 'dependencies.lock')
+    assert 'cmp' in lock['dependencies']
+    assert 'espressif/esp_codec_dev' in lock['dependencies']
+    assert (
+        '$CONFIG{ESP_BOARD_DEV_AUDIO_CODEC_SUPPORT} == True'
+        in lock['dependencies']['cmp']['dependencies'][0]['matches'][0]['if']
+    )
+    assert 'service' in lock['dependencies']['espressif/esp_codec_dev']['source']['type']

tests/fixtures/components/cmp_with_kconfig_var/cmp/Kconfig.in

@@ -0,0 +1,11 @@
+menu "Device Support"
+
+config ESP_BOARD_DEV_AUDIO_CODEC_SUPPORT
+    bool "Device 'audio_codec' support"
+    default y
+    help
+        Enable audio_codec device support.
+        This option enables the audio_codec device driver.
+        The driver is located at: devices/dev_audio_codec
+
+endmenu

tests/fixtures/components/cmp_with_kconfig_var/cmp/Kconfig.projbuild

@@ -0,0 +1,15 @@
+menu "ESP Board Manager Configuration"
+
+    menu "Board Manager Setting"
+        config ESP_BOARD_MANAGER_AUTO_CONFIG_DEVICE_AND_PERIPHERAL
+            bool "Enable automatic configure device and peripheral in sdkconfig"
+            default y
+            help
+                When enabled, the board manager will automatically configure corresponding
+                device and peripheral types in user's sdkconfig based on detected
+                device and peripheral information from YAML configuration files.
+    endmenu
+
+    orsource "./Kconfig.in"
+
+endmenu

tests/fixtures/components/cmp_with_kconfig_var/cmp/LICENSE

@@ -0,0 +1 @@
+license

tests/fixtures/components/cmp_with_kconfig_var/cmp/cmp.c

@@ -0,0 +1,11 @@
+/*
+ * SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <stdio.h>
+
+void cmp_hello()
+{
+        printf("Hello from CMP version 1.0.0\n");
+}

tests/fixtures/components/cmp_with_kconfig_var/cmp/idf_component.yml

@@ -0,0 +1,11 @@
+version: '1.0.0'
+targets:
+  - esp32
+maintainers:
+  - 'Test Tester <[email protected]>'
+dependencies:
+  espressif/esp_codec_dev:
+    matches:
+      - if: $CONFIG{ESP_BOARD_DEV_AUDIO_CODEC_SUPPORT} == True
+    require: public
+    version: 1.5.0