Merge branch 'fix/kconfig_build_system' into 'main'

fix: Kconfig build system

Closes PACMAN-1204

See merge request espressif/idf-component-manager!563

Author: XDanielPaul

idf_component_manager/core.py

@@ -805,6 +805,7 @@ def inject_requirements(
         self,
         component_requires_file: t.Union[Path, str],
         component_list_file: t.Union[Path, str],
+        cm_run_counter: int,
     ):
         """Set build dependencies for components with manifests"""
         requirements_manager = CMakeRequirementsManager(component_requires_file)
@@ -884,6 +885,14 @@ def add_req(key: str) -> None:
         handle_project_requirements(new_requirements)
         requirements_manager.dump(new_requirements)
 
+        # In case of the CM second run when dealing with KConfig Variables
+        # We need to overwrite the retried to 0 to run CM the third time
+        # It's needed due to reason that in sdkconfig.json there will be
+        # correct values for configs of downloaded deps
+        if cm_run_counter == 1:
+            with open(requirements_manager.path, 'a', encoding='utf-8') as f:
+                f.write('set(retried 0 PARENT_SCOPE)\n')
+
     @staticmethod
     def _override_requirements_by_component_sources(
         requirements: t.OrderedDict[ComponentName, t.Dict[str, t.Union[t.List[str], str]]],

idf_component_manager/prepare_components/prepare.py

@@ -25,6 +25,43 @@ def _component_list_file(build_dir):
     return os.path.join(build_dir, 'components_with_manifests_list.temp')
 
 
+class RunCounter:
+    def __init__(self, build_dir: t.Union[str, Path]):
+        self._file_path = Path(build_dir) / f'component_manager_run_counter.{os.getppid()}'
+
+        if not self._file_path.exists():
+            self._file_path.write_text('0')
+
+    @property
+    def value(self) -> int:
+        """
+        Current value of the counter.
+        """
+        try:
+            return int(self._file_path.read_text().strip())
+        except (FileNotFoundError, ValueError):
+            # Fallback if file was deleted externally or contains garbage
+            return 0
+
+    def increase(self) -> None:
+        """
+        Increments the counter by 1.
+        """
+        # Since __init__ guarantees creation, we can simply update logic.
+        # We still catch FileNotFoundError in case it was deleted externally.
+        if not self._file_path.exists():
+            return
+
+        self._file_path.write_text(str(self.value + 1))
+
+    def cleanup(self) -> None:
+        """
+        Removes the counter file.
+        """
+        if self._file_path.exists():
+            self._file_path.unlink()
+
+
 def _get_ppid_file_path(local_component_list_file: t.Optional[str]) -> Path:
     return Path(f'{local_component_list_file}.{os.getppid()}')
 
@@ -53,28 +90,28 @@ def _get_component_list_file(local_components_list_file):
         return local_components_list_file
 
 
-def _get_sdkconfig_json_file_path(args) -> t.Optional[Path]:
+def _get_sdkconfig_json_file_path(args, build_dir) -> t.Optional[Path]:
     """
     Returns the path to the sdkconfig.json file if found, None otherwise.
     `sdkconfig_json_file` argument is not provided in some ESP-IDF versions (5.5.0, 5.5.1,...) when injecting deps
     so there's a fallback to the default known location.
     """
     if args.sdkconfig_json_file:
         return Path(args.sdkconfig_json_file)
-    elif args.interface_version >= 4 and args.build_dir:
-        return Path(args.build_dir) / 'config' / 'sdkconfig.json'
+    elif args.interface_version >= 4 and build_dir:
+        return Path(build_dir) / 'config' / 'sdkconfig.json'
 
     return None
 
 
 def prepare_dep_dirs(args):
-    sdk_config_json_path = _get_sdkconfig_json_file_path(args)
+    build_dir = args.build_dir or os.path.dirname(args.managed_components_list_file)
 
-    if sdk_config_json_path:
+    # If the Component Manager has been run before, we need to update the Kconfig context with the sdkconfig.json file
+    sdk_config_json_path = _get_sdkconfig_json_file_path(args, build_dir)
+    if sdk_config_json_path and RunCounter(build_dir).value > 0:
         KCONFIG_CONTEXT.get().update_from_file(sdk_config_json_path)
 
-    build_dir = args.build_dir or os.path.dirname(args.managed_components_list_file)
-
     local_components_list_file = _get_component_list_file(args.local_components_list_file)
 
     ComponentManager(
@@ -128,6 +165,7 @@ def debug_message(req: ComponentRequirement) -> str:
                     raise FatalError(
                         f"Failed to copy '{args.local_components_list_file}' → '{ppid_file}': {e}"
                     ) from e
+
         # Exiting with code 10 to signal CMake to re-run component discovery due to missing KConfig options
         sys.exit(10)
 
@@ -139,9 +177,9 @@ def debug_message(req: ComponentRequirement) -> str:
 
 
 def inject_requirements(args):
-    sdk_config_json_path = _get_sdkconfig_json_file_path(args)
+    sdk_config_json_path = _get_sdkconfig_json_file_path(args, args.build_dir)
 
-    if sdk_config_json_path:
+    if sdk_config_json_path and RunCounter(args.build_dir).value > 0:
         KCONFIG_CONTEXT.get().update_from_file(sdk_config_json_path)
 
     ComponentManager(
@@ -151,8 +189,16 @@ def inject_requirements(args):
     ).inject_requirements(
         component_requires_file=args.component_requires_file,
         component_list_file=_component_list_file(args.build_dir),
+        cm_run_counter=RunCounter(args.build_dir).value,
     )
 
+    # Last run of prepare_dep_dirs was successful
+    # Clean up CM Run counter
+    if not Path(_get_ppid_file_path(f'{args.build_dir}/local_components_list.temp.yml')).exists():
+        RunCounter(args.build_dir).cleanup()
+    else:
+        RunCounter(args.build_dir).increase()
+
 
 def main():
     setup_logging()

integration_tests/test_kconfig.py

@@ -0,0 +1,100 @@
+# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import os
+from pathlib import Path
+
+import pytest
+from ruamel.yaml import YAML
+
+from idf_component_tools.semver.base import Version
+from integration_tests.integration_test_helpers import fixtures_path, project_action
+
+idf_version = Version.coerce(os.getenv('ESP_IDF_VERSION'))
+
+
+@pytest.mark.skipif(
+    idf_version < Version.coerce('5.3'),
+    reason='KConfig variables in the manifest are not supported in ESP-IDF < 5.3',
+)
+@pytest.mark.parametrize(
+    'project',
+    [
+        {
+            'components': {
+                'main': {
+                    'dependencies': {
+                        'cmp': {
+                            'override_path': fixtures_path(
+                                'components', 'cmp_with_kconfig_var', 'cmp'
+                            ),
+                        },
+                    },
+                },
+            },
+        },
+    ],
+    indirect=True,
+)
+def test_prepare_dep_dirs_with_kconfig(project):
+    res = project_action(
+        project,
+        'reconfigure',
+    )
+
+    # Count how many times "Processing X dependencies" appears in the output
+    # This indicates how many times Component Manager has been run
+    processing_count = res.count('NOTICE: Processing')
+    assert processing_count == 2
+
+    # Verify that valid Kconfig options are resolved correctly
+    lock = YAML().load(Path(project) / 'dependencies.lock')
+    assert 'cmp' in lock['dependencies']
+    assert 'espressif/esp_codec_dev' in lock['dependencies']
+    assert (
+        '$CONFIG{ESP_BOARD_DEV_AUDIO_CODEC_SUPPORT} == True'
+        in lock['dependencies']['cmp']['dependencies'][0]['matches'][0]['if']
+    )
+    assert 'service' in lock['dependencies']['espressif/esp_codec_dev']['source']['type']
+
+
+@pytest.mark.skipif(
+    idf_version < Version.coerce('5.3'),
+    reason='KConfig variables in the manifest are not supported in ESP-IDF < 5.3',
+)
+@pytest.mark.parametrize(
+    'project',
+    [
+        {
+            'components': {
+                'main': {
+                    'dependencies': {
+                        'cmp': {
+                            'matches': [{'if': '$CONFIG{ADC_ENABLE_DEBUG_LOG} == True'}],
+                            'override_path': fixtures_path(
+                                'components', 'cmp_with_kconfig_var', 'cmp'
+                            ),
+                        },
+                    },
+                },
+            },
+        },
+    ],
+    indirect=True,
+)
+def test_three_runs_cm_kconfig(project):
+    (Path(project) / 'sdkconfig').write_text('CONFIG_ADC_ENABLE_DEBUG_LOG=y')
+
+    res = project_action(
+        project,
+        'reconfigure',
+    )
+
+    # Count how many times "Processing X dependencies" appears in the output
+    # This indicates how many times Component Manager has been run
+    processing_count = res.count('NOTICE: Processing')
+    assert processing_count == 3
+
+    assert 'Configuring done' in res
+    lock = YAML().load(Path(project) / 'dependencies.lock')
+    assert 'cmp' in lock['dependencies']
+    assert 'espressif/esp_codec_dev' in lock['dependencies']

tests/test_component_manager_run_counter.py

@@ -0,0 +1,135 @@
+# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import os
+
+from idf_component_manager.prepare_components.prepare import RunCounter
+
+
+def test_create_counter_file_with_zero(tmp_path):
+    counter = RunCounter(tmp_path)
+    assert counter.value == 0
+    # Verify the file was created
+    assert counter._file_path.exists()
+
+
+def test_file_path_includes_ppid(tmp_path):
+    counter = RunCounter(tmp_path)
+    expected_filename = f'component_manager_run_counter.{os.getppid()}'
+    assert counter._file_path.name == expected_filename
+
+
+def test_value_handles_missing_file(tmp_path):
+    counter = RunCounter(tmp_path)
+    assert counter.value == 0
+
+
+def test_increase_increments_counter_by_one(tmp_path):
+    counter = RunCounter(tmp_path)
+    assert counter.value == 0
+    counter.increase()
+    assert counter.value == 1
+
+
+def test_increase_multiple_times(tmp_path):
+    counter = RunCounter(tmp_path)
+    counter.increase()
+    counter.increase()
+    counter.increase()
+    assert counter.value == 3
+
+
+def test_increase_does_nothing_if_file_deleted(tmp_path):
+    counter = RunCounter(tmp_path)
+    counter.cleanup()
+    # Should not raise an exception
+    counter.increase()
+    assert counter.value == 0  # Fallback
+
+
+def test_cleanup_removes_counter_file(tmp_path):
+    counter = RunCounter(tmp_path)
+    assert counter._file_path.exists()
+    counter.cleanup()
+    assert not counter._file_path.exists()
+
+
+def test_cleanup_value_after_cleanup(tmp_path):
+    counter = RunCounter(tmp_path)
+    counter.increase()
+    assert counter.value == 1
+    counter.cleanup()
+    assert counter.value == 0
+
+
+def test_cleanup_called_multiple_times(tmp_path):
+    counter = RunCounter(tmp_path)
+    counter.cleanup()
+    counter.cleanup()  # Should not raise
+    assert not counter._file_path.exists()
+
+
+def test_single_run_typical_cmake_workflow(tmp_path):
+    # First CMake run: prepare_dependencies
+    assert not RunCounter(tmp_path).value > 0
+
+    # First CMake run: inject_requirements
+    assert not RunCounter(tmp_path).value > 0
+    RunCounter(tmp_path).increase()
+
+    # Cleanup after successful build
+    cnt = RunCounter(tmp_path)
+    cnt.cleanup()
+
+    assert not cnt._file_path.exists()
+
+
+def test_two_run_typical_cmake_workflow(tmp_path):
+    # First CMake run: prepare_dependencies
+    assert not RunCounter(tmp_path).value > 0
+
+    # First CMake run: inject_requirements
+    assert not RunCounter(tmp_path).value > 0
+    RunCounter(tmp_path).increase()
+
+    # Second CMake run: prepare_dependencies checks if run before
+    assert RunCounter(tmp_path).value > 0  # Was run before
+
+    # Second CMake run: inject_requirements
+    assert RunCounter(tmp_path).value > 0
+    RunCounter(tmp_path).increase()
+
+    # Cleanup after successful build
+    cnt = RunCounter(tmp_path)
+    cnt.cleanup()
+
+    assert not cnt._file_path.exists()
+
+
+def test_three_run_typical_cmake_workflow(tmp_path):
+    # First CMake run: prepare_dependencies
+    assert not RunCounter(tmp_path).value > 0
+
+    # First CMake run: inject_requirements
+    assert not RunCounter(tmp_path).value > 0
+    RunCounter(tmp_path).increase()
+
+    # Second CMake run: prepare_dependencies checks if run before
+    assert RunCounter(tmp_path).value > 0  # Was run before
+
+    # Second CMake run: inject_requirements
+    assert RunCounter(tmp_path).value > 0
+
+    RunCounter(tmp_path).increase()
+
+    # Third CMake run: prepare_dependencies checks if run before
+    assert RunCounter(tmp_path).value > 0  # Was run before
+
+    # Third CMake run: inject_requirements
+    assert RunCounter(tmp_path).value > 0
+    RunCounter(tmp_path).increase()
+
+    # Cleanup after successful build
+    cnt = RunCounter(tmp_path)
+    cnt.cleanup()
+
+    assert not cnt._file_path.exists()