feat: support reuse local existing versions while version solving

Author: hfudev

idf_component_manager/dependencies.py

@@ -1,6 +1,5 @@
-# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 import os
 import shutil
 from functools import total_ordering
@@ -12,7 +11,7 @@
 from idf_component_manager.version_solver.mixology.failure import SolverFailure
 from idf_component_manager.version_solver.mixology.package import Package
 from idf_component_manager.version_solver.version_solver import VersionSolver
-from idf_component_tools.build_system_tools import build_name
+from idf_component_tools.build_system_tools import build_name, get_idf_version
 from idf_component_tools.environment import getenv_bool
 from idf_component_tools.errors import (
     ComponentModifiedError,
@@ -25,11 +24,13 @@
     validate_managed_component_hash,
 )
 from idf_component_tools.lock import LockManager
-from idf_component_tools.manifest import ProjectRequirements
+from idf_component_tools.manifest import ComponentVersion, ProjectRequirements
 from idf_component_tools.manifest.solved_component import SolvedComponent
 from idf_component_tools.manifest.solved_manifest import SolvedManifest
 from idf_component_tools.messages import hint, warn
 from idf_component_tools.registry.api_client_errors import NetworkConnectionError
+from idf_component_tools.semver import Version
+from idf_component_tools.sources import IDFSource
 from idf_component_tools.sources.fetcher import ComponentFetcher
 
 
@@ -104,6 +105,14 @@ def is_solve_required(project_requirements, solution):
         print_info('Manifest files have changed, solving dependencies.')
         return True
 
+    if solution.idf_version and solution.idf_version != Version(get_idf_version()):
+        print_info(
+            'IDF version changed from {} to {}, solving dependencies.'.format(
+                solution.idf_version, get_idf_version()
+            )
+        )
+        return True
+
     if solution.target and project_requirements.target != solution.target:
         print_info(
             'Target changed from {} to {}, solving dependencies.'.format(
@@ -130,14 +139,14 @@ def is_solve_required(project_requirements, solution):
                 )
             except FetchingError:
                 print_warn(
-                    'Version {} of dependency {} not found, probably it was deleted, solving dependencies.'.format(
-                        component.version, component.name
-                    )
+                    'Version {} of dependency {} not found, probably it was deleted, '
+                    'solving dependencies.'.format(component.version, component.name)
                 )
                 return True
             except NetworkConnectionError:
                 hint(
-                    'Cannot establish a connection to the component registry. Skipping checks of dependency changes.'
+                    'Cannot establish a connection to the component registry. '
+                    'Skipping checks of dependency changes.'
                 )
                 return False
 
@@ -222,7 +231,7 @@ def abs_posix_path(self):  # type: () -> str
 def check_for_new_component_versions(project_requirements, old_solution):
     if getenv_bool('IDF_COMPONENT_CHECK_NEW_VERSION', False):
         # Check for newer versions of components
-        solver = VersionSolver(project_requirements, old_solution)
+        solver = VersionSolver(project_requirements)
         try:
             new_solution = solver.solve()
             new_deps_names = [dep.name for dep in new_solution.dependencies]
@@ -246,9 +255,8 @@ def check_for_new_component_versions(project_requirements, old_solution):
             if updateable_components_messages:
                 hint(
                     '\nFollowing dependencies have new versions available:\n{}'
-                    '\nConsider running "idf.py update-dependencies" to update your lock file.\n'.format(
-                        '\n'.join(updateable_components_messages)
-                    )
+                    '\nConsider running "idf.py update-dependencies" '
+                    'to update your lock file.\n'.format('\n'.join(updateable_components_messages))
                 )
 
         except (SolverFailure, NetworkConnectionError):
@@ -262,10 +270,18 @@ def download_project_dependencies(
     """Solves dependencies and download components"""
     lock_manager = LockManager(lock_path)
     solution = lock_manager.load()
+
     check_manifests_targets(project_requirements)
 
     if is_solve_required(project_requirements, solution):
-        solver = VersionSolver(project_requirements, solution, component_solved_callback=print_dot)
+        # replace the old solution with the current idf
+        for dep in solution.dependencies:
+            if dep.name == IDFSource().NAME:
+                dep.version = ComponentVersion(get_idf_version())
+
+        solver = VersionSolver(
+            project_requirements, old_solution=solution, component_solved_callback=print_dot
+        )
 
         try:
             solution = solver.solve()
@@ -274,11 +290,11 @@ def download_project_dependencies(
             components_introduce_conflict = []
             for conflict_constraint in conflict_constraints:
                 for manifest in project_requirements.manifests:
-                    for dep in manifest.dependencies:
-                        for source in dep.sources:
+                    for req in manifest.dependencies:
+                        for source in req.sources:
                             if Package(
-                                dep.name, source
-                            ) == conflict_constraint.package and dep.version_spec == str(
+                                req.name, source
+                            ) == conflict_constraint.package and req.version_spec == str(
                                 conflict_constraint.constraint
                             ):
                                 components_introduce_conflict.append(manifest.name)

idf_component_manager/version_solver/version_solver.py

@@ -1,9 +1,10 @@
 # SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
+import logging
 import os
 
 from idf_component_tools.errors import DependencySolveError, FetchingError, SolverError
+from idf_component_tools.lock.manager import EMPTY_LOCK
 from idf_component_tools.manifest import (
     ComponentRequirement,
     ComponentWithVersions,
@@ -17,6 +18,7 @@
 
 from ..utils import print_info, print_warn
 from .helper import PackageSource
+from .mixology.failure import SolverFailure
 from .mixology.package import Package
 from .mixology.version_solver import VersionSolver as Solver
 
@@ -25,6 +27,8 @@
 except ImportError:
     pass
 
+logger = logging.getLogger(__name__)
+
 
 class VersionSolver(object):
     """
@@ -38,18 +42,21 @@ def __init__(self, requirements, old_solution=None, component_solved_callback=No
         self.old_solution = old_solution
         self.component_solved_callback = component_solved_callback
 
+        self._init()
+
+    def _init(self):
+        # put all the intermediate generated attrs here
+        # to reset them when the solver is re-used
         self._source = PackageSource()
         self._solver = Solver(self._source)
         self._target = None
         self._overriders = set()  # type: set[str]
         self._local_root_requirements = dict()  # type: dict[str, ComponentRequirement]
-        self._solved_requirements = set()  # type: set[ComponentRequirement]
+        self._parse_local_root_requirements()
 
-    def solve(self):  # type: () -> SolvedManifest
-        # scan all root local requirements
-        # root local requirements defined in the file system manifest files
-        # would have higher priorities
+        self._solved_requirements = set()  # type: set[ComponentRequirement]
 
+    def _parse_local_root_requirements(self):  # type: () -> None
         # scan all root local requirements
         for manifest in self.requirements.manifests:
             for requirement in manifest.dependencies:  # type: ComponentRequirement
@@ -68,7 +75,7 @@ def solve(self):  # type: () -> SolvedManifest
                     else:
                         self._local_root_requirements[requirement.build_name] = requirement
 
-        # scan all root local components
+        # add all local components, except [0] -> main component
         for manifest in self.requirements.manifests[1:]:
             # add itself as highest priority component
             if manifest.name and manifest.version and manifest._manifest_manager:
@@ -87,8 +94,17 @@ def solve(self):  # type: () -> SolvedManifest
                     version_spec=str(manifest.version),
                 )
 
+    def _solve(self, cur_solution=None):  # type: (SolvedManifest | None) -> SolvedManifest
+        """
+        Solve the version requirements and return the result.
+
+        :param cur_solution: The current solution to be used as a starting point.
+        :raises SolverError: If the solver fails to solve the requirements.
+        """
+        # root local requirements defined in the file system manifest files
+        # would have higher priorities
         for manifest in self.requirements.manifests:
-            self.solve_manifest(manifest)
+            self.solve_manifest(manifest, cur_solution=cur_solution)
 
         self._source.override_dependencies(self._overriders)
 
@@ -108,18 +124,42 @@ def solve(self):  # type: () -> SolvedManifest
             solved_components, self.requirements.manifest_hash, self.requirements.target
         )
 
+    def solve(self):  # type: () -> SolvedManifest
+        if self.old_solution != SolvedManifest.fromdict(EMPTY_LOCK):
+            try:
+                return self._solve(self.old_solution)
+            except SolverFailure as e:
+                logger.debug(
+                    'Solver failed to solve the requirements with the current solution. '
+                    'Error: %s.\n'
+                    'Retrying without the current solution. ',
+                    e,
+                )
+
+        self._init()
+        return self._solve()
+
     def get_versions_from_sources(
-        self, requirement
-    ):  # type: (ComponentRequirement) -> tuple[ComponentWithVersions | None, BaseSource | None]
+        self,
+        requirement,  # type: ComponentRequirement
+        cur_solution=None,  # type: SolvedManifest | None
+    ):  # type: (...) -> tuple[ComponentWithVersions | None, BaseSource | None]
         latest_source = None
         cmp_with_versions = None
         for source in requirement.sources:
             try:
-                cmp_with_versions = source.versions(
-                    name=requirement.name,
-                    spec=requirement.version_spec,
-                    target=self.requirements.target,
-                )
+                if cur_solution and requirement.name in cur_solution.solved_components:
+                    cmp_with_versions = requirement.source.versions(
+                        name=requirement.name,
+                        spec=str(cur_solution.solved_components[requirement.name].version),
+                        target=self.requirements.target,
+                    )
+                else:
+                    cmp_with_versions = source.versions(
+                        name=requirement.name,
+                        spec=requirement.version_spec,
+                        target=self.requirements.target,
+                    )
                 latest_source = source
                 if cmp_with_versions.versions:
                     break
@@ -129,18 +169,22 @@ def get_versions_from_sources(
                 pass
         return cmp_with_versions, latest_source
 
-    def solve_manifest(self, manifest):  # type: (Manifest) -> None
+    def solve_manifest(
+        self,
+        manifest,  # type: Manifest
+        cur_solution=None,  # type: SolvedManifest | None
+    ):  # type: (...) -> None
         for dep in self._dependencies_with_local_precedence(
             manifest.dependencies, manifest_path=manifest.path
         ):
             if len(dep.sources) == 1:
                 source = dep.source
             else:
-                _, source = self.get_versions_from_sources(dep)
+                _, source = self.get_versions_from_sources(dep, cur_solution=cur_solution)
 
             self._source.root_dep(Package(dep.name, source), dep.version_spec)
             try:
-                self.solve_component(dep, manifest_path=manifest.path)
+                self.solve_component(dep, manifest_path=manifest.path, cur_solution=cur_solution)
             except DependencySolveError as e:
                 raise SolverError(
                     'Solver failed processing dependency "{dependency}" '
@@ -155,12 +199,14 @@ def solve_manifest(self, manifest):  # type: (Manifest) -> None
                 )
 
     def solve_component(
-        self, requirement, manifest_path=None
-    ):  # type: (ComponentRequirement, str | None) -> None
+        self, requirement, manifest_path=None, cur_solution=None
+    ):  # type: (ComponentRequirement, str | None, SolvedManifest | None) -> None
         if requirement in self._solved_requirements:
             return
 
-        cmp_with_versions, source = self.get_versions_from_sources(requirement)
+        cmp_with_versions, source = self.get_versions_from_sources(
+            requirement, cur_solution=cur_solution
+        )
 
         if not cmp_with_versions or not cmp_with_versions.versions or not source:
             print_warn('Component "{}" not found'.format(requirement.name))

idf_component_tools/lock/manager.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 import os
@@ -18,6 +18,7 @@
 from ..manifest import ComponentVersion, known_targets
 from ..manifest.solved_component import SolvedComponent
 from ..manifest.solved_manifest import SolvedManifest
+from ..messages import notice
 from ..sources import IDFSource
 
 FORMAT_VERSION = '1.0.0'
@@ -36,11 +37,13 @@
                 'source': Or(*[source.schema() for source in tools.sources.KNOWN_SOURCES]),
                 'version': Or(*string_types),
                 Optional('component_hash'): HASH_SCHEMA,
+                Optional(str): object,  # optional fields for future extensions
             }
         },
         'manifest_hash': HASH_SCHEMA,
         'version': And(Or(*string_types), len),
         Optional('target'): And(Use(str.lower), lambda s: s in known_targets()),
+        Optional(str): object,  # optional fields for future extensions
     }
 )
 
@@ -104,10 +107,15 @@ def load(self):  # type: () -> SolvedManifest
 
                 version = lock.pop('version')
                 if version != FORMAT_VERSION:
-                    raise LockError(
-                        'Cannot parse components lock file.'
-                        'Lock file format version is %s, while only %s is supported'
-                        % (version, FORMAT_VERSION)
+                    notice(
+                        'Current idf-component-manager default lock file version is '
+                        '{}, '
+                        'but found {} in {}. '
+                        'Recreating lock file with the current version.'.format(
+                            FORMAT_VERSION,
+                            version,
+                            self._path,
+                        )
                     )
 
                 return SolvedManifest.fromdict(lock)

idf_component_tools/manifest/solved_manifest.py

@@ -1,6 +1,7 @@
-# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
+from ..semver import Version
 from .solved_component import SolvedComponent
 
 try:
@@ -53,3 +54,10 @@ def serialize(self):
     @property
     def solved_components(self):  # type: () -> dict[str, SolvedComponent]
         return {cmp.name: cmp for cmp in self.dependencies}
+
+    @property
+    def idf_version(self):  # type: () -> Version | None
+        if 'idf' in self.solved_components:
+            return self.solved_components['idf'].version.semver
+
+        return None

idf_component_tools/sources/web_service_keys.py

@@ -1,5 +1,10 @@
-# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 WEB_SERVICE_REQUIRED_KEYS = {}  # type: dict[str, str]
-WEB_SERVICE_OPTIONAL_KEYS = {'pre_release': 'bool', 'storage_url': 'str', 'service_url': 'str'}
+WEB_SERVICE_OPTIONAL_KEYS = {
+    'pre_release': 'bool',
+    'storage_url': 'str',
+    'service_url': 'str',
+    'registry_url': 'str',
+}