Fix add_pip_as_python_dependency not respected with sharded repodata

.github/workflows/tests.yml

@@ -141,6 +141,10 @@ jobs:
           condarc-file: conda\.github\condarc-${{ matrix.default-channel }}
           run-post: false  # skip post cleanup
 
+      - name: Upgrade conda-libmamba-solver (without sharded repodata)
+        shell: bash -el {0}
+        run: CONDA_PLUGINS_USE_SHARDED_REPODATA=0 conda install --yes -n base conda-canary/label/dev::conda-libmamba-solver
+
       - name: Conda Install
         working-directory: conda  # CONDA-LIBMAMBA-SOLVER CHANGE
         # CONDA-LIBMAMBA-SOLVER CHANGE: add conda-libmamba-solver requirements.txt
@@ -306,6 +310,9 @@ jobs:
           condarc-file: conda/.github/condarc-${{ matrix.default-channel }}
           run-post: false  # skip post cleanup
 
+      - name: Upgrade conda-libmamba-solver (without sharded repodata)
+        run: CONDA_PLUGINS_USE_SHARDED_REPODATA=0 conda install --yes -n base conda-canary/label/dev::conda-libmamba-solver
+
       - name: Conda Install
         working-directory: conda
         run: >
@@ -582,6 +589,9 @@ jobs:
           miniforge-version: ${{ (matrix.default-channel == 'conda-forge' && matrix.arch == 'osx-64') && 'latest' || null }}
           architecture: ${{ runner.arch }}
 
+      - name: Upgrade conda-libmamba-solver (without sharded repodata)
+        run: CONDA_PLUGINS_USE_SHARDED_REPODATA=0 conda install --yes -n base conda-canary/label/dev::conda-libmamba-solver
+
       - name: Conda Install
         working-directory: conda # CONDA-LIBMAMBA-SOLVER CHANGE
         # CONDA-LIBMAMBA-SOLVER CHANGE: add conda-libmamba-solver requirements.txt

conda_libmamba_solver/index.py

@@ -160,7 +160,11 @@ def _is_sharded_repodata_enabled():
 
 
 def _package_info_from_package_dict(
-    record: PackageRecordDict, filename: str, url: str, channel_id: str
+    record: PackageRecordDict,
+    filename: str,
+    url: str,
+    channel_id: str,
+    add_pip_as_python_dependency=False,
 ) -> PackageInfo:
     """
     Build libmamba PackageInfo from an unprocessed repodata "packages", "packages.conda" entry.
@@ -193,6 +197,16 @@ def _package_info_from_package_dict(
     if track_features and isinstance(track_features, str):
         track_features = track_features.replace(" ", ",").split(",")
         track_features = list(f for f in (ff.strip() for ff in track_features) if f)
+
+    # Build depends list and append pip if conditions are met
+    depends = list(record.get("depends") or [])
+    if (
+        add_pip_as_python_dependency
+        and record["name"] == "python"
+        and record["version"].startswith(("2.", "3."))
+    ):
+        depends.append("pip")
+
     return PackageInfo(
         name=record["name"],
         version=record["version"],
@@ -208,7 +222,7 @@ def _package_info_from_package_dict(
         signatures=record.get("signatures") or "",
         track_features=track_features,
         # conda can have list or tuple, but libmamba only accepts lists
-        depends=list(record.get("depends") or []),
+        depends=depends,
         constrains=list(record.get("constrains") or []),
         defaulted_keys=list(record.get("defaulted_keys") or []),
         noarch=noarch,
@@ -257,6 +271,7 @@ def __init__(
         self.subdirs = subdirs or context.subdirs
         self.repodata_fn = repodata_fn
         self.in_state = in_state
+        self._add_pip_as_python_dependency = context.add_pip_as_python_dependency
         self.db = self._init_db()
 
         self.repos: list[_ChannelRepoInfo] = self._load_channels()
@@ -641,10 +656,17 @@ def _load_repo_info_from_repodata_dict(
                         filename,
                         url=f"{base_url}{filename}",
                         channel_id=channel_id,
+                        add_pip_as_python_dependency=self._add_pip_as_python_dependency,
                     )
                     packages.append(package)
 
-            repo = self.db.add_repo_from_packages(packages=packages, name=channel_url)
+            repo = self.db.add_repo_from_packages(
+                packages=packages,
+                name=channel_url,
+                add_pip_as_python_dependency=PipAsPythonDependency(
+                    context.add_pip_as_python_dependency
+                ),
+            )
             repos.append(
                 _ChannelRepoInfo(
                     channel=channel_object,

conda_libmamba_solver/shards.py

@@ -138,10 +138,10 @@ def spec_to_package_name(spec: str) -> str:
     return name
 
 
-def shard_mentioned_packages(shard: ShardDict) -> Iterable[str]:
+def shard_mentioned_packages(shard: ShardDict, extra: Iterable[str] = ()) -> Iterable[str]:
     """
     Return all dependency names mentioned in a shard, not including the shard's
-    own package name.
+    own package name. Additional names can be injected via ``extra``.
     """
     unique_specs = set()
     for package in (*shard["packages"].values(), *shard["packages.conda"].values()):
@@ -152,6 +152,7 @@ def shard_mentioned_packages(shard: ShardDict) -> Iterable[str]:
             unique_specs.add(spec)
             name = spec_to_package_name(spec)
             yield name  # not much improvement from only yielding unique names
+    yield from extra
 
 
 class ShardBase(abc.ABC):

conda_libmamba_solver/shards_subset.py

@@ -189,6 +189,7 @@ def __init__(self, shardlikes: Iterable[ShardBase]):
         self.nodes = {}
         self.shardlikes = list(shardlikes)
         self._use_only_tar_bz2 = context.use_only_tar_bz2
+        self._add_pip_as_python_dependency = context.add_pip_as_python_dependency
 
     @classmethod
     def has_strategy(cls, strategy: str) -> bool:
@@ -217,7 +218,11 @@ def neighbors(self, node: Node) -> Iterator[Node]:
             shard = filter_redundant_packages(shard, self._use_only_tar_bz2)
             shardlike.visit_shard(node.package, shard)
 
-            for package in shard_mentioned_packages(shard):
+            # ensure solver has "pip" record if add_pip_as_python_dependency:
+            extra = (
+                ("pip",) if self._add_pip_as_python_dependency and node.package == "python" else ()
+            )
+            for package in shard_mentioned_packages(shard, extra=extra):
                 node_id = NodeId(package, shardlike.url)
 
                 if node_id not in self.nodes:
@@ -449,7 +454,15 @@ def log_timeout():
                 shardlike = shardlikes_by_url[node_id.channel]
                 shardlike.visit_shard(node_id.package, shard)
 
-                pending.update(self.visit_node(parent_node, shard_mentioned_packages(shard)))
+                # ensure solver has "pip" record if add_pip_as_python_dependency:
+                extra = (
+                    ("pip",)
+                    if self._add_pip_as_python_dependency and parent_node.package == "python"
+                    else ()
+                )
+                pending.update(
+                    self.visit_node(parent_node, shard_mentioned_packages(shard, extra=extra))
+                )
 
     def visit_node(self, parent_node: Node, mentioned_packages: Iterable[str]) -> Iterable[NodeId]:
         """Broadcast mentioned packages across channels. yield pending NodeId's."""

news/918-fix-add-pip-as-python-dependency-sharded

@@ -0,0 +1,20 @@
+### Enhancements
+
+* <news item>
+
+### Bug fixes
+
+* Fix `add_pip_as_python_dependency` not being honored when sharded repodata
+  is enabled. (#918 via #929)
+
+### Deprecations
+
+* <news item>
+
+### Docs
+
+* <news item>
+
+### Other
+
+* <news item>