Add targets for lockfiles.

[ci skip-rust]

[ci skip-build-wheels]

Author: kaos

src/python/pants/backend/python/goals/lockfile.py

@@ -3,9 +3,11 @@
 
 from __future__ import annotations
 
-import logging
+import itertools
+import os.path
 from collections import defaultdict
 from dataclasses import dataclass
+from operator import itemgetter
 from typing import Iterable
 
 from pants.backend.python.pip_requirement import PipRequirement
@@ -35,6 +37,8 @@
 )
 from pants.core.util_rules.lockfile_metadata import calculate_invalidation_digest
 from pants.engine.fs import CreateDigest, Digest, DigestContents, FileContent, MergeDigests
+from pants.engine.internals.synthetic_targets import SyntheticAddressMaps, SyntheticTargetsRequest
+from pants.engine.internals.target_adaptor import TargetAdaptor
 from pants.engine.process import ProcessCacheScope, ProcessResult
 from pants.engine.rules import Get, collect_rules, rule, rule_helper
 from pants.engine.target import AllTargets
@@ -43,8 +47,6 @@
 from pants.util.logging import LogLevel
 from pants.util.ordered_set import FrozenOrderedSet
 
-logger = logging.getLogger(__name__)
-
 
 @dataclass(frozen=True)
 class GeneratePythonLockfile(GenerateLockfile):
@@ -268,10 +270,43 @@ async def setup_user_lockfile_requests(
     )
 
 
+@dataclass(frozen=True)
+class PythonSyntheticLockfileTargetsRequest(SyntheticTargetsRequest):
+    path: str = ""  # Indicate that all targets are provided with a single request.
+
+
+@rule
+async def python_lockfile_synthetic_targets(
+    request: PythonSyntheticLockfileTargetsRequest,
+    python_setup: PythonSetup,
+) -> SyntheticAddressMaps:
+    if not python_setup.enable_resolves:
+        return SyntheticAddressMaps()
+
+    resolves = [
+        (os.path.dirname(lockfile), os.path.basename(lockfile), name)
+        for name, lockfile in python_setup.resolves.items()
+    ]
+    return SyntheticAddressMaps.for_targets_request(
+        request,
+        [
+            (
+                os.path.join(spec_path, "BUILD.python-lockfiles"),
+                tuple(
+                    TargetAdaptor("lockfiles", name=name, sources=[lockfile])
+                    for _, lockfile, name in lockfiles
+                ),
+            )
+            for spec_path, lockfiles in itertools.groupby(sorted(resolves), key=itemgetter(0))
+        ],
+    )
+
+
 def rules():
     return (
         *collect_rules(),
         UnionRule(GenerateLockfile, GeneratePythonLockfile),
         UnionRule(KnownUserResolveNamesRequest, KnownPythonUserResolveNamesRequest),
         UnionRule(RequestedUserResolveNames, RequestedPythonUserResolveNames),
+        UnionRule(SyntheticTargetsRequest, PythonSyntheticLockfileTargetsRequest),
     )

src/python/pants/backend/python/macros/python_requirements.py

@@ -14,6 +14,7 @@
     TypeStubsModuleMappingField,
 )
 from pants.backend.python.pip_requirement import PipRequirement
+from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import (
     PythonRequirementModulesField,
     PythonRequirementResolveField,
@@ -82,7 +83,9 @@ class GenerateFromPythonRequirementsRequest(GenerateTargetsRequest):
 
 @rule(desc="Generate `python_requirement` targets from requirements.txt", level=LogLevel.DEBUG)
 async def generate_from_python_requirement(
-    request: GenerateFromPythonRequirementsRequest, union_membership: UnionMembership
+    request: GenerateFromPythonRequirementsRequest,
+    union_membership: UnionMembership,
+    python_setup: PythonSetup,
 ) -> GeneratedTargets:
     generator = request.generator
     requirements_rel_path = generator[PythonRequirementsSourceField].value
@@ -102,6 +105,15 @@ async def generate_from_python_requirement(
         union_membership,
     )
 
+    req_deps = [file_tgt.address.spec]
+
+    resolve = request.template.get(
+        PythonRequirementResolveField.alias, python_setup.default_resolve
+    )
+    lockfile = python_setup.resolves.get(resolve) if python_setup.enable_resolves else None
+    if lockfile:
+        req_deps.append(f"{lockfile}:{resolve}")
+
     digest_contents = await Get(
         DigestContents,
         PathGlobs(
@@ -126,9 +138,7 @@ def generate_tgt(
         normalized_proj_name = canonicalize_project_name(project_name)
         tgt_overrides = overrides.pop(normalized_proj_name, {})
         if Dependencies.alias in tgt_overrides:
-            tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + [
-                file_tgt.address.spec
-            ]
+            tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + req_deps
 
         return PythonRequirementTarget(
             {
@@ -140,7 +150,7 @@ def generate_tgt(
                 ),
                 # This may get overridden by `tgt_overrides`, which will have already added in
                 # the file tgt.
-                Dependencies.alias: [file_tgt.address.spec],
+                Dependencies.alias: req_deps,
                 **tgt_overrides,
             },
             request.template_address.create_generated(project_name),

src/python/pants/core/register.py

@@ -29,6 +29,8 @@
     FileTarget,
     GenericTarget,
     HTTPSource,
+    LockfilesGeneratorTarget,
+    LockfileTarget,
     RelocatedFiles,
     ResourcesGeneratorTarget,
     ResourceTarget,
@@ -91,15 +93,17 @@ def rules():
 def target_types():
     return [
         ArchiveTarget,
-        FileTarget,
+        DockerEnvironmentTarget,
         FilesGeneratorTarget,
+        FileTarget,
         GenericTarget,
-        ResourceTarget,
-        ResourcesGeneratorTarget,
-        RelocatedFiles,
         LocalEnvironmentTarget,
-        DockerEnvironmentTarget,
+        LockfilesGeneratorTarget,
+        LockfileTarget,
+        RelocatedFiles,
         RemoteEnvironmentTarget,
+        ResourcesGeneratorTarget,
+        ResourceTarget,
     ]
 
 

src/python/pants/core/target_types.py

@@ -187,7 +187,7 @@ async def _hydrate_asset_source(request: GenerateSourcesRequest) -> GeneratedSou
 
 
 # -----------------------------------------------------------------------------------------------
-# `file` and`files` targets
+# `file` and `files` targets
 # -----------------------------------------------------------------------------------------------
 class FileSourceField(AssetSourceField):
     uses_source_roots = False
@@ -721,6 +721,49 @@ async def package_archive_target(field_set: ArchiveFieldSet) -> BuiltPackage:
     return BuiltPackage(archive, (BuiltPackageArtifact(output_filename),))
 
 
+# -----------------------------------------------------------------------------------------------
+# `lockfile` and `lockfiles` target
+# -----------------------------------------------------------------------------------------------
+
+
+class LockfileSourceField(SingleSourceField):
+    uses_source_roots = False
+    required = True
+
+
+class LockfileDependenciesField(Dependencies):
+    pass
+
+
+class LockfileTarget(Target):
+    alias = "lockfile"
+    core_fields = (*COMMON_TARGET_FIELDS, LockfileSourceField, LockfileDependenciesField)
+    help = softwrap(
+        """
+        A target for lockfiles in order to include them in the dependency graph of other targets.
+
+        This tracks them so that `--changed-since --changed-dependees` works properly for targets
+        relying on a particular lockfile.
+        """
+    )
+
+
+class LockfilesGeneratorSourcesField(MultipleSourcesField):
+    pass
+
+
+class LockfilesGeneratorTarget(TargetFilesGenerator):
+    alias = "lockfiles"
+    core_fields = (
+        *COMMON_TARGET_FIELDS,
+        LockfilesGeneratorSourcesField,
+    )
+    generated_target_cls = LockfileTarget
+    copied_fields = COMMON_TARGET_FIELDS
+    moved_fields = (LockfileDependenciesField,)
+    help = "Generate a `lockfile` target for each file in the `sources` field."
+
+
 def rules():
     return (
         *collect_rules(),