[Call-by-name] Migrated django framework (#22007)

sureshjoshi · web-flow · commit 78962d6875c7 · 2025-02-26T11:58:36.000Z
This is a PR for the call-by-name migration on the Django backend.
diff --git a/docs/notes/2.26.x.md b/docs/notes/2.26.x.md
@@ -33,6 +33,7 @@ Pants has a new mechanism for `@rule` invocation in backends. In this release th
 - [autoflake](https://www.pantsbuild.org/stable/reference/subsystems/autoflake)
 - [bandit](https://www.pantsbuild.org/stable/reference/subsystems/bandit)
 - [black](https://www.pantsbuild.org/stable/reference/subsystems/black)
+- [django](https://www.pantsbuild.org/dev/docs/using-pants/key-concepts/backends#available-experimental-backends)
 - [docformatter](https://www.pantsbuild.org/stable/reference/subsystems/docformatter)
 - [flake8](https://www.pantsbuild.org/stable/reference/subsystems/flake8)
 - [isort](https://www.pantsbuild.org/stable/reference/subsystems/isort)
diff --git a/src/python/pants/backend/python/framework/django/dependency_inference.py b/src/python/pants/backend/python/framework/django/dependency_inference.py
@@ -14,24 +14,24 @@
 from pants.backend.python.dependency_inference.rules import (
     ImportOwnerStatus,
     PythonImportDependenciesInferenceFieldSet,
-    ResolvedParsedPythonDependencies,
     ResolvedParsedPythonDependenciesRequest,
+    resolve_parsed_dependencies,
 )
 from pants.backend.python.framework.django.detect_apps import DjangoApps
 from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import EntryPoint
 from pants.backend.python.util_rules import pex
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
-from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess
+from pants.backend.python.util_rules.pex import PexRequest, VenvPexProcess, create_venv_pex
 from pants.base.specs import FileGlobSpec, RawSpecs
 from pants.core.util_rules.source_files import SourceFilesRequest
-from pants.core.util_rules.stripped_source_files import StrippedSourceFiles
+from pants.core.util_rules.stripped_source_files import strip_source_roots
 from pants.engine.fs import CreateDigest, FileContent
-from pants.engine.internals.native_engine import Digest
-from pants.engine.internals.selectors import Get
-from pants.engine.process import ProcessResult
-from pants.engine.rules import collect_rules, rule
-from pants.engine.target import InferDependenciesRequest, InferredDependencies, Targets
+from pants.engine.internals.graph import resolve_targets
+from pants.engine.intrinsics import create_digest
+from pants.engine.process import execute_process_or_raise
+from pants.engine.rules import collect_rules, implicitly, rule
+from pants.engine.target import InferDependenciesRequest, InferredDependencies
 from pants.engine.unions import UnionRule
 from pants.util.logging import LogLevel
 from pants.util.resources import read_resource
@@ -53,34 +53,36 @@ async def _django_migration_dependencies(
     python_setup: PythonSetup,
     django_apps: DjangoApps,
 ) -> InferredDependencies:
-    stripped_sources = await Get(
-        StrippedSourceFiles, SourceFilesRequest([request.field_set.source])
+    stripped_sources = await strip_source_roots(
+        **implicitly(SourceFilesRequest([request.field_set.source]))
     )
     assert len(stripped_sources.snapshot.files) == 1
 
     file_content = FileContent("__visitor.py", read_resource(__name__, _visitor_resource))
-    visitor_digest = await Get(Digest, CreateDigest([file_content]))
-    venv_pex = await Get(
-        VenvPex,
-        PexRequest(
-            output_filename="__visitor.pex",
-            internal_only=True,
-            main=EntryPoint("__visitor"),
-            interpreter_constraints=InterpreterConstraints.create_from_compatibility_fields(
-                [request.field_set.interpreter_constraints], python_setup=python_setup
-            ),
-            sources=visitor_digest,
-        ),
+    visitor_digest = await create_digest(CreateDigest([file_content]))
+    venv_pex = await create_venv_pex(
+        **implicitly(
+            PexRequest(
+                output_filename="__visitor.pex",
+                internal_only=True,
+                main=EntryPoint("__visitor"),
+                interpreter_constraints=InterpreterConstraints.create_from_compatibility_fields(
+                    [request.field_set.interpreter_constraints], python_setup=python_setup
+                ),
+                sources=visitor_digest,
+            )
+        )
     )
-    process_result = await Get(
-        ProcessResult,
-        VenvPexProcess(
-            venv_pex,
-            argv=[stripped_sources.snapshot.files[0]],
-            description=f"Determine Django app dependencies for {request.field_set.address}",
-            input_digest=stripped_sources.snapshot.digest,
-            level=LogLevel.DEBUG,
-        ),
+    process_result = await execute_process_or_raise(
+        **implicitly(
+            VenvPexProcess(
+                venv_pex,
+                argv=[stripped_sources.snapshot.files[0]],
+                description=f"Determine Django app dependencies for {request.field_set.address}",
+                input_digest=stripped_sources.snapshot.digest,
+                level=LogLevel.DEBUG,
+            )
+        )
     )
     # See in script for where we explicitly encoded as utf8. Even though utf8 is the
     # default for decode(), we make that explicit here for emphasis.
@@ -92,8 +94,7 @@ async def _django_migration_dependencies(
     ]
     resolve = request.field_set.resolve.normalized_value(python_setup)
 
-    resolved_dependencies = await Get(
-        ResolvedParsedPythonDependencies,
+    resolved_dependencies = await resolve_parsed_dependencies(
         ResolvedParsedPythonDependenciesRequest(
             request.field_set,
             ParsedPythonDependencies(
@@ -104,6 +105,7 @@ async def _django_migration_dependencies(
             ),
             resolve,
         ),
+        **implicitly(),
     )
 
     return InferredDependencies(
@@ -136,8 +138,7 @@ async def _django_app_implicit_dependencies(
 
     resolve = request.field_set.resolve.normalized_value(python_setup)
 
-    resolved_dependencies = await Get(
-        ResolvedParsedPythonDependencies,
+    resolved_dependencies = await resolve_parsed_dependencies(
         ResolvedParsedPythonDependenciesRequest(
             request.field_set,
             ParsedPythonDependencies(
@@ -149,6 +150,7 @@ async def _django_app_implicit_dependencies(
             ),
             resolve,
         ),
+        **implicitly(),
     )
 
     spec_paths = [
@@ -158,13 +160,13 @@ async def _django_app_implicit_dependencies(
         for address in result.address
     ]
 
-    targets = await Get(
-        Targets,
-        RawSpecs,
-        RawSpecs.create(
-            specs=[FileGlobSpec(f"{spec_path}/*.py") for spec_path in spec_paths],
-            description_of_origin="Django implicit dependency detection",
-        ),
+    targets = await resolve_targets(
+        **implicitly(
+            RawSpecs.create(
+                specs=[FileGlobSpec(f"{spec_path}/*.py") for spec_path in spec_paths],
+                description_of_origin="Django implicit dependency detection",
+            )
+        )
     )
 
     return InferredDependencies(sorted(target.address for target in targets))
@@ -188,9 +190,9 @@ async def infer_django_dependencies(
 
 
 def rules():
-    return [
+    return (
         *collect_rules(),
         *pex.rules(),
         *dependency_inference.rules.rules(),
         UnionRule(InferDependenciesRequest, InferDjangoDependencies),
-    ]
+    )
diff --git a/src/python/pants/backend/python/framework/django/detect_apps.py b/src/python/pants/backend/python/framework/django/detect_apps.py
@@ -5,25 +5,22 @@
 import json
 import os
 from collections import defaultdict
+from collections.abc import Iterable
 from dataclasses import dataclass
 
 from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import InterpreterConstraintsField
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
-from pants.backend.python.util_rules.pex_environment import PythonExecutable
+from pants.backend.python.util_rules.pex import find_interpreter
 from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
 from pants.base.specs import FileGlobSpec, RawSpecs
-from pants.engine.fs import AddPrefix, CreateDigest, Digest, FileContent, MergeDigests
-from pants.engine.internals.selectors import Get, MultiGet
-from pants.engine.process import Process, ProcessResult
-from pants.engine.rules import collect_rules, rule
-from pants.engine.target import (
-    HydratedSources,
-    HydrateSourcesRequest,
-    SourcesField,
-    Target,
-    Targets,
-)
+from pants.engine.fs import AddPrefix, CreateDigest, FileContent, MergeDigests
+from pants.engine.internals.graph import hydrate_sources, resolve_targets
+from pants.engine.internals.selectors import concurrently
+from pants.engine.intrinsics import add_prefix, create_digest, merge_digests
+from pants.engine.process import Process, execute_process_or_raise
+from pants.engine.rules import Rule, collect_rules, implicitly, rule
+from pants.engine.target import HydrateSourcesRequest, SourcesField, Target
 from pants.util.frozendict import FrozenDict
 from pants.util.resources import read_resource
 
@@ -43,7 +40,7 @@ def label_to_name(self) -> FrozenDict[str, str]:
     def label_to_file(self) -> FrozenDict[str, str]:
         return FrozenDict((label, app.config_file) for label, app in self.items())
 
-    def add_from_json(self, json_bytes: bytes, strip_prefix="") -> DjangoApps:
+    def add_from_json(self, json_bytes: bytes, strip_prefix: str = "") -> DjangoApps:
         json_dict: dict[str, dict[str, str]] = json.loads(json_bytes.decode())
         apps = {
             label: DjangoApp(
@@ -85,22 +82,22 @@ async def detect_django_apps(python_setup: PythonSetup) -> DjangoApps:
     #  be dep-inferred as a whole via the full package path in settings.py anyway.
     #  In the future we may find a way to map third-party apps here as well.
     django_apps = DjangoApps(FrozenDict())
-    targets = await Get(
-        Targets,
-        RawSpecs,
-        RawSpecs.create(
-            specs=[FileGlobSpec("**/apps.py")],
-            description_of_origin="Django app detection",
-            unmatched_glob_behavior=GlobMatchErrorBehavior.ignore,
-        ),
+    targets = await resolve_targets(
+        **implicitly(
+            RawSpecs.create(
+                specs=[FileGlobSpec("**/apps.py")],
+                description_of_origin="Django app detection",
+                unmatched_glob_behavior=GlobMatchErrorBehavior.ignore,
+            )
+        )
     )
     if not targets:
         return django_apps
 
     script_file_content = FileContent(
         "script/__visitor.py", read_resource(__name__, _script_resource)
     )
-    script_digest = await Get(Digest, CreateDigest([script_file_content]))
+    script_digest = await create_digest(CreateDigest([script_file_content]))
     apps_sandbox_prefix = "_apps_to_detect"
 
     # Partition by ICs, so we can run the detector on the appropriate interpreter.
@@ -112,34 +109,30 @@ async def detect_django_apps(python_setup: PythonSetup) -> DjangoApps:
         ics_to_tgts[ics].append(tgt)
 
     for ics, tgts in ics_to_tgts.items():
-        sources = await MultiGet(  # noqa: PNT30: requires triage
-            [Get(HydratedSources, HydrateSourcesRequest(tgt[SourcesField])) for tgt in tgts]
-        )
-        apps_digest = await Get(  # noqa: PNT30: requires triage
-            Digest, MergeDigests([src.snapshot.digest for src in sources])
+        sources = await concurrently(  # noqa: PNT30: requires triage
+            [
+                hydrate_sources(HydrateSourcesRequest(tgt[SourcesField]), **implicitly())
+                for tgt in tgts
+            ]
         )
-        prefixed_apps_digest = await Get(  # noqa: PNT30: requires triage
-            Digest, AddPrefix(apps_digest, apps_sandbox_prefix)
-        )
-
-        input_digest = await Get(  # noqa: PNT30: requires triage
-            Digest, MergeDigests([prefixed_apps_digest, script_digest])
-        )
-        python_interpreter = await Get(  # noqa: PNT30: requires triage
-            PythonExecutable, InterpreterConstraints, ics
-        )
-
-        process_result = await Get(  # noqa: PNT30: requires triage
-            ProcessResult,
-            Process(
-                argv=[
-                    python_interpreter.path,
-                    script_file_content.path,
-                    apps_sandbox_prefix,
-                ],
-                input_digest=input_digest,
-                description="Detect Django apps",
-            ),
+        apps_digest = await merge_digests(MergeDigests([src.snapshot.digest for src in sources]))
+        prefixed_apps_digest = await add_prefix(AddPrefix(apps_digest, apps_sandbox_prefix))
+
+        input_digest = await merge_digests(MergeDigests([prefixed_apps_digest, script_digest]))
+        python_interpreter = await find_interpreter(ics, **implicitly())
+
+        process_result = await execute_process_or_raise(
+            **implicitly(
+                Process(
+                    argv=[
+                        python_interpreter.path,
+                        script_file_content.path,
+                        apps_sandbox_prefix,
+                    ],
+                    input_digest=input_digest,
+                    description="Detect Django apps",
+                )
+            )
         )
         django_apps = django_apps.add_from_json(
             process_result.stdout or b"{}", strip_prefix=apps_sandbox_prefix
@@ -148,7 +141,5 @@ async def detect_django_apps(python_setup: PythonSetup) -> DjangoApps:
     return django_apps
 
 
-def rules():
-    return [
-        *collect_rules(),
-    ]
+def rules() -> Iterable[Rule]:
+    return collect_rules()
