Begin adding goal to assist with the call-by-name @rule syntax migration. (#20574)

As discussed in #20572, this change adds a built-in goal
`migrate-call-by-name` which emits all `Get` calls which should be
migrated to call-by-name syntax.

A followup change should use the resulting information (`@rule` function
pointers and `Get` signatures) to execute rewrites of the relevant
files.

Emits lines like:
```
<function infer_python_conftest_dependencies at 0x113633af0>
  Get(<class 'pants.engine.target.Targets'>, [<class 'pants.engine.addresses.Addresses'>]) -> <function resolve_targets at 0x112335d30>
  Get(<class 'pants.engine.internals.graph.Owners'>, [<class 'pants.engine.internals.graph.OwnersRequest'>]) -> <function find_owners at 0x112349160>
  Get(<class 'pants.backend.python.util_rules.ancestor_files.AncestorFiles'>, [<class 'pants.backend.python.util_rules.ancestor_files.AncestorFilesRequest'>]) -> <function find_ancestor_files at 0x1131e6790>
```

Author: stuhood

src/python/pants/engine/internals/native_engine.pyi

@@ -722,6 +722,9 @@ def validate_reachability(scheduler: PyScheduler) -> None: ...
 def rule_graph_consumed_types(
     scheduler: PyScheduler, param_types: Sequence[type], product_type: type
 ) -> list[type]: ...
+def rule_graph_rule_gets(
+    scheduler: PyScheduler,
+) -> dict[Callable, list[tuple[type, list[type], Callable]]]: ...
 def rule_graph_visualize(scheduler: PyScheduler, path: str) -> None: ...
 def rule_subgraph_visualize(
     scheduler: PyScheduler, param_types: Sequence[type], product_type: type, path: str

src/python/pants/engine/internals/scheduler.py

@@ -9,7 +9,7 @@
 from dataclasses import dataclass
 from pathlib import PurePath
 from types import CoroutineType
-from typing import Any, Dict, Iterable, NoReturn, Sequence, cast
+from typing import Any, Callable, Dict, Iterable, NoReturn, Sequence, cast
 
 from typing_extensions import TypedDict
 
@@ -405,6 +405,9 @@ def visualize_graph_to_file(self, filename: str) -> None:
     def visualize_rule_graph_to_file(self, filename: str) -> None:
         self._scheduler.visualize_rule_graph_to_file(filename)
 
+    def rule_graph_rule_gets(self) -> dict[Callable, list[tuple[type, list[type], Callable]]]:
+        return native_engine.rule_graph_rule_gets(self.py_scheduler)
+
     def execution_request(
         self,
         requests: Sequence[tuple[type, Any | Params]],

src/python/pants/goal/builtins.py

@@ -8,6 +8,7 @@
 from pants.goal import help
 from pants.goal.builtin_goal import BuiltinGoal
 from pants.goal.explorer import ExplorerBuiltinGoal
+from pants.goal.migrate_call_by_name import MigrateCallByNameBuiltinGoal
 
 
 def register_builtin_goals(build_configuration: BuildConfiguration.Builder) -> None:
@@ -18,6 +19,7 @@ def builtin_goals() -> tuple[type[BuiltinGoal], ...]:
     return (
         BSPGoal,
         ExplorerBuiltinGoal,
+        MigrateCallByNameBuiltinGoal,
         help.AllHelpBuiltinGoal,
         help.NoGoalHelpBuiltinGoal,
         help.ThingHelpBuiltinGoal,

src/python/pants/goal/migrate_call_by_name.py

@@ -0,0 +1,37 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+import logging
+
+from pants.base.exiter import ExitCode
+from pants.base.specs import Specs
+from pants.build_graph.build_configuration import BuildConfiguration
+from pants.engine.unions import UnionMembership
+from pants.goal.builtin_goal import BuiltinGoal
+from pants.init.engine_initializer import GraphSession
+from pants.option.options import Options
+
+logger = logging.getLogger(__name__)
+
+
+class MigrateCallByNameBuiltinGoal(BuiltinGoal):
+    name = "migrate-call-by-name"
+    help = "Migrate from `Get` syntax to call-by-name syntax. See #19730."
+
+    def run(
+        self,
+        build_config: BuildConfiguration,
+        graph_session: GraphSession,
+        options: Options,
+        specs: Specs,
+        union_membership: UnionMembership,
+    ) -> ExitCode:
+        # Emit all `@rules` which use non-union Gets.
+        for rule, dependencies in graph_session.scheduler_session.rule_graph_rule_gets().items():
+            print(f"{rule}")
+            for output_type, input_types, rule_dep in dependencies:
+                print(f"  Get({output_type}, {input_types}) -> {rule_dep}")
+
+        return 0

src/rust/engine/rule_graph/src/lib.rs

@@ -307,6 +307,34 @@ impl<R: Rule> RuleGraph<R> {
             .collect()
     }
 
+    ///
+    /// Returns a mapping from a `Rule` to the dependencies it has that are themselves satisfied by
+    /// `Rule`s.
+    ///
+    #[allow(clippy::type_complexity)]
+    pub fn rule_dependencies(&self) -> HashMap<&R, Vec<(&DependencyKey<R::TypeId>, &R)>> {
+        let mut result = HashMap::default();
+        for (entry, rule_edges) in &self.rule_dependency_edges {
+            let EntryWithDeps::Rule(RuleEntry { rule, .. }) = entry.as_ref() else {
+                continue;
+            };
+
+            let mut function_satisfied_gets = Vec::new();
+            for (dependency, source) in &rule_edges.dependencies {
+                let Entry::WithDeps(entry_with_deps) = source.as_ref() else {
+                    continue;
+                };
+                let EntryWithDeps::Rule(RuleEntry { rule, .. }) = entry_with_deps.as_ref() else {
+                    continue;
+                };
+
+                function_satisfied_gets.push((dependency, rule));
+            }
+            result.insert(rule, function_satisfied_gets);
+        }
+        result
+    }
+
     ///
     /// Find the entrypoint in this RuleGraph for the given product and params.
     ///

src/rust/engine/src/externs/interface.rs

@@ -125,6 +125,7 @@ fn native_engine(py: Python, m: &PyModule) -> PyO3Result<()> {
 
     m.add_function(wrap_pyfunction!(validate_reachability, m)?)?;
     m.add_function(wrap_pyfunction!(rule_graph_consumed_types, m)?)?;
+    m.add_function(wrap_pyfunction!(rule_graph_rule_gets, m)?)?;
     m.add_function(wrap_pyfunction!(rule_graph_visualize, m)?)?;
     m.add_function(wrap_pyfunction!(rule_subgraph_visualize, m)?)?;
 
@@ -1399,6 +1400,48 @@ fn rule_graph_consumed_types<'py>(
     })
 }
 
+#[pyfunction]
+fn rule_graph_rule_gets<'p>(py: Python<'p>, py_scheduler: &PyScheduler) -> PyO3Result<&'p PyDict> {
+    let core = &py_scheduler.0.core;
+    core.executor.enter(|| {
+        let result = PyDict::new(py);
+        for (rule, rule_dependencies) in core.rule_graph.rule_dependencies() {
+            let Rule::Task(task) = rule else { continue };
+
+            let function = &task.func;
+            let mut dependencies = Vec::new();
+            for (dependency_key, rule) in rule_dependencies {
+                // NB: We are only migrating non-union Gets, which are those in the `gets` list
+                // which do not have `in_scope_params` marking them as being for unions, or a call
+                // signature marking them as already being call-by-name.
+                if dependency_key.call_signature.is_some()
+                    || dependency_key.in_scope_params.is_some()
+                    || !task.gets.contains(dependency_key)
+                {
+                    continue;
+                }
+                let Rule::Task(task) = rule else { continue };
+
+                let provided_params = dependency_key
+                    .provided_params
+                    .iter()
+                    .map(|p| p.as_py_type(py))
+                    .collect::<Vec<_>>();
+                dependencies.push((
+                    dependency_key.product.as_py_type(py),
+                    provided_params,
+                    task.func.0.value.into_py(py),
+                ));
+            }
+            if dependencies.is_empty() {
+                continue;
+            }
+            result.set_item(function.0.value.into_py(py), dependencies.into_py(py))?;
+        }
+        Ok(result)
+    })
+}
+
 #[pyfunction]
 fn rule_graph_visualize(py_scheduler: &PyScheduler, path: PathBuf) -> PyO3Result<()> {
     let core = &py_scheduler.0.core;

src/rust/engine/src/externs/mod.rs

@@ -66,7 +66,7 @@ pub struct PyFailure(pub Failure);
 impl PyFailure {
     fn get_error(&self, py: Python) -> PyErr {
         match &self.0 {
-            Failure::Throw { val, .. } => val.into_py(py),
+            Failure::Throw { val, .. } => PyErr::from_value(val.as_ref().as_ref(py)),
             f @ (Failure::Invalidated | Failure::MissingDigest { .. }) => {
                 EngineError::new_err(format!("{f}"))
             }

src/rust/engine/src/python.rs

@@ -378,15 +378,9 @@ impl From<PyObject> for Value {
     }
 }
 
-impl From<PyErr> for Value {
-    fn from(py_err: PyErr) -> Self {
-        Python::with_gil(|py| Value::new(py_err.into_py(py)))
-    }
-}
-
-impl IntoPy<PyErr> for &Value {
-    fn into_py(self, py: Python) -> PyErr {
-        PyErr::from_value((*self.0).as_ref(py))
+impl IntoPy<PyObject> for &Value {
+    fn into_py(self, py: Python) -> PyObject {
+        (*self.0).as_ref(py).into_py(py)
     }
 }