Make call boundaries own deferred quantified finishing

Summary:
Deferred finishing should be boundary-scoped and explicit, not reachability-driven. This change makes
`CallContext` tracking the source of truth for which fresh quantified vars must be finished, defers
Forall-instantiated vars during call analysis, and keeps eager finishing only for ad-hoc subset checks outside calls.

Also, we're now able to clean up a good chunk of code since the new helpers added in D103328383
are now used uniformly.

This fixes a number of regressions that were related to the incorrect early finishing of generics.

Reviewed By: rchen152

Differential Revision: D103337567

fbshipit-source-id: 808e848b890b744386b87d89b22c70d20638c4bb

Author: stroxler

pyrefly/lib/solver/solver.rs

@@ -2596,96 +2596,25 @@ impl Solver {
             .collect();
         let mut roots: SmallSet<Var> = vs.0.into_iter().collect();
         roots.extend(tracked_fresh_vars.0);
-        // Forall instantiation during call analysis can unify call-scope vars
-        // with additional fresh vars that are only visible through Answer /
-        // ResidualAnswer payloads. Finish the full reachable closure so no
-        // reachable Variable::Quantified can leak to pinning.
-        //
-        // We also must include reachable vars that already became Answer but
-        // still have pending instantiation errors. Those errors are surfaced by
-        // finish_quantified, so excluding Answer vars here can silently drop
-        // call-site specialization diagnostics.
-        let roots = roots.into_iter().collect::<Vec<_>>();
-        let mut already_finished: SmallSet<Var> = SmallSet::new();
-        loop {
-            // Fixed-point: finishing can mutate solver state and expose new
-            // reachable vars that also require finishing.
-            let reachable_finish_vars = self.reachable_finish_vars_from_roots(&roots);
-            let mut next_round: Vec<Var> = reachable_finish_vars
-                .into_iter()
-                .filter(|var| !already_finished.contains(var))
-                .collect();
-            // Payload-driven overload pruning must consider solved vars even if
-            // they already collapsed to `Answer` before finishing and therefore
-            // are not selected by reachability-based finishing alone.
-            next_round.extend(
-                payload_vars
-                    .iter()
-                    .copied()
-                    .filter(|var| !already_finished.contains(var)),
-            );
-            next_round.sort_unstable();
-            next_round.dedup();
-            if next_round.is_empty() {
-                break;
-            }
-            already_finished.extend(next_round.iter().copied());
-            let mut subset = self.subset(type_order);
-            self.finish_quantified_with_subset_and_payloads(
-                QuantifiedHandle(next_round),
-                infer_with_first_use,
-                &mut |got, want| subset.is_subset_eq_probe_for_pruning(got, want),
-                Some(&overload_witness_payloads),
-            )?;
-        }
-        Ok(())
-    }
-
-    fn reachable_finish_vars_from_roots(&self, roots: &[Var]) -> SmallSet<Var> {
-        if roots.is_empty() {
-            return SmallSet::new();
-        }
-        let variables = self.variables.lock();
-        let instantiation_errors = self.instantiation_errors.read();
-        let mut visited: SmallSet<Var> = SmallSet::new();
-        let mut reachable_finish_vars: SmallSet<Var> = SmallSet::new();
-        let mut stack = roots.to_vec();
-        while let Some(var) = stack.pop() {
-            if !visited.insert(var) {
-                continue;
-            }
-            let variable = variables.get(var);
-            let needs_finish = match &*variable {
-                Variable::Quantified { .. } => true,
-                Variable::Answer(_) | Variable::ResidualAnswer { .. } => {
-                    instantiation_errors.contains_key(&var)
-                }
-                _ => false,
-            };
-            if needs_finish {
-                reachable_finish_vars.insert(var);
-            }
-            match &*variable {
-                Variable::Answer(ty) => {
-                    stack.extend(ty.collect_maybe_placeholder_vars());
-                }
-                Variable::ResidualAnswer { target_vars: _, ty } => {
-                    // `target_vars` are read-gates for residual visibility,
-                    // not ownership edges for finishing reachability.
-                    stack.extend(ty.collect_maybe_placeholder_vars());
-                }
-                Variable::Quantified { .. } => {}
-                Variable::Unwrap(bounds) => {
-                    for ty in bounds.lower.iter().chain(bounds.upper.iter()) {
-                        stack.extend(ty.collect_maybe_placeholder_vars());
-                    }
-                }
-                Variable::PartialQuantified(_)
-                | Variable::PartialContained(_)
-                | Variable::Recursive => {}
-            }
+        // Solve boundaries explicitly own fresh quantified tracking. We finish
+        // the exact boundary set (explicit roots + fresh vars + payload vars),
+        // rather than using reachability expansion that can miss or overreach.
+        let mut all_boundary_vars: Vec<Var> = roots.into_iter().collect();
+        // Payload-driven overload pruning must include solved vars even if they
+        // already collapsed to `Answer` before boundary finishing.
+        all_boundary_vars.extend(payload_vars);
+        all_boundary_vars.sort_unstable();
+        all_boundary_vars.dedup();
+        if all_boundary_vars.is_empty() {
+            return Ok(());
         }
-        reachable_finish_vars
+        let mut subset = self.subset(type_order);
+        self.finish_quantified_with_subset_and_payloads(
+            QuantifiedHandle(all_boundary_vars),
+            infer_with_first_use,
+            &mut |got, want| subset.is_subset_eq_probe_for_pruning(got, want),
+            Some(&overload_witness_payloads),
+        )
     }
 
     /// Finish all quantified vars reachable from `ty` using the solver default

pyrefly/lib/solver/subset.rs

@@ -2246,11 +2246,14 @@ impl<'a, Ans: LookupAnswer> Subset<'a, Ans> {
             ),
             (Type::Forall(forall), _) => {
                 let forall_type = Type::Forall(forall.clone());
-                // Finalizing the quantified vars returns instantiation errors
+                // Instantiate once, then either defer finishing to the active
+                // call boundary (when inside call analysis) or finish eagerly
+                // for ad-hoc subset checks outside calls.
                 let (vs, got) = self.type_order.instantiate_fresh_forall((**forall).clone());
                 self.active_call_context
                     .register_fresh_quantified_vars(vs.vars());
                 let argument_side = self.active_argument_side();
+                let in_call_analysis = !matches!(argument_side, ArgumentSide::NotAnalyzingACall);
                 let witness = self.make_forall_witness(&forall_type, &vs, want);
                 let (result, mut maybe_witness) =
                     self.with_active_witness_and_side(witness, argument_side, |me| {
@@ -2267,15 +2270,24 @@ impl<'a, Ans: LookupAnswer> Subset<'a, Ans> {
                     }
                     self.solver.record_generic_residuals_for_witness(witness);
                 }
-                result.and(
-                    self.solver
+                if in_call_analysis {
+                    result
+                } else {
+                    // Even when subset checking fails, finish the fresh vars
+                    // to avoid leaking Quantified placeholders in ad-hoc paths.
+                    let finish_result = self
+                        .solver
                         .finish_quantified_with_subset(
                             vs,
                             self.solver.infer_with_first_use,
                             &mut |got, want| self.is_subset_eq_probe_for_pruning(got, want),
                         )
-                        .map_err(SubsetError::TypeVarSpecialization),
-                )
+                        .map_err(SubsetError::TypeVarSpecialization);
+                    match result {
+                        Ok(()) => finish_result,
+                        Err(e) => Err(e),
+                    }
+                }
             }
             (_, Type::Forall(forall)) => self.is_subset_eq(got, &forall.body.clone().as_type()),
             (Type::TypeVar(l), Type::TypeVar(u)) => {

pyrefly/lib/test/calls.rs

@@ -226,11 +226,10 @@ f()  # E: Deprecated
 );
 
 testcase!(
-    bug = "There's a bug in generic overload resolution where we finish too early",
     test_reduce_call,
     r#"
 from functools import reduce
-reduce(max, [1,2])  # E: Overload type was not compatible with solved type variables: _T = int
+reduce(max, [1,2])
     "#,
 );
 

pyrefly/lib/test/constructors.rs

@@ -820,18 +820,17 @@ takes_Cstr_wrong(C) # E: Argument `type[C]` is not assignable to parameter `x` w
 );
 
 testcase!(
-    bug = "Early finishing of generic residuals is causing false positives here",
     test_init_to_callable_generics,
     r#"
 from typing import Generic, TypeVar, assert_type, Callable
 T = TypeVar("T")
 class C(Generic[T]):
     def __init__[V](self: "C[V]", x: V) -> None: pass
 def takes_callable[V](x: Callable[[V], C[V]], y: V) -> C[V]: ...
-out1 = takes_callable(C, 42)  # E: Argument `Literal[42]` is not assignable to parameter `y` with type `GenericResidual@V` in function `takes_callable`
-assert_type(out1, C[int])  # E: assert_type(C[GenericResidual@V], C[int]) failed
-out2 = takes_callable(C, "hello")  # E: Argument `Literal['hello']` is not assignable to parameter `y` with type `GenericResidual@V` in function `takes_callable`
-assert_type(out2, C[str])  # E: assert_type(C[GenericResidual@V], C[str]) failed
+out1 = takes_callable(C, 42)
+assert_type(out1, C[int])
+out2 = takes_callable(C, "hello")
+assert_type(out2, C[str])
     "#,
 );
 

pyrefly/lib/test/generic_restrictions.rs

@@ -668,14 +668,13 @@ assert_type(f(), int)
 );
 
 testcase!(
-    bug = "Early finishing of generic residuals is causing false positives here",
     test_arg_against_typevar_bound,
     r#"
 from typing import Callable, Iterable
 def reduce[_S](function: Callable[[_S, _S], _S], iterable: Iterable[_S]) -> _S: ...
 def f[_T: str](arg1: _T, arg2: _T) -> _T: ...
-reduce(f, [1])  # E: Argument `list[int]` is not assignable to parameter `iterable` with type `Iterable[GenericResidual@_T]` in function `reduce`
-reduce(f, ["ok"])  # E: Argument `list[str]` is not assignable to parameter `iterable` with type `Iterable[GenericResidual@_T]` in function `reduce`
+reduce(f, [1])  # E: `int` is not assignable to upper bound `str` of type variable `_T`
+reduce(f, ["ok"])
     "#,
 );