add tests for steps 2 and 3 of overload evaluation

Author: carljm

conformance/results/mypy/overloads_evaluation.toml

@@ -4,22 +4,24 @@ Does not pick a winning overload based on arity, prior to considering argument t
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 36: Unexpected errors ['overloads_evaluation.py:36: error: Expression is of type "Any", not "int"  [assert-type]']
-Line 42: Unexpected errors ['overloads_evaluation.py:42: error: Expression is of type "Any", not "str"  [assert-type]']
+Line 35: Unexpected errors ['overloads_evaluation.py:35: error: Expression is of type "Any", not "int"  [assert-type]']
+Line 41: Unexpected errors ['overloads_evaluation.py:41: error: Expression is of type "Any", not "str"  [assert-type]']
 """
 output = """
-overloads_evaluation.py:26: error: All overload variants of "num_args" require at least one argument  [call-overload]
+overloads_evaluation.py:26: error: All overload variants of "example1" require at least one argument  [call-overload]
 overloads_evaluation.py:26: note: Possible overload variants:
-overloads_evaluation.py:26: note:     def num_args(x: int, y: str) -> int
-overloads_evaluation.py:26: note:     def num_args(x: str) -> str
-overloads_evaluation.py:35: error: No overload variant of "num_args" matches argument types "int", "int"  [call-overload]
-overloads_evaluation.py:35: note: Possible overload variants:
-overloads_evaluation.py:35: note:     def num_args(x: int, y: str) -> int
-overloads_evaluation.py:35: note:     def num_args(x: str) -> str
-overloads_evaluation.py:36: error: Expression is of type "Any", not "int"  [assert-type]
-overloads_evaluation.py:41: error: No overload variant of "num_args" matches argument type "int"  [call-overload]
-overloads_evaluation.py:41: note: Possible overload variants:
-overloads_evaluation.py:41: note:     def num_args(x: int, y: str) -> int
-overloads_evaluation.py:41: note:     def num_args(x: str) -> str
-overloads_evaluation.py:42: error: Expression is of type "Any", not "str"  [assert-type]
+overloads_evaluation.py:26: note:     def example1(x: int, y: str) -> int
+overloads_evaluation.py:26: note:     def example1(x: str) -> str
+overloads_evaluation.py:34: error: No overload variant of "example1" matches argument types "int", "int"  [call-overload]
+overloads_evaluation.py:34: note: Possible overload variants:
+overloads_evaluation.py:34: note:     def example1(x: int, y: str) -> int
+overloads_evaluation.py:34: note:     def example1(x: str) -> str
+overloads_evaluation.py:35: error: Expression is of type "Any", not "int"  [assert-type]
+overloads_evaluation.py:40: error: No overload variant of "example1" matches argument type "int"  [call-overload]
+overloads_evaluation.py:40: note: Possible overload variants:
+overloads_evaluation.py:40: note:     def example1(x: int, y: str) -> int
+overloads_evaluation.py:40: note:     def example1(x: str) -> str
+overloads_evaluation.py:41: error: Expression is of type "Any", not "str"  [assert-type]
+overloads_evaluation.py:85: error: Argument 1 to "example2" has incompatible type "int | str"; expected "int"  [arg-type]
+overloads_evaluation.py:85: error: Argument 2 to "example2" has incompatible type "int | str"; expected "str"  [arg-type]
 """

conformance/results/pyre/overloads_evaluation.toml

@@ -3,7 +3,9 @@ conformance_automated = "Pass"
 errors_diff = """
 """
 output = """
-overloads_evaluation.py:26:0 Missing argument [20]: Call `num_args` expects argument `x`.
-overloads_evaluation.py:35:19 Incompatible parameter type [6]: In call `num_args`, for 2nd positional argument, expected `str` but got `int`.
-overloads_evaluation.py:41:16 Incompatible parameter type [6]: In call `num_args`, for 1st positional argument, expected `str` but got `int`.
+overloads_evaluation.py:26:0 Missing argument [20]: Call `example1` expects argument `x`.
+overloads_evaluation.py:34:19 Incompatible parameter type [6]: In call `example1`, for 2nd positional argument, expected `str` but got `int`.
+overloads_evaluation.py:40:16 Incompatible parameter type [6]: In call `example1`, for 1st positional argument, expected `str` but got `int`.
+overloads_evaluation.py:85:13 Incompatible parameter type [6]: In call `example2`, for 1st positional argument, expected `int` but got `Union[int, str]`.
+overloads_evaluation.py:85:16 Incompatible parameter type [6]: In call `example2`, for 2nd positional argument, expected `str` but got `Union[int, str]`.
 """

conformance/results/pyre/version.toml

@@ -1,2 +1,2 @@
 version = "pyre 0.9.23"
-test_duration = 6.7
+test_duration = 6.1

conformance/results/pyright/overloads_evaluation.toml

@@ -3,10 +3,17 @@ conformance_automated = "Pass"
 errors_diff = """
 """
 output = """
-overloads_evaluation.py:26:1 - error: No overloads for "num_args" match the provided arguments
+overloads_evaluation.py:26:1 - error: No overloads for "example1" match the provided arguments
   Argument types: () (reportCallIssue)
-overloads_evaluation.py:35:20 - error: Argument of type "Literal[1]" cannot be assigned to parameter "y" of type "str" in function "num_args"
+overloads_evaluation.py:34:20 - error: Argument of type "Literal[1]" cannot be assigned to parameter "y" of type "str" in function "example1"
   "Literal[1]" is not assignable to "str" (reportArgumentType)
-overloads_evaluation.py:41:17 - error: Argument of type "Literal[1]" cannot be assigned to parameter "x" of type "str" in function "num_args"
+overloads_evaluation.py:40:17 - error: Argument of type "Literal[1]" cannot be assigned to parameter "x" of type "str" in function "example1"
   "Literal[1]" is not assignable to "str" (reportArgumentType)
+overloads_evaluation.py:85:5 - error: No overloads for "example2" match the provided arguments (reportCallIssue)
+overloads_evaluation.py:85:14 - error: Argument of type "int | str" cannot be assigned to parameter "x" of type "int" in function "example2"
+  Type "int | str" is not assignable to type "int"
+    "str" is not assignable to "int" (reportArgumentType)
+overloads_evaluation.py:85:17 - error: Argument of type "int | str" cannot be assigned to parameter "y" of type "int" in function "example2"
+  Type "int | str" is not assignable to type "int"
+    "str" is not assignable to "int" (reportArgumentType)
 """

conformance/results/pyright/version.toml

@@ -1,2 +1,2 @@
 version = "pyright 1.1.391"
-test_duration = 1.5
+test_duration = 1.6

conformance/results/pytype/overloads_evaluation.toml

@@ -1,44 +1,58 @@
 conformant = "Partial"
 notes = """
 Does not pick a winning overload based on arity, prior to considering argument types.
+Does not perform argument expansion on unions and union return types of all matching overloads.
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 24: Unexpected errors ['overloads_evaluation.py:24:12: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in num_args: bad return type [bad-return-type]']
-Line 36: Unexpected errors ['overloads_evaluation.py:36:1: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in <module>: Any [assert-type]']
-Line 42: Unexpected errors ['overloads_evaluation.py:42:1: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in <module>: Any [assert-type]']
+Line 85: Expected 1 errors
+Line 22: Unexpected errors ['overloads_evaluation.py:22:12: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in example1: bad return type [bad-return-type]']
+Line 35: Unexpected errors ['overloads_evaluation.py:35:1: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in <module>: Any [assert-type]']
+Line 41: Unexpected errors ['overloads_evaluation.py:41:1: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in <module>: Any [assert-type]']
+Line 59: Unexpected errors ['overloads_evaluation.py:59:12: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in example2: bad return type [bad-return-type]']
+Line 78: Unexpected errors ['overloads_evaluation.py:78:5: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in _: str [assert-type]']
 """
 output = """
-overloads_evaluation.py:24:12: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in num_args: bad return type [bad-return-type]
+overloads_evaluation.py:22:12: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in example1: bad return type [bad-return-type]
 
     return 1
            \u001b[1m\u001b[31m~\u001b[39m\u001b[0m
 
 Called from (traceback):
-  line 38, in current file
-overloads_evaluation.py:26:1: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Missing parameter 'x' in call to function num_args [missing-parameter]
+  line 37, in current file
+overloads_evaluation.py:26:1: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Missing parameter 'x' in call to function example1 [missing-parameter]
 
-num_args()  # E: no matching overload
+example1()  # E: no matching overload
 \u001b[1m\u001b[31m~~~~~~~~~~\u001b[39m\u001b[0m
 
-overloads_evaluation.py:35:8: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Function num_args was called with the wrong arguments [wrong-arg-types]
+overloads_evaluation.py:34:8: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Function example1 was called with the wrong arguments [wrong-arg-types]
 
-ret2 = num_args(1, 1)  # E: Literal[1] not assignable to str
+ret2 = example1(1, 1)  # E: Literal[1] not assignable to str
        \u001b[1m\u001b[31m~~~~~~~~~~~~~~\u001b[39m\u001b[0m
 
-overloads_evaluation.py:36:1: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Any [assert-type]
+overloads_evaluation.py:35:1: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Any [assert-type]
 
 assert_type(ret2, int)
 \u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
 
-overloads_evaluation.py:41:8: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Function num_args was called with the wrong arguments [wrong-arg-types]
+overloads_evaluation.py:40:8: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Function example1 was called with the wrong arguments [wrong-arg-types]
 
-ret4 = num_args(1)  # E: Literal[1] not assignable to str
+ret4 = example1(1)  # E: Literal[1] not assignable to str
        \u001b[1m\u001b[31m~~~~~~~~~~~\u001b[39m\u001b[0m
 
-overloads_evaluation.py:42:1: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Any [assert-type]
+overloads_evaluation.py:41:1: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Any [assert-type]
 
 assert_type(ret4, str)
 \u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
 
+overloads_evaluation.py:59:12: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in example2: bad return type [bad-return-type]
+
+    return 1
+           \u001b[1m\u001b[31m~\u001b[39m\u001b[0m
+
+overloads_evaluation.py:78:5: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in _: str [assert-type]
+
+    assert_type(ret1, int | str)
+    \u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
+
 """

conformance/results/pytype/version.toml

@@ -1,2 +1,2 @@
 version = "pytype 2024.10.11"
-test_duration = 30.7
+test_duration = 31.5

conformance/results/results.html

@@ -162,13 +162,13 @@ <h3>Python Type System Conformance Test Results</h3>
 <div class='tc-time'>1.7sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pyright 1.1.391</div>
-<div class='tc-time'>1.5sec</div>
+<div class='tc-time'>1.6sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pyre 0.9.23</div>
-<div class='tc-time'>6.7sec</div>
+<div class='tc-time'>6.1sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pytype 2024.10.11</div>
-<div class='tc-time'>30.7sec</div>
+<div class='tc-time'>31.5sec</div>
 </th>
 </tr>
 <tr><th class="column" colspan="5">
@@ -694,7 +694,7 @@ <h3>Python Type System Conformance Test Results</h3>
 <th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not pick a winning overload based on arity, prior to considering argument types.</p></span></div></th>
 <th class="column col2 conformant">Pass</th>
 <th class="column col2 conformant">Pass</th>
-<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not pick a winning overload based on arity, prior to considering argument types.</p></span></div></th>
+<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not pick a winning overload based on arity, prior to considering argument types.</p><p>Does not perform argument expansion on unions and union return types of all matching overloads.</p></span></div></th>
 </tr>
 <tr><th class="column col1">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;overloads_overlap</th>
 <th class="column col2 conformant">Pass</th>

conformance/tests/overloads_evaluation.py

@@ -10,35 +10,77 @@
 # > overload candidates that are not plausible based on their
 # > input signatures.
 
-# > - If no candidate overloads remain, generate an error and stop.
-
 @overload
-def num_args(x: int, y: str) -> int:
+def example1(x: int, y: str) -> int:
     ...
 
 @overload
-def num_args(x: str) -> str:
+def example1(x: str) -> str:
     ...
 
-def num_args(x: int | str, y: str = "") -> int | str:
+def example1(x: int | str, y: str = "") -> int | str:
     return 1
 
-num_args()  # E: no matching overload
+# > - If no candidate overloads remain, generate an error and stop.
 
+example1()  # E: no matching overload
 
 # > - If only one candidate overload remains, it is the winning match. Evaluate
 # >   it as if it were a non-overloaded function call and stop.
 
-ret1 = num_args(1, "")
+ret1 = example1(1, "")
 assert_type(ret1, int)
 
-ret2 = num_args(1, 1)  # E: Literal[1] not assignable to str
+ret2 = example1(1, 1)  # E: Literal[1] not assignable to str
 assert_type(ret2, int)
 
-ret3 = num_args("")
+ret3 = example1("")
 assert_type(ret3, str)
 
-ret4 = num_args(1)  # E: Literal[1] not assignable to str
+ret4 = example1(1)  # E: Literal[1] not assignable to str
 assert_type(ret4, str)
 
 
+# > Step 2: Evaluate each remaining overload as a regular (non-overloaded)
+# > call to determine whether it is compatible with the supplied
+# > argument list. Unlike step 1, this step considers the types of the parameters
+# > and arguments. During this step, do not generate any user-visible errors.
+# > Simply record which of the overloads result in evaluation errors.
+
+@overload
+def example2(x: int, y: str, z: int) -> str:
+    ...
+
+@overload
+def example2(x: int, y: int, z: int) -> int:
+    ...
+
+def example2(x: int, y: int | str, z: int) -> int | str:
+    return 1
+
+# > - If only one overload evaluates without error, it is the winning match.
+# >   Evaluate it as if it were a non-overloaded function call and stop.
+
+ret5 = example2(1, 2, 3)
+assert_type(ret5, int)
+
+# > Step 3: If step 2 produces errors for all overloads, perform
+# > "argument type expansion". Union types can be expanded
+# > into their constituent subtypes. For example, the type ``int | str`` can
+# > be expanded into ``int`` and ``str``.
+
+# > - If all argument lists evaluate successfully, combine their
+# >   respective return types by union to determine the final return type
+# >   for the call, and stop.
+
+def _(v: int | str) -> None:
+    ret1 = example2(1, v, 1)
+    assert_type(ret1, int | str)
+
+# > - If argument expansion has been applied to all arguments and one or
+# >   more of the expanded argument lists cannot be evaluated successfully,
+# >   generate an error and stop.
+
+def _(v: int | str) -> None:
+    example2(v, v, 1)  # E: no overload matches (str, ..., ...)
+