Update semantic analyzer for TypeVar defaults (PEP 696) (#14873)

This PR updates the semantic analyzer to support most forms of TypeVars
with defaults while also providing basic argument validation.

Ref: #14851

Author: cdce8p

mypy/message_registry.py

@@ -181,7 +181,7 @@ def with_additional_msg(self, info: str) -> ErrorMessage:
 INVALID_TYPEVAR_ARG_BOUND: Final = 'Type argument {} of "{}" must be a subtype of {}'
 INVALID_TYPEVAR_ARG_VALUE: Final = 'Invalid type argument value for "{}"'
 TYPEVAR_VARIANCE_DEF: Final = 'TypeVar "{}" may only be a literal bool'
-TYPEVAR_BOUND_MUST_BE_TYPE: Final = 'TypeVar "bound" must be a type'
+TYPEVAR_ARG_MUST_BE_TYPE: Final = '{} "{}" must be a type'
 TYPEVAR_UNEXPECTED_ARGUMENT: Final = 'Unexpected argument to "TypeVar()"'
 UNBOUND_TYPEVAR: Final = (
     "A function returning TypeVar should receive at least "

mypy/semanal.py

@@ -4135,28 +4135,15 @@ def process_typevar_parameters(
                 if has_values:
                     self.fail("TypeVar cannot have both values and an upper bound", context)
                     return None
-                try:
-                    # We want to use our custom error message below, so we suppress
-                    # the default error message for invalid types here.
-                    analyzed = self.expr_to_analyzed_type(
-                        param_value, allow_placeholder=True, report_invalid_types=False
-                    )
-                    if analyzed is None:
-                        # Type variables are special: we need to place them in the symbol table
-                        # soon, even if upper bound is not ready yet. Otherwise avoiding
-                        # a "deadlock" in this common pattern would be tricky:
-                        #     T = TypeVar('T', bound=Custom[Any])
-                        #     class Custom(Generic[T]):
-                        #         ...
-                        analyzed = PlaceholderType(None, [], context.line)
-                    upper_bound = get_proper_type(analyzed)
-                    if isinstance(upper_bound, AnyType) and upper_bound.is_from_error:
-                        self.fail(message_registry.TYPEVAR_BOUND_MUST_BE_TYPE, param_value)
-                        # Note: we do not return 'None' here -- we want to continue
-                        # using the AnyType as the upper bound.
-                except TypeTranslationError:
-                    self.fail(message_registry.TYPEVAR_BOUND_MUST_BE_TYPE, param_value)
+                tv_arg = self.get_typevarlike_argument("TypeVar", param_name, param_value, context)
+                if tv_arg is None:
                     return None
+                upper_bound = tv_arg
+            elif param_name == "default":
+                tv_arg = self.get_typevarlike_argument(
+                    "TypeVar", param_name, param_value, context, allow_unbound_tvars=True
+                )
+                default = tv_arg or AnyType(TypeOfAny.from_error)
             elif param_name == "values":
                 # Probably using obsolete syntax with values=(...). Explain the current syntax.
                 self.fail('TypeVar "values" argument not supported', context)
@@ -4184,6 +4171,52 @@ def process_typevar_parameters(
             variance = INVARIANT
         return variance, upper_bound, default
 
+    def get_typevarlike_argument(
+        self,
+        typevarlike_name: str,
+        param_name: str,
+        param_value: Expression,
+        context: Context,
+        *,
+        allow_unbound_tvars: bool = False,
+        allow_param_spec_literals: bool = False,
+        report_invalid_typevar_arg: bool = True,
+    ) -> ProperType | None:
+        try:
+            # We want to use our custom error message below, so we suppress
+            # the default error message for invalid types here.
+            analyzed = self.expr_to_analyzed_type(
+                param_value,
+                allow_placeholder=True,
+                report_invalid_types=False,
+                allow_unbound_tvars=allow_unbound_tvars,
+                allow_param_spec_literals=allow_param_spec_literals,
+            )
+            if analyzed is None:
+                # Type variables are special: we need to place them in the symbol table
+                # soon, even if upper bound is not ready yet. Otherwise avoiding
+                # a "deadlock" in this common pattern would be tricky:
+                #     T = TypeVar('T', bound=Custom[Any])
+                #     class Custom(Generic[T]):
+                #         ...
+                analyzed = PlaceholderType(None, [], context.line)
+            typ = get_proper_type(analyzed)
+            if report_invalid_typevar_arg and isinstance(typ, AnyType) and typ.is_from_error:
+                self.fail(
+                    message_registry.TYPEVAR_ARG_MUST_BE_TYPE.format(typevarlike_name, param_name),
+                    param_value,
+                )
+                # Note: we do not return 'None' here -- we want to continue
+                # using the AnyType.
+            return typ
+        except TypeTranslationError:
+            if report_invalid_typevar_arg:
+                self.fail(
+                    message_registry.TYPEVAR_ARG_MUST_BE_TYPE.format(typevarlike_name, param_name),
+                    param_value,
+                )
+            return None
+
     def extract_typevarlike_name(self, s: AssignmentStmt, call: CallExpr) -> str | None:
         if not call:
             return None
@@ -4216,13 +4249,50 @@ def process_paramspec_declaration(self, s: AssignmentStmt) -> bool:
         if name is None:
             return False
 
-        # ParamSpec is different from a regular TypeVar:
-        # arguments are not semantically valid. But, allowed in runtime.
-        # So, we need to warn users about possible invalid usage.
-        if len(call.args) > 1:
-            self.fail("Only the first argument to ParamSpec has defined semantics", s)
+        n_values = call.arg_kinds[1:].count(ARG_POS)
+        if n_values != 0:
+            self.fail('Too many positional arguments for "ParamSpec"', s)
 
         default: Type = AnyType(TypeOfAny.from_omitted_generics)
+        for param_value, param_name in zip(
+            call.args[1 + n_values :], call.arg_names[1 + n_values :]
+        ):
+            if param_name == "default":
+                tv_arg = self.get_typevarlike_argument(
+                    "ParamSpec",
+                    param_name,
+                    param_value,
+                    s,
+                    allow_unbound_tvars=True,
+                    allow_param_spec_literals=True,
+                    report_invalid_typevar_arg=False,
+                )
+                default = tv_arg or AnyType(TypeOfAny.from_error)
+                if isinstance(tv_arg, Parameters):
+                    for i, arg_type in enumerate(tv_arg.arg_types):
+                        typ = get_proper_type(arg_type)
+                        if isinstance(typ, AnyType) and typ.is_from_error:
+                            self.fail(
+                                f"Argument {i} of ParamSpec default must be a type", param_value
+                            )
+                elif (
+                    isinstance(default, AnyType)
+                    and default.is_from_error
+                    or not isinstance(default, (AnyType, UnboundType))
+                ):
+                    self.fail(
+                        "The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec",
+                        param_value,
+                    )
+                    default = AnyType(TypeOfAny.from_error)
+            else:
+                # ParamSpec is different from a regular TypeVar:
+                # arguments are not semantically valid. But, allowed in runtime.
+                # So, we need to warn users about possible invalid usage.
+                self.fail(
+                    "The variance and bound arguments to ParamSpec do not have defined semantics yet",
+                    s,
+                )
 
         # PEP 612 reserves the right to define bound, covariant and contravariant arguments to
         # ParamSpec in a later PEP. If and when that happens, we should do something
@@ -4256,10 +4326,32 @@ def process_typevartuple_declaration(self, s: AssignmentStmt) -> bool:
         if not call:
             return False
 
-        if len(call.args) > 1:
-            self.fail("Only the first argument to TypeVarTuple has defined semantics", s)
+        n_values = call.arg_kinds[1:].count(ARG_POS)
+        if n_values != 0:
+            self.fail('Too many positional arguments for "TypeVarTuple"', s)
 
         default: Type = AnyType(TypeOfAny.from_omitted_generics)
+        for param_value, param_name in zip(
+            call.args[1 + n_values :], call.arg_names[1 + n_values :]
+        ):
+            if param_name == "default":
+                tv_arg = self.get_typevarlike_argument(
+                    "TypeVarTuple",
+                    param_name,
+                    param_value,
+                    s,
+                    allow_unbound_tvars=True,
+                    report_invalid_typevar_arg=False,
+                )
+                default = tv_arg or AnyType(TypeOfAny.from_error)
+                if not isinstance(default, UnpackType):
+                    self.fail(
+                        "The default argument to TypeVarTuple must be an Unpacked tuple",
+                        param_value,
+                    )
+                    default = AnyType(TypeOfAny.from_error)
+            else:
+                self.fail(f'Unexpected keyword argument "{param_name}" for "TypeVarTuple"', s)
 
         if not self.incomplete_feature_enabled(TYPE_VAR_TUPLE, s):
             return False
@@ -6359,6 +6451,8 @@ def expr_to_analyzed_type(
         report_invalid_types: bool = True,
         allow_placeholder: bool = False,
         allow_type_any: bool = False,
+        allow_unbound_tvars: bool = False,
+        allow_param_spec_literals: bool = False,
     ) -> Type | None:
         if isinstance(expr, CallExpr):
             # This is a legacy syntax intended mostly for Python 2, we keep it for
@@ -6387,6 +6481,8 @@ def expr_to_analyzed_type(
             report_invalid_types=report_invalid_types,
             allow_placeholder=allow_placeholder,
             allow_type_any=allow_type_any,
+            allow_unbound_tvars=allow_unbound_tvars,
+            allow_param_spec_literals=allow_param_spec_literals,
         )
 
     def analyze_type_expr(self, expr: Expression) -> None:

mypy/types.py

@@ -3049,6 +3049,8 @@ def visit_type_var(self, t: TypeVarType) -> str:
             s = f"{t.name}`{t.id}"
         if self.id_mapper and t.upper_bound:
             s += f"(upper_bound={t.upper_bound.accept(self)})"
+        if t.has_default():
+            s += f" = {t.default.accept(self)}"
         return s
 
     def visit_param_spec(self, t: ParamSpecType) -> str:
@@ -3064,6 +3066,8 @@ def visit_param_spec(self, t: ParamSpecType) -> str:
             s += f"{t.name_with_suffix()}`{t.id}"
         if t.prefix.arg_types:
             s += "]"
+        if t.has_default():
+            s += f" = {t.default.accept(self)}"
         return s
 
     def visit_parameters(self, t: Parameters) -> str:
@@ -3102,6 +3106,8 @@ def visit_type_var_tuple(self, t: TypeVarTupleType) -> str:
         else:
             # Named type variable type.
             s = f"{t.name}`{t.id}"
+        if t.has_default():
+            s += f" = {t.default.accept(self)}"
         return s
 
     def visit_callable_type(self, t: CallableType) -> str:
@@ -3138,6 +3144,8 @@ def visit_callable_type(self, t: CallableType) -> str:
             if s:
                 s += ", "
             s += f"*{n}.args, **{n}.kwargs"
+            if param_spec.has_default():
+                s += f" = {param_spec.default.accept(self)}"
 
         s = f"({s})"
 
@@ -3156,12 +3164,18 @@ def visit_callable_type(self, t: CallableType) -> str:
                         vals = f"({', '.join(val.accept(self) for val in var.values)})"
                         vs.append(f"{var.name} in {vals}")
                     elif not is_named_instance(var.upper_bound, "builtins.object"):
-                        vs.append(f"{var.name} <: {var.upper_bound.accept(self)}")
+                        vs.append(
+                            f"{var.name} <: {var.upper_bound.accept(self)}{f' = {var.default.accept(self)}' if var.has_default() else ''}"
+                        )
                     else:
-                        vs.append(var.name)
+                        vs.append(
+                            f"{var.name}{f' = {var.default.accept(self)}' if var.has_default()  else ''}"
+                        )
                 else:
-                    # For other TypeVarLikeTypes, just use the name
-                    vs.append(var.name)
+                    # For other TypeVarLikeTypes, use the name and default
+                    vs.append(
+                        f"{var.name}{f' = {var.default.accept(self)}' if var.has_default() else ''}"
+                    )
             s = f"[{', '.join(vs)}] {s}"
 
         return f"def {s}"

test-data/unit/check-parameter-specification.test

@@ -6,11 +6,11 @@ P = ParamSpec('P')
 [case testInvalidParamSpecDefinitions]
 from typing import ParamSpec
 
-P1 = ParamSpec("P1", covariant=True)  # E: Only the first argument to ParamSpec has defined semantics
-P2 = ParamSpec("P2", contravariant=True)  # E: Only the first argument to ParamSpec has defined semantics
-P3 = ParamSpec("P3", bound=int)  # E: Only the first argument to ParamSpec has defined semantics
-P4 = ParamSpec("P4", int, str)  # E: Only the first argument to ParamSpec has defined semantics
-P5 = ParamSpec("P5", covariant=True, bound=int)  # E: Only the first argument to ParamSpec has defined semantics
+P1 = ParamSpec("P1", covariant=True)  # E: The variance and bound arguments to ParamSpec do not have defined semantics yet
+P2 = ParamSpec("P2", contravariant=True)  # E: The variance and bound arguments to ParamSpec do not have defined semantics yet
+P3 = ParamSpec("P3", bound=int)  # E: The variance and bound arguments to ParamSpec do not have defined semantics yet
+P4 = ParamSpec("P4", int, str)  # E: Too many positional arguments for "ParamSpec"
+P5 = ParamSpec("P5", covariant=True, bound=int)  # E: The variance and bound arguments to ParamSpec do not have defined semantics yet
 [builtins fixtures/paramspec.pyi]
 
 [case testParamSpecLocations]

test-data/unit/check-typevar-defaults.test

@@ -0,0 +1,74 @@
+[case testTypeVarDefaultsBasic]
+import builtins
+from typing import Generic, TypeVar, ParamSpec, Callable, Tuple, List
+from typing_extensions import TypeVarTuple, Unpack
+
+T1 = TypeVar("T1", default=int)
+P1 = ParamSpec("P1", default=[int, str])
+Ts1 = TypeVarTuple("Ts1", default=Unpack[Tuple[int, str]])
+
+def f1(a: T1) -> List[T1]: ...
+reveal_type(f1)  # N: Revealed type is "def [T1 = builtins.int] (a: T1`-1 = builtins.int) -> builtins.list[T1`-1 = builtins.int]"
+
+def f2(a: Callable[P1, None] ) -> Callable[P1, None]: ...
+reveal_type(f2)  # N: Revealed type is "def [P1 = [builtins.int, builtins.str]] (a: def (*P1.args, **P1.kwargs)) -> def (*P1.args, **P1.kwargs)"
+
+def f3(a: Tuple[Unpack[Ts1]]) -> Tuple[Unpack[Ts1]]: ...
+reveal_type(f3)  # N: Revealed type is "def [Ts1 = Unpack[Tuple[builtins.int, builtins.str]]] (a: Tuple[Unpack[Ts1`-1 = Unpack[Tuple[builtins.int, builtins.str]]]]) -> Tuple[Unpack[Ts1`-1 = Unpack[Tuple[builtins.int, builtins.str]]]]"
+
+
+class ClassA1(Generic[T1]): ...
+class ClassA2(Generic[P1]): ...
+class ClassA3(Generic[Unpack[Ts1]]): ...
+
+reveal_type(ClassA1)  # N: Revealed type is "def [T1 = builtins.int] () -> __main__.ClassA1[T1`1 = builtins.int]"
+reveal_type(ClassA2)  # N: Revealed type is "def [P1 = [builtins.int, builtins.str]] () -> __main__.ClassA2[P1`1 = [builtins.int, builtins.str]]"
+reveal_type(ClassA3)  # N: Revealed type is "def [Ts1 = Unpack[Tuple[builtins.int, builtins.str]]] () -> __main__.ClassA3[Unpack[Ts1`1 = Unpack[Tuple[builtins.int, builtins.str]]]]"
+[builtins fixtures/tuple.pyi]
+
+[case testTypeVarDefaultsValid]
+from typing import TypeVar, ParamSpec, Any, List, Tuple
+from typing_extensions import TypeVarTuple, Unpack
+
+S0 = TypeVar("S0")
+S1 = TypeVar("S1", bound=int)
+
+P0 = ParamSpec("P0")
+Ts0 = TypeVarTuple("Ts0")
+
+T1 = TypeVar("T1", default=int)
+T2 = TypeVar("T2", bound=float, default=int)
+T3 = TypeVar("T3", bound=List[Any], default=List[int])
+T4 = TypeVar("T4", int, str, default=int)
+T5 = TypeVar("T5", default=S0)
+T6 = TypeVar("T6", bound=float, default=S1)
+# T7 = TypeVar("T7", bound=List[Any], default=List[S0])  # TODO
+
+P1 = ParamSpec("P1", default=[])
+P2 = ParamSpec("P2", default=...)
+P3 = ParamSpec("P3", default=[int, str])
+P4 = ParamSpec("P4", default=P0)
+
+Ts1 = TypeVarTuple("Ts1", default=Unpack[Tuple[int]])
+Ts2 = TypeVarTuple("Ts2", default=Unpack[Tuple[int, ...]])
+# Ts3 = TypeVarTuple("Ts3", default=Unpack[Ts0])  # TODO
+[builtins fixtures/tuple.pyi]
+
+[case testTypeVarDefaultsInvalid]
+from typing import TypeVar, ParamSpec, Tuple
+from typing_extensions import TypeVarTuple, Unpack
+
+T1 = TypeVar("T1", default=2)  # E: TypeVar "default" must be a type
+T2 = TypeVar("T2", default=[int, str])  # E: Bracketed expression "[...]" is not valid as a type \
+                                        # N: Did you mean "List[...]"? \
+                                        # E: TypeVar "default" must be a type
+
+P1 = ParamSpec("P1", default=int)  # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec
+P2 = ParamSpec("P2", default=2)  # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec
+P3 = ParamSpec("P3", default=(2, int))  # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec
+P4 = ParamSpec("P4", default=[2, int])  # E: Argument 0 of ParamSpec default must be a type
+
+Ts1 = TypeVarTuple("Ts1", default=2) # E: The default argument to TypeVarTuple must be an Unpacked tuple
+Ts2 = TypeVarTuple("Ts2", default=int)  # E: The default argument to TypeVarTuple must be an Unpacked tuple
+Ts3 = TypeVarTuple("Ts3", default=Tuple[int])  # E: The default argument to TypeVarTuple must be an Unpacked tuple
+[builtins fixtures/tuple.pyi]

test-data/unit/semanal-errors.test

@@ -1465,8 +1465,9 @@ TVariadic2 = TypeVarTuple('TVariadic2')
 TP = TypeVarTuple('?')  # E: String argument 1 "?" to TypeVarTuple(...) does not match variable name "TP"
 TP2: int = TypeVarTuple('TP2')  # E: Cannot declare the type of a TypeVar or similar construct
 TP3 = TypeVarTuple()  # E: Too few arguments for TypeVarTuple()
-TP4 = TypeVarTuple('TP4', 'TP4')  # E: Only the first argument to TypeVarTuple has defined semantics
+TP4 = TypeVarTuple('TP4', 'TP4')  # E: Too many positional arguments for "TypeVarTuple"
 TP5 = TypeVarTuple(t='TP5')  # E: TypeVarTuple() expects a string literal as first argument
+TP6 = TypeVarTuple('TP6', bound=int)  # E: Unexpected keyword argument "bound" for "TypeVarTuple"
 
 x: TVariadic  # E: TypeVarTuple "TVariadic" is unbound
 y: Unpack[TVariadic]  # E: TypeVarTuple "TVariadic" is unbound