From 973539105c231890edc080d6dc22054d17e33cc2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 3 Jan 2025 15:23:14 +0000 Subject: [PATCH 01/80] WIP some initial prototyping --- mypy/main.py | 8 ++++++++ mypy/options.py | 3 +++ 2 files changed, 11 insertions(+) diff --git a/mypy/main.py b/mypy/main.py index 77d8cefe9866..155d53212014 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -860,6 +860,14 @@ def add_invertible_flag( group=strictness_group, ) + add_invertible_flag( + "--allow-redefinition2", + default=False, + strict_flag=False, + help="Allow variable redefinition with a new type (including conditional)", + group=strictness_group, + ) + add_invertible_flag( "--no-implicit-reexport", default=True, diff --git a/mypy/options.py b/mypy/options.py index c1047657dd77..59096bb77354 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -219,6 +219,9 @@ def __init__(self) -> None: # and the same nesting level as the initialization self.allow_redefinition = False + # TODO + self.allow_redefinition2 = False + # Prohibit equality, identity, and container checks for non-overlapping types. # This makes 1 == '1', 1 in ['1'], and 1 is '1' errors. self.strict_equality = False From 946d4d5771029f1ce76b122271e853d47bf60286 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 28 Jan 2025 11:39:59 +0000 Subject: [PATCH 02/80] WIP add failing test case --- test-data/unit/check-redefine2.test | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 test-data/unit/check-redefine2.test diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test new file mode 100644 index 000000000000..554ec42f26d1 --- /dev/null +++ b/test-data/unit/check-redefine2.test @@ -0,0 +1,9 @@ +-- Test cases for the redefinition of variable with a different type (new version). + +[case testRedefine2LocalWithDifferentType] +# flags: --allow-redefinition2 +def f() -> None: + x = 0 + reveal_type(x) # N: Revealed type is "builtins.int" + x = '' + reveal_type(x) # N: Revealed type is "builtins.str" From 7f7e25f2e3b4898a27756a9ccad2ee8f54132cf2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 29 Jan 2025 11:25:41 +0000 Subject: [PATCH 03/80] WIP minimal support for merging control flow --- mypy/checker.py | 15 ++++++++++++++- test-data/unit/check-redefine2.test | 26 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index b8d5bbd4fa2d..106e9a614025 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3286,7 +3286,9 @@ def check_assignment( lvalue_type = make_optional_type(lvalue_type) self.set_inferred_type(lvalue.node, lvalue, lvalue_type) - rvalue_type = self.check_simple_assignment(lvalue_type, rvalue, context=rvalue) + rvalue_type = self.check_simple_assignment( + lvalue_type, rvalue, context=rvalue, inferred=inferred, lvalue=lvalue) + inferred = None # Special case: only non-abstract non-protocol classes can be assigned to # variables with explicit type Type[A], where A is protocol or abstract. @@ -4284,6 +4286,8 @@ def check_lvalue(self, lvalue: Lvalue) -> tuple[Type | None, IndexExpr | None, V self.store_type(lvalue, lvalue_type) elif isinstance(lvalue, NameExpr): lvalue_type = self.expr_checker.analyze_ref_expr(lvalue, lvalue=True) + if isinstance(lvalue.node, Var) and lvalue.node.is_inferred and self.options.allow_redefinition2: + inferred = lvalue.node self.store_type(lvalue, lvalue_type) elif isinstance(lvalue, (TupleExpr, ListExpr)): types = [ @@ -4484,6 +4488,8 @@ def check_simple_assignment( rvalue_name: str = "expression", *, notes: list[str] | None = None, + lvalue: Expression | None = None, + inferred: Var| None = None, ) -> Type: if self.is_stub and isinstance(rvalue, EllipsisExpr): # '...' is always a valid initializer in a stub. @@ -4495,6 +4501,13 @@ def check_simple_assignment( rvalue_type = self.expr_checker.accept( rvalue, lvalue_type, always_allow_any=always_allow_any ) + if inferred is not None and lvalue is not None: + old_lvalue = lvalue_type + lvalue_type = make_simplified_union([inferred.type, + remove_instance_last_known_values(rvalue_type)]) + # TODO: Do we really need to pass lvalue? + self.set_inferred_type(inferred, lvalue, lvalue_type) + self.binder.put(lvalue, rvalue_type) if ( isinstance(get_proper_type(lvalue_type), UnionType) # Skip literal types, as they have special logic (for better errors). diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 554ec42f26d1..539ba1d4c9a7 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -7,3 +7,29 @@ def f() -> None: reveal_type(x) # N: Revealed type is "builtins.int" x = '' reveal_type(x) # N: Revealed type is "builtins.str" + +[case testRedefine2ConditionalLocalWithDifferentType] +# flags: --allow-redefinition2 +def f() -> None: + if int(): + x = 0 + reveal_type(x) # N: Revealed type is "builtins.int" + else: + x = '' + reveal_type(x) # N: Revealed type is "builtins.str" + +[case testRedefine2MergeConditionalLocal1] +# flags: --allow-redefinition2 +def f() -> None: + if int(): + x = 0 + else: + x = '' + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +def g() -> None: + if int(): + x = 0 + else: + x = None + reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" From 0c9f04928968ce25ab37a58b8795d0fba5b156bc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 29 Jan 2025 11:59:40 +0000 Subject: [PATCH 04/80] Add globals test case --- test-data/unit/check-redefine2.test | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 539ba1d4c9a7..18af165225bb 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -33,3 +33,16 @@ def g() -> None: else: x = None reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" + +[case testRedefine2GlobalVariableSimple] +# flags: --allow-redefinition2 +if int(): + x = 0 + reveal_type(x) # N: Revealed type is "builtins.int" +else: + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +def f() -> None: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" From 99355372655a65641c34d275d23fe58b24f0b752 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 29 Jan 2025 12:01:07 +0000 Subject: [PATCH 05/80] Add class body test case --- test-data/unit/check-redefine2.test | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 18af165225bb..bbdd69e66b80 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -46,3 +46,16 @@ reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" def f() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +[case testRedefine2ClassBody] +# flags: --allow-redefinition2 +class C: + if int(): + x = 0 + reveal_type(x) # N: Revealed type is "builtins.int" + else: + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +reveal_type(C.x) # N: Revealed type is "Union[builtins.int, builtins.str]" From fd12f85945ae8b1c433da8caa1c464b7e76b0a71 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 29 Jan 2025 12:04:22 +0000 Subject: [PATCH 06/80] Require --local-partial-types --- mypy/main.py | 3 +++ test-data/unit/check-redefine2.test | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 155d53212014..c4cedd31368a 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -93,6 +93,9 @@ def main( stdout, stderr, options.hide_error_codes, hide_success=bool(options.output) ) + if options.allow_redefinition2 and not options.local_partial_types: + fail("error: --local-partial-types must be used if using --allow-redefinition2", stderr, options) + if options.install_types and (stdout is not sys.stdout or stderr is not sys.stderr): # Since --install-types performs user input, we want regular stdout and stderr. fail("error: --install-types not supported in this mode of running mypy", stderr, options) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index bbdd69e66b80..80aed3bbf380 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -1,7 +1,7 @@ -- Test cases for the redefinition of variable with a different type (new version). [case testRedefine2LocalWithDifferentType] -# flags: --allow-redefinition2 +# flags: --allow-redefinition2 --local-partial-types def f() -> None: x = 0 reveal_type(x) # N: Revealed type is "builtins.int" @@ -9,7 +9,7 @@ def f() -> None: reveal_type(x) # N: Revealed type is "builtins.str" [case testRedefine2ConditionalLocalWithDifferentType] -# flags: --allow-redefinition2 +# flags: --allow-redefinition2 --local-partial-types def f() -> None: if int(): x = 0 @@ -19,7 +19,7 @@ def f() -> None: reveal_type(x) # N: Revealed type is "builtins.str" [case testRedefine2MergeConditionalLocal1] -# flags: --allow-redefinition2 +# flags: --allow-redefinition2 --local-partial-types def f() -> None: if int(): x = 0 @@ -35,7 +35,7 @@ def g() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" [case testRedefine2GlobalVariableSimple] -# flags: --allow-redefinition2 +# flags: --allow-redefinition2 --local-partial-types if int(): x = 0 reveal_type(x) # N: Revealed type is "builtins.int" @@ -48,7 +48,7 @@ def f() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" [case testRedefine2ClassBody] -# flags: --allow-redefinition2 +# flags: --allow-redefinition2 --local-partial-types class C: if int(): x = 0 From b489b6a47f92b0abc6b14f4dd3a9a06d0b702061 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 29 Jan 2025 13:16:21 +0000 Subject: [PATCH 07/80] Fix optional types --- mypy/checker.py | 21 +++++++----- mypy/checkexpr.py | 2 +- test-data/unit/check-redefine2.test | 51 +++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 9 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 106e9a614025..b9d1944db1e7 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3220,7 +3220,7 @@ def check_assignment( return var = lvalue_type.var - if is_valid_inferred_type(rvalue_type, is_lvalue_final=var.is_final): + if is_valid_inferred_type(rvalue_type, self.options, is_lvalue_final=var.is_final): partial_types = self.find_partial_types(var) if partial_types is not None: if not self.current_node_deferred: @@ -3402,7 +3402,7 @@ def try_infer_partial_generic_type_from_assignment( rvalue_type = self.expr_checker.accept(rvalue) rvalue_type = get_proper_type(rvalue_type) if isinstance(rvalue_type, Instance): - if rvalue_type.type == typ.type and is_valid_inferred_type(rvalue_type): + if rvalue_type.type == typ.type and is_valid_inferred_type(rvalue_type, self.options): var.type = rvalue_type del partial_types[var] elif isinstance(rvalue_type, AnyType): @@ -4328,7 +4328,9 @@ def infer_variable_type( if isinstance(init_type, DeletedType): self.msg.deleted_as_rvalue(init_type, context) elif ( - not is_valid_inferred_type(init_type, is_lvalue_final=name.is_final) + not is_valid_inferred_type(init_type, self.options, + is_lvalue_final=name.is_final, + is_lvalue_member=isinstance(lvalue, MemberExpr)) and not self.no_partial_types ): # We cannot use the type of the initialization expression for full type @@ -4358,7 +4360,7 @@ def infer_variable_type( def infer_partial_type(self, name: Var, lvalue: Lvalue, init_type: Type) -> bool: init_type = get_proper_type(init_type) - if isinstance(init_type, NoneType): + if isinstance(init_type, NoneType) and (isinstance(lvalue, MemberExpr) or not self.options.allow_redefinition2): partial_type = PartialType(None, name) elif isinstance(init_type, Instance): fullname = init_type.type.fullname @@ -4527,7 +4529,7 @@ def check_simple_assignment( not local_errors.has_new_errors() # Skip Any type, since it is special cased in binder. and not isinstance(get_proper_type(alt_rvalue_type), AnyType) - and is_valid_inferred_type(alt_rvalue_type) + and is_valid_inferred_type(alt_rvalue_type, self.options) and is_proper_subtype(alt_rvalue_type, rvalue_type) ): rvalue_type = alt_rvalue_type @@ -4737,7 +4739,7 @@ def try_infer_partial_type_from_indexed_assignment( key_type = self.expr_checker.accept(lvalue.index) value_type = self.expr_checker.accept(rvalue) if ( - is_valid_inferred_type(key_type) + is_valid_inferred_type(key_type, self.options) and is_valid_inferred_type(value_type) and not self.current_node_deferred and not ( @@ -8486,7 +8488,10 @@ def _find_inplace_method(inst: Instance, method: str, operator: str) -> str | No return None -def is_valid_inferred_type(typ: Type, is_lvalue_final: bool = False) -> bool: +def is_valid_inferred_type(typ: Type, + options: Options, + is_lvalue_final: bool = False, + is_lvalue_member: bool = False) -> bool: """Is an inferred type valid and needs no further refinement? Examples of invalid types include the None type (when we are not assigning @@ -8505,7 +8510,7 @@ def is_valid_inferred_type(typ: Type, is_lvalue_final: bool = False) -> bool: # type could either be NoneType or an Optional type, depending on # the context. This resolution happens in leave_partial_types when # we pop a partial types scope. - return is_lvalue_final + return is_lvalue_final or (not is_lvalue_member and options.allow_redefinition2) elif isinstance(proper_type, UninhabitedType): return False return not typ.accept(InvalidInferredTypes()) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 1017009ce7ab..03ad80506ebd 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1176,7 +1176,7 @@ def try_infer_partial_value_type_from_call( and e.arg_kinds == [ARG_POS] ): item_type = self.accept(e.args[0]) - if mypy.checker.is_valid_inferred_type(item_type): + if mypy.checker.is_valid_inferred_type(item_type, self.chk.options): return self.chk.named_generic_type(typename, [item_type]) elif ( typename in self.container_args diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 80aed3bbf380..cd15e246a19c 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -59,3 +59,54 @@ class C: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" reveal_type(C.x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +[case testRedefine2OptionalTypesSimple] +# flags: --allow-redefinition2 --local-partial-types +def f1() -> None: + x = None + if int(): + x = "" + reveal_type(x) # N: Revealed type is "Union[None, builtins.str]" + +def f2() -> None: + if int(): + x = None + elif int(): + x = "" + else: + x = 1 + reveal_type(x) # N: Revealed type is "Union[None, builtins.str, builtins.int]" + +def f3() -> None: + if int(): + x = None + else: + x = "" + reveal_type(x) # N: Revealed type is "Union[None, builtins.str]" + +def f4() -> None: + x = None + reveal_type(x) # N: Revealed type is "None" + +y = None +if int(): + y = 1 +reveal_type(y) # N: Revealed type is "Union[None, builtins.int]" + +if int(): + z = None +elif int(): + z = 1 +else: + z = "" +reveal_type(z) # N: Revealed type is "Union[None, builtins.int, builtins.str]" + +[case testRedefine2OptionalTypeForInstanceVariabls] +# flags: --allow-redefinition2 --local-partial-types +class C: + def __init__(self) -> None: + self.x = None + if int(): + self.x = 1 + reveal_type(self.x) # N: Revealed type is "builtins.int" + reveal_type(self.x) # N: Revealed type is "Union[builtins.int, None]" From d2b75580e6d54d79661c949a32debffe64976cfc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 29 Jan 2025 13:34:29 +0000 Subject: [PATCH 08/80] Add partial type test cases --- test-data/unit/check-redefine2.test | 35 +++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index cd15e246a19c..8eb526294cb1 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -110,3 +110,38 @@ class C: self.x = 1 reveal_type(self.x) # N: Revealed type is "builtins.int" reveal_type(self.x) # N: Revealed type is "Union[builtins.int, None]" + +[case testRedefine2PartialGenericTypes] +# flags: --allow-redefinition2 --local-partial-types +def f1() -> None: + a = [] + a.append(1) + reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + +def f2() -> None: + a = [] + a.append(1) + reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + # Type context prevents type refinement + a = [""] # E: List item 0 has incompatible type "str"; expected "int" + reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + +def f3() -> None: + if int(): + a = [] + a.append(1) + reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + else: + b = [""] + a = b + reveal_type(a) # N: Revealed type is "builtins.list[builtins.str]" + reveal_type(a) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.str]]" + +def f4() -> None: + a = [] + a.append(1) + reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + b = [""] + a = b + reveal_type(a) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/list.pyi] From 63bf0af64d0ff93109f3cac4fb22f00132f946be Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 29 Jan 2025 13:38:54 +0000 Subject: [PATCH 09/80] Pass options consistently --- mypy/checker.py | 2 +- mypy/checkexpr.py | 6 +++--- mypy/plugins/functools.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index b9d1944db1e7..52a1c530b93f 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4740,7 +4740,7 @@ def try_infer_partial_type_from_indexed_assignment( value_type = self.expr_checker.accept(rvalue) if ( is_valid_inferred_type(key_type, self.options) - and is_valid_inferred_type(value_type) + and is_valid_inferred_type(value_type, self.options) and not self.current_node_deferred and not ( typename == "collections.defaultdict" diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 03ad80506ebd..cc138e571021 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1136,7 +1136,7 @@ def try_infer_partial_type(self, e: CallExpr) -> None: if value_type is not None: # Infer key type. key_type = self.accept(index) - if mypy.checker.is_valid_inferred_type(key_type): + if mypy.checker.is_valid_inferred_type(key_type, self.chk.options): # Store inferred partial type. assert partial_type.type is not None typename = partial_type.type.fullname @@ -1188,7 +1188,7 @@ def try_infer_partial_value_type_from_call( arg_typename = arg_type.type.fullname if arg_typename in self.container_args[typename][methodname]: if all( - mypy.checker.is_valid_inferred_type(item_type) + mypy.checker.is_valid_inferred_type(item_type, self.chk.options) for item_type in arg_type.args ): return self.chk.named_generic_type(typename, list(arg_type.args)) @@ -5830,7 +5830,7 @@ def visit_conditional_expr(self, e: ConditionalExpr, allow_none_return: bool = F else_map, e.else_expr, context=ctx, allow_none_return=allow_none_return ) - if not mypy.checker.is_valid_inferred_type(if_type): + if not mypy.checker.is_valid_inferred_type(if_type, self.chk.options): # Analyze the right branch disregarding the left branch. else_type = full_context_else_type # we want to keep the narrowest value of else_type for union'ing the branches diff --git a/mypy/plugins/functools.py b/mypy/plugins/functools.py index c435dde7fde7..25a8c83007ba 100644 --- a/mypy/plugins/functools.py +++ b/mypy/plugins/functools.py @@ -276,7 +276,7 @@ def handle_partial_with_callee(ctx: mypy.plugin.FunctionContext, callee: Type) - for i, actuals in enumerate(formal_to_actual): if len(bound.arg_types) == len(fn_type.arg_types): arg_type = bound.arg_types[i] - if not mypy.checker.is_valid_inferred_type(arg_type): + if not mypy.checker.is_valid_inferred_type(arg_type, ctx.api.options): arg_type = fn_type.arg_types[i] # bit of a hack else: # TODO: I assume that bound and fn_type have the same arguments. It appears this isn't @@ -301,7 +301,7 @@ def handle_partial_with_callee(ctx: mypy.plugin.FunctionContext, callee: Type) - partial_names.append(fn_type.arg_names[i]) ret_type = bound.ret_type - if not mypy.checker.is_valid_inferred_type(ret_type): + if not mypy.checker.is_valid_inferred_type(ret_type, ctx.api.options): ret_type = fn_type.ret_type # same kind of hack as above partially_applied = fn_type.copy_modified( From ec0265440d6d273db8311c07648b953b132f33e2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 29 Jan 2025 14:33:23 +0000 Subject: [PATCH 10/80] Fix interaction with Final --- mypy/checker.py | 7 +++++-- test-data/unit/check-redefine2.test | 9 +++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 52a1c530b93f..fed9be8774ae 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4505,8 +4505,11 @@ def check_simple_assignment( ) if inferred is not None and lvalue is not None: old_lvalue = lvalue_type - lvalue_type = make_simplified_union([inferred.type, - remove_instance_last_known_values(rvalue_type)]) + if not inferred.is_final: + new_inferred = remove_instance_last_known_values(rvalue_type) + else: + new_inferred = rvalue_type + lvalue_type = make_simplified_union([inferred.type, new_inferred]) # TODO: Do we really need to pass lvalue? self.set_inferred_type(inferred, lvalue, lvalue_type) self.binder.put(lvalue, rvalue_type) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 8eb526294cb1..0c2864257850 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -145,3 +145,12 @@ def f4() -> None: a = b reveal_type(a) # N: Revealed type is "builtins.list[builtins.str]" [builtins fixtures/list.pyi] + +[case testRedefine2FinalLiteral] +# flags: --allow-redefinition2 --local-partial-types +from typing_extensions import Final, Literal + +x: Final = "foo" +reveal_type(x) # N: Revealed type is "Literal['foo']?" +a: Literal["foo"] = x +[builtins fixtures/tuple.pyi] From 429085734635817d6d6a4fe27762febf20d5c593 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 29 Jan 2025 15:43:59 +0000 Subject: [PATCH 11/80] Add annotated variable test case --- test-data/unit/check-redefine2.test | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 0c2864257850..da3485c4bf7b 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -154,3 +154,31 @@ x: Final = "foo" reveal_type(x) # N: Revealed type is "Literal['foo']?" a: Literal["foo"] = x [builtins fixtures/tuple.pyi] + +[case testRedefine2AnnotatedVariable] +# flags: --allow-redefinition2 --local-partial-types +from typing import Optional + +def f1() -> None: + x: int = 0 + if int(): + x = "" # E: Incompatible types in assignment (expression has type "str", variable has type "int") + reveal_type(x) # N: Revealed type is "builtins.int" + reveal_type(x) # N: Revealed type is "builtins.int" + +def f2(x: Optional[str]) -> None: + if x is not None: + reveal_type(x) # N: Revealed type is "builtins.str" + else: + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" + +class C: + x: Optional[str] + + def f(self) -> None: + if self.x is not None: + reveal_type(self.x) # N: Revealed type is "builtins.str" + else: + self.x = "" + reveal_type(self.x) # N: Revealed type is "builtins.str" From 5c8ab17777d2fcfc2817cc641923b806b74f5519 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 29 Jan 2025 15:49:09 +0000 Subject: [PATCH 12/80] Add test --- test-data/unit/check-redefine2.test | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index da3485c4bf7b..afa7786fae09 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -182,3 +182,59 @@ class C: else: self.x = "" reveal_type(self.x) # N: Revealed type is "builtins.str" + +[case testRedefine2AnyType] +# flags: --allow-redefinition2 --local-partial-types +def a(): pass + +def f1() -> None: + if int(): + x = "" + else: + x = a() + reveal_type(x) # N: Revealed type is "Any" + reveal_type(x) # N: Revealed type is "Union[builtins.str, Any]" + x = 1 + reveal_type(x) # N: Revealed type is "builtins.int" + +def f2() -> None: + if int(): + x = a() + else: + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "Union[Any, builtins.str]" + x = 1 + reveal_type(x) # N: Revealed type is "builtins.int" + +def f3() -> None: + x = 1 + x = a() + reveal_type(x) # N: Revealed type is "Any" + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" + +def f4() -> None: + x = a() + x = 1 + reveal_type(x) # N: Revealed type is "builtins.int" + x = a() + reveal_type(x) # N: Revealed type is "Any" + +def f5() -> None: + x = a() + if int(): + x = 1 + reveal_type(x) # N: Revealed type is "builtins.int" + elif int(): + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "Union[Any, builtins.int, builtins.str]" + +def f6() -> None: + x = a() + if int(): + x = 1 + else: + x = "" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" From c1164512993a6cf8d9a7a517ea8fef4585e2d24b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 29 Jan 2025 15:53:17 +0000 Subject: [PATCH 13/80] Add failing test --- test-data/unit/check-redefine2.test | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index afa7786fae09..33a150193f1c 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -238,3 +238,9 @@ def f6() -> None: else: x = "" reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +# TODO: declared type should take precedence +#def f7() -> None: +# x: int +# x = a() +# reveal_type(x) # Revealed type is "builtins.int" From 9794308bf7107fc3c11a48b127d535297ba5bd64 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 30 Jan 2025 12:02:12 +0000 Subject: [PATCH 14/80] Only use type context in assignment if inference fails without type context --- mypy/checker.py | 9 ++++++++- test-data/unit/check-redefine2.test | 23 +++++++++++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index fed9be8774ae..bfdc085bca08 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4501,8 +4501,15 @@ def check_simple_assignment( get_proper_type(lvalue_type), AnyType ) rvalue_type = self.expr_checker.accept( - rvalue, lvalue_type, always_allow_any=always_allow_any + rvalue, type_context=None, always_allow_any=always_allow_any ) + if ( + lvalue_type is not None and + not is_valid_inferred_type(rvalue_type, self.options) # TODO + ): + rvalue_type = self.expr_checker.accept( + rvalue, type_context=lvalue_type, always_allow_any=always_allow_any + ) if inferred is not None and lvalue is not None: old_lvalue = lvalue_type if not inferred.is_final: diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 33a150193f1c..a6e935d16e00 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -122,11 +122,26 @@ def f2() -> None: a = [] a.append(1) reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" - # Type context prevents type refinement - a = [""] # E: List item 0 has incompatible type "str"; expected "int" - reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + a = [""] + reveal_type(a) # N: Revealed type is "builtins.list[builtins.str]" def f3() -> None: + a = [] + a.append(1) + reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + a = [] + reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + +def f4() -> None: + a = [] + a.append(1) + reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + # Partial types are currently not supported on reassignment + a = [] + a.append("x") # E: Argument 1 to "append" of "list" has incompatible type "str"; expected "int" + reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + +def f5() -> None: if int(): a = [] a.append(1) @@ -137,7 +152,7 @@ def f3() -> None: reveal_type(a) # N: Revealed type is "builtins.list[builtins.str]" reveal_type(a) # N: Revealed type is "Union[builtins.list[builtins.int], builtins.list[builtins.str]]" -def f4() -> None: +def f6() -> None: a = [] a.append(1) reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" From a355473aacebc512a48bd59281cb82728e02bb31 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 30 Jan 2025 12:10:29 +0000 Subject: [PATCH 15/80] Always use type annotation as context --- mypy/checker.py | 8 ++++++-- test-data/unit/check-redefine2.test | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index bfdc085bca08..c99e960b8a2f 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4500,11 +4500,15 @@ def check_simple_assignment( always_allow_any = lvalue_type is not None and not isinstance( get_proper_type(lvalue_type), AnyType ) + if inferred is None: + type_context = lvalue_type + else: + type_context = None rvalue_type = self.expr_checker.accept( - rvalue, type_context=None, always_allow_any=always_allow_any + rvalue, type_context=type_context, always_allow_any=always_allow_any ) if ( - lvalue_type is not None and + lvalue_type is not None and type_context is None and not is_valid_inferred_type(rvalue_type, self.options) # TODO ): rvalue_type = self.expr_checker.accept( diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index a6e935d16e00..9073bed61f3e 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -188,6 +188,12 @@ def f2(x: Optional[str]) -> None: x = "" reveal_type(x) # N: Revealed type is "builtins.str" +def f3() -> None: + a: list[Optional[str]] = [""] + reveal_type(a) # N: Revealed type is "builtins.list[Union[builtins.str, None]]" + a = [""] + reveal_type(a) # N: Revealed type is "builtins.list[Union[builtins.str, None]]" + class C: x: Optional[str] From 8162a455a167e64e6e07c0ef653e554fc5ce120d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 31 Jan 2025 10:47:52 +0000 Subject: [PATCH 16/80] Add while loop test case --- test-data/unit/check-redefine2.test | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 9073bed61f3e..994db4745f54 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -265,3 +265,30 @@ def f6() -> None: # x: int # x = a() # reveal_type(x) # Revealed type is "builtins.int" + +[case testRedefine2WhileLoopSimple] +# flags: --allow-redefinition2 --local-partial-types +def f() -> None: + while int(): + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" + x = 0 + reveal_type(x) # N: Revealed type is "builtins.int" + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int]" + while int(): + x = None + reveal_type(x) # N: Revealed type is "None" + x = b"" + reveal_type(x) # N: Revealed type is "builtins.bytes" + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int, None, builtins.bytes]" + x = [1] + reveal_type(x) # N: Revealed type is "builtins.list[builtins.int]" + +[case testRedefine2WhileLoopOptional] +# flags: --allow-redefinition2 --local-partial-types +def f() -> None: + x = None + while int(): + if int(): + x = "" + reveal_type(x) # N: Revealed type is "Union[None, builtins.str]" From b1f75a79a8953c8a6e11e1d3b85b551da493d9ea Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 31 Jan 2025 11:44:55 +0000 Subject: [PATCH 17/80] Fix type inference in loops --- mypy/checker.py | 24 +++++++++++++++++------- test-data/unit/check-redefine2.test | 20 +++++++++++++++++++- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index c99e960b8a2f..7e4a6a9b5e30 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -316,6 +316,8 @@ class TypeChecker(NodeVisitor[None], CheckerPluginInterface): # Vars for which partial type errors are already reported # (to avoid logically duplicate errors with different error context). partial_reported: set[Var] + # Short names of Var nodes whose previous inferred type has been widened via assignment + widened_vars: list[str] globals: SymbolTable modules: dict[str, MypyFile] # Nodes that couldn't be checked because some types weren't available. We'll run @@ -384,6 +386,7 @@ def __init__( self.partial_reported = set() self.var_decl_frames = {} self.deferred_nodes = [] + self.widened_vars = [] self._type_maps = [{}] self.module_refs = set() self.pass_num = 0 @@ -523,6 +526,7 @@ def check_second_pass( return True def check_partial(self, node: DeferredNodeType | FineGrainedDeferredNodeType) -> None: + self.widened_vars = [] if isinstance(node, MypyFile): self.check_top_level(node) else: @@ -592,6 +596,7 @@ def accept_loop( # Check for potential decreases in the number of partial types so as not to stop the # iteration too early: partials_old = sum(len(pts.map) for pts in self.partial_types) + widened_old = len(self.widened_vars) # Disable error types that we cannot safely identify in intermediate iteration steps: warn_unreachable = self.options.warn_unreachable @@ -599,6 +604,7 @@ def accept_loop( self.options.warn_unreachable = False self.options.enabled_error_codes.discard(codes.REDUNDANT_EXPR) + iter = 1 while True: with self.binder.frame_context(can_skip=True, break_frame=2, continue_frame=1): if on_enter_body is not None: @@ -606,9 +612,12 @@ def accept_loop( self.accept(body) partials_new = sum(len(pts.map) for pts in self.partial_types) - if (partials_new == partials_old) and not self.binder.last_pop_changed: + widened_new = len(self.widened_vars) + if (partials_new == partials_old) and not self.binder.last_pop_changed and (widened_new == widened_old or iter > 1): break partials_old = partials_new + widened_old = widened_new + iter += 1 # If necessary, reset the modified options and make up for the postponed error checks: self.options.warn_unreachable = warn_unreachable @@ -4514,16 +4523,17 @@ def check_simple_assignment( rvalue_type = self.expr_checker.accept( rvalue, type_context=lvalue_type, always_allow_any=always_allow_any ) - if inferred is not None and lvalue is not None: - old_lvalue = lvalue_type + if inferred is not None and lvalue is not None and inferred.type is not None: if not inferred.is_final: new_inferred = remove_instance_last_known_values(rvalue_type) else: new_inferred = rvalue_type - lvalue_type = make_simplified_union([inferred.type, new_inferred]) - # TODO: Do we really need to pass lvalue? - self.set_inferred_type(inferred, lvalue, lvalue_type) - self.binder.put(lvalue, rvalue_type) + if not is_same_type(inferred.type, new_inferred): + lvalue_type = make_simplified_union([inferred.type, new_inferred]) + if not is_same_type(lvalue_type, inferred.type): + self.widened_vars.append(inferred.name) + self.set_inferred_type(inferred, lvalue, lvalue_type) + self.binder.put(lvalue, rvalue_type) if ( isinstance(get_proper_type(lvalue_type), UnionType) # Skip literal types, as they have special logic (for better errors). diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 994db4745f54..e79cacb089d1 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -286,9 +286,27 @@ def f() -> None: [case testRedefine2WhileLoopOptional] # flags: --allow-redefinition2 --local-partial-types -def f() -> None: +def f1() -> None: x = None while int(): if int(): x = "" reveal_type(x) # N: Revealed type is "Union[None, builtins.str]" + +def f2() -> None: + x = None + while int(): + reveal_type(x) # N: Revealed type is "None" \ + # N: Revealed type is "Union[None, builtins.str]" + if int(): + x = "" + reveal_type(x) # N: Revealed type is "Union[None, builtins.str]" + +[case testRedefine2WhileLoopPartialType] +# flags: --allow-redefinition2 --local-partial-types +def f1() -> None: + x = [] + while int(): + x.append(1) + reveal_type(x) # N: Revealed type is "builtins.list[builtins.int]" +[builtins fixtures/list.pyi] From 62dfbb0bf9488ad074e4d10f85ac096d2a17f39e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 5 Feb 2025 16:11:25 +0000 Subject: [PATCH 18/80] Update tests --- test-data/unit/check-redefine2.test | 36 ++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index e79cacb089d1..619745b89004 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -204,7 +204,7 @@ class C: self.x = "" reveal_type(self.x) # N: Revealed type is "builtins.str" -[case testRedefine2AnyType] +[case testRedefine2AnyType1] # flags: --allow-redefinition2 --local-partial-types def a(): pass @@ -260,11 +260,35 @@ def f6() -> None: x = "" reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" -# TODO: declared type should take precedence -#def f7() -> None: -# x: int -# x = a() -# reveal_type(x) # Revealed type is "builtins.int" +def f7() -> None: + x: int + x = a() + reveal_type(x) # N: Revealed type is "builtins.int" + +[case testRedefine2AnyType2] +# flags: --allow-redefinition2 --local-partial-types +from typing import Any + +def f1() -> None: + x: Any + x = int() + reveal_type(x) # N: Revealed type is "Any" + +def f2() -> None: + x: Any + if int(): + x = 0 + reveal_type(x) # N: Revealed type is "Any" + else: + x = "" + reveal_type(x) # N: Revealed type is "Any" + reveal_type(x) # N: Revealed type is "Any" + +def f3(x) -> None: + if int(): + x = 0 + reveal_type(x) # N: Revealed type is "Any" + reveal_type(x) # N: Revealed type is "Any" [case testRedefine2WhileLoopSimple] # flags: --allow-redefinition2 --local-partial-types From bb6a24601552da77f4d547b253c727e28e508416 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 5 Feb 2025 16:31:12 +0000 Subject: [PATCH 19/80] Remove underscore special case --- mypy/semanal.py | 2 +- test-data/unit/check-redefine2.test | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index a0cfdcce1e33..9c72779009f3 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4335,7 +4335,7 @@ def analyze_name_lvalue( else: lvalue.fullname = lvalue.name if self.is_func_scope(): - if unmangle(name) == "_": + if unmangle(name) == "_" and not self.options.allow_redefinition2: # Special case for assignment to local named '_': always infer 'Any'. typ = AnyType(TypeOfAny.special_form) self.store_declared_types(lvalue, typ) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 619745b89004..002289fff52d 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -334,3 +334,14 @@ def f1() -> None: x.append(1) reveal_type(x) # N: Revealed type is "builtins.list[builtins.int]" [builtins fixtures/list.pyi] + +[case testRedefine2Underscore] +# flags: --allow-redefinition2 --local-partial-types +def f() -> None: + if int(): + _ = 0 + reveal_type(_) # N: Revealed type is "builtins.int" + else: + _ = "" + reveal_type(_) # N: Revealed type is "builtins.str" + reveal_type(_) # N: Revealed type is "Union[builtins.int, builtins.str]" From 51c61c7c022e8c6f02b6b5d164de32e6f6a6e499 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 5 Feb 2025 16:58:31 +0000 Subject: [PATCH 20/80] Don't perform renaming when using new semantics --- mypy/build.py | 6 ++++-- test-data/unit/check-redefine2.test | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index f6272ed808cf..a43b0e7a72b5 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -2240,8 +2240,10 @@ def semantic_analysis_pass1(self) -> None: # TODO: Do this while constructing the AST? self.tree.names = SymbolTable() if not self.tree.is_stub: - # Always perform some low-key variable renaming - self.tree.accept(LimitedVariableRenameVisitor()) + if not self.options.allow_redefinition2: + # Always perform some low-key variable renaming when assignments can't + # widen inferred types + self.tree.accept(LimitedVariableRenameVisitor()) if options.allow_redefinition: # Perform more renaming across the AST to allow variable redefinitions self.tree.accept(VariableRenameVisitor()) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 002289fff52d..dd20286fef26 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -345,3 +345,27 @@ def f() -> None: _ = "" reveal_type(_) # N: Revealed type is "builtins.str" reveal_type(_) # N: Revealed type is "Union[builtins.int, builtins.str]" + +[case testRedefine2WithStatement] +# flags: --allow-redefinition2 --local-partial-types +class C: + def __enter__(self) -> int: ... + def __exit__(self, x, y, z): ... +class D: + def __enter__(self) -> str: ... + def __exit__(self, x, y, z): ... + +def f1() -> None: + with C() as x: + reveal_type(x) # N: Revealed type is "builtins.int" + with D() as x: + reveal_type(x) # N: Revealed type is "builtins.str" + +def f2() -> None: + if int(): + with C() as x: + reveal_type(x) # N: Revealed type is "builtins.int" + else: + with D() as x: + reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" From 03847958e10438771afe131bda69ad6669985083 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Feb 2025 11:00:12 +0000 Subject: [PATCH 21/80] Fix for loops --- mypy/checker.py | 1 + test-data/unit/check-redefine2.test | 66 +++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/mypy/checker.py b/mypy/checker.py index 7e4a6a9b5e30..825d16afd5bc 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3328,6 +3328,7 @@ def check_assignment( and lvalue.node.is_inferred and lvalue.node.is_index_var and lvalue_type is not None + and not self.options.allow_redefinition2 # TODO WHAT ): lvalue.node.type = remove_instance_last_known_values(lvalue_type) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index dd20286fef26..7eed0a5249c7 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -369,3 +369,69 @@ def f2() -> None: with D() as x: reveal_type(x) # N: Revealed type is "builtins.str" reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +[case testRedefine2MultipleAssignment] +# flags: --allow-redefinition2 --local-partial-types +def f1() -> None: + x, y = 1, "" + reveal_type(x) # N: Revealed type is "builtins.int" + reveal_type(y) # N: Revealed type is "builtins.str" + x, y = None, 2 + reveal_type(x) # N: Revealed type is "None" + reveal_type(y) # N: Revealed type is "builtins.int" + +def f2() -> None: + if int(): + x, y = 1, "" + reveal_type(x) # N: Revealed type is "builtins.int" + reveal_type(y) # N: Revealed type is "builtins.str" + else: + x, y = None, 2 + reveal_type(x) # N: Revealed type is "None" + reveal_type(y) # N: Revealed type is "builtins.int" + reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" + reveal_type(y) # N: Revealed type is "Union[builtins.str, builtins.int]" + +[case testRedefine2ForLoop] +# flags: --allow-redefinition2 --local-partial-types +def f1() -> None: + for x in [1]: + reveal_type(x) # N: Revealed type is "builtins.int" + for x in [""]: + reveal_type(x) # N: Revealed type is "builtins.str" + +def f2() -> None: + if int(): + for x, y in [(1, "x")]: + reveal_type(x) # N: Revealed type is "builtins.int" + reveal_type(y) # N: Revealed type is "builtins.str" + else: + for x, y in [(None, 1)]: + reveal_type(x) # N: Revealed type is "None" + reveal_type(y) # N: Revealed type is "builtins.int" + + reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" + reveal_type(y) # N: Revealed type is "Union[builtins.str, builtins.int]" +[builtins fixtures/for.pyi] + +[case testRedefine2ForStatementIndexNarrowing] +# flags: --allow-redefinition2 --local-partial-types +from typing_extensions import TypedDict + +class X(TypedDict): + hourly: int + daily: int + +x: X +for a in ("hourly", "daily"): + reveal_type(a) # N: Revealed type is "Union[Literal['hourly']?, Literal['daily']?]" + reveal_type(x[a]) # N: Revealed type is "builtins.int" + reveal_type(a.upper()) # N: Revealed type is "builtins.str" + c = a + reveal_type(c) # N: Revealed type is "builtins.str" + +b: str +for b in ("hourly", "daily"): + reveal_type(b) # N: Revealed type is "builtins.str" + reveal_type(b.upper()) # N: Revealed type is "builtins.str" +[builtins fixtures/for.pyi] From a003c83b9b72f0725c8055cd735b03ef18aba220 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Feb 2025 12:10:51 +0000 Subject: [PATCH 22/80] WIP failing tests --- test-data/unit/check-redefine2.test | 46 +++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 7eed0a5249c7..1a02046ba7f0 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -20,20 +20,26 @@ def f() -> None: [case testRedefine2MergeConditionalLocal1] # flags: --allow-redefinition2 --local-partial-types -def f() -> None: +def f1() -> None: if int(): x = 0 else: x = '' reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" -def g() -> None: +def f2() -> None: if int(): x = 0 else: x = None reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" +def f3() -> None: + if int(): + x = 0 + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" + [case testRedefine2GlobalVariableSimple] # flags: --allow-redefinition2 --local-partial-types if int(): @@ -392,7 +398,7 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" reveal_type(y) # N: Revealed type is "Union[builtins.str, builtins.int]" -[case testRedefine2ForLoop] +[case testRedefine2ForLoopBasics] # flags: --allow-redefinition2 --local-partial-types def f1() -> None: for x in [1]: @@ -412,6 +418,15 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" reveal_type(y) # N: Revealed type is "Union[builtins.str, builtins.int]" + +def l() -> list[int]: + return [] + +def f3() -> None: + x = "" + for x in l(): + reveal_type(x) # N: Revealed type is "builtins.int" + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int]" [builtins fixtures/for.pyi] [case testRedefine2ForStatementIndexNarrowing] @@ -435,3 +450,28 @@ for b in ("hourly", "daily"): reveal_type(b) # N: Revealed type is "builtins.str" reveal_type(b.upper()) # N: Revealed type is "builtins.str" [builtins fixtures/for.pyi] + +[case testRedefine2ForLoopIndexWidening] +# flags: --allow-redefinition2 --local-partial-types + +def f1() -> None: + for x in [1]: + reveal_type(x) # N: Revealed type is "builtins.int" + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "builtins.str" + +def f2() -> None: + for x in [1]: + reveal_type(x) # N: Revealed type is "builtins.int" + if int(): + break + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +def f3() -> None: + if int(): + for x in [1]: + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" From a35870e8535ff5fbad82241fe939245ffe79ba42 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Feb 2025 14:50:45 +0000 Subject: [PATCH 23/80] Add try/except test case --- test-data/unit/check-redefine2.test | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 1a02046ba7f0..28b46012dea7 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -376,6 +376,28 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "builtins.str" reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[case testRedefine2TryStatement] +# flags: --allow-redefinition2 --local-partial-types +class E(Exception): pass + +def g(): ... + +def f1() -> None: + try: + x = 1 + g() + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" + except RuntimeError as e: + reveal_type(e) # N: Revealed type is "builtins.RuntimeError" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + except E as e: + reveal_type(e) # N: Revealed type is "__main__.E" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + reveal_type(e) # N: Revealed type is "" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/exception.pyi] + [case testRedefine2MultipleAssignment] # flags: --allow-redefinition2 --local-partial-types def f1() -> None: From dea914803069a574a7f0d2b3b411c0278c3e2235 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Feb 2025 14:54:55 +0000 Subject: [PATCH 24/80] Add match statement test --- test-data/unit/check-python310.test | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 016f50552a5f..6ab438174672 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -2552,3 +2552,16 @@ def f(t: T) -> None: case T([K() as k]): reveal_type(k) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.K]" [builtins fixtures/tuple.pyi] + +[case testRedefine2MatchBasics] +# flags: --allow-redefinition2 --local-partial-types + +def f1(x: int | str | list[bytes]) -> None: + match x: + case int(): + reveal_type(x) # N: Revealed type is "builtins.int" + case str(y): + reveal_type(y) # N: Revealed type is "builtins.str" + case [y]: + reveal_type(y) # N: Revealed type is "builtins.bytes" + reveal_type(y) # N: Revealed type is "Union[builtins.str, builtins.bytes]" From 2b104dc75b872dd4057097ae8022ed581a6cb03a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Feb 2025 14:58:23 +0000 Subject: [PATCH 25/80] Add simple nested function test case --- test-data/unit/check-redefine2.test | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 28b46012dea7..a8a442e032a5 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -66,6 +66,31 @@ class C: reveal_type(C.x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[case testRedefine2NestedFunctionBasics] +# flags: --allow-redefinition2 --local-partial-types +def f1() -> None: + if int(): + x = 0 + else: + x = "" + + def nested() -> None: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +def f2() -> None: + if int(): + x = 0 + else: + x = "" + + def nested() -> None: + nonlocal x + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + x = 0 + reveal_type(x) # N: Revealed type is "builtins.int" + + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + [case testRedefine2OptionalTypesSimple] # flags: --allow-redefinition2 --local-partial-types def f1() -> None: From 6a05baf30e862f2ea497823c7fe66ac4d96a3228 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Feb 2025 14:59:35 +0000 Subject: [PATCH 26/80] Update globals tests case --- test-data/unit/check-redefine2.test | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index a8a442e032a5..f1abd882ee07 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -50,8 +50,16 @@ else: reveal_type(x) # N: Revealed type is "builtins.str" reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" -def f() -> None: +def f1() -> None: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +def f2() -> None: + global x reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + x = 0 + reveal_type(x) # N: Revealed type is "builtins.int" + +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" [case testRedefine2ClassBody] # flags: --allow-redefinition2 --local-partial-types From b58be6a053f6e6eaf84a2b5771a4d546ddc80acf Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Feb 2025 15:03:42 +0000 Subject: [PATCH 27/80] Add assignment expression test case --- test-data/unit/check-redefine2.test | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index f1abd882ee07..60df6caede42 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -99,6 +99,15 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[case testRedefine2AssignmentExpression] +# flags: --allow-redefinition2 --local-partial-types +def f() -> None: + if x := int(): + reveal_type(x) # N: Revealed type is "builtins.int" + elif x := str(): + reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + [case testRedefine2OptionalTypesSimple] # flags: --allow-redefinition2 --local-partial-types def f1() -> None: From 5b319504e1baaf9f9c8b328d1316b913d2537f33 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Feb 2025 15:53:53 +0000 Subject: [PATCH 28/80] Add lambda test case --- test-data/unit/check-redefine2.test | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 60df6caede42..c15e83d99683 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -99,6 +99,19 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[case testRedefine2LambdaBasics] +# flags: --allow-redefinition2 --local-partial-types +def f1() -> None: + x = 0 + if int(): + x = None + f = lambda: reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" + reveal_type(f) # N: Revealed type is "def () -> Union[builtins.int, None]" + if x is None: + x = "" + f = lambda: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + reveal_type(f) # N: Revealed type is "def () -> Union[builtins.int, builtins.str]" + [case testRedefine2AssignmentExpression] # flags: --allow-redefinition2 --local-partial-types def f() -> None: From bcaace17c70ab5dfc4e78ea69a3858284c34771a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Feb 2025 16:14:33 +0000 Subject: [PATCH 29/80] Tests for imports --- test-data/unit/check-redefine2.test | 35 +++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index c15e83d99683..b268448267be 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -121,6 +121,41 @@ def f() -> None: reveal_type(x) # N: Revealed type is "builtins.str" reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[case testRedefine2ImportFrom] +# flags: --allow-redefinition2 --local-partial-types +if int(): + from m import x +else: + # TODO: This could be useful to allow + from m import y as x # E: Incompatible import of "x" (imported name has type "str", local name has type "int") +reveal_type(x) # N: Revealed type is "builtins.int" + +if int(): + from m import y +else: + y = 1 +reveal_type(y) # N: Revealed type is "Union[builtins.str, builtins.int]" + +[file m.py] +x = 1 +y = "" + +[case testRedefine2Import] +# flags: --allow-redefinition2 --local-partial-types +if int(): + import m +else: + import m2 as m # E: Name "m" already defined (by an import) +m.x +m.y # E: Module has no attribute "y" + +[file m.py] +x = 1 + +[file m2.py] +y = "" +[builtins fixtures/module.pyi] + [case testRedefine2OptionalTypesSimple] # flags: --allow-redefinition2 --local-partial-types def f1() -> None: From ad901251b7cd491d9aa32b1e578f0c369fc900a8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Feb 2025 16:19:36 +0000 Subject: [PATCH 30/80] Add operator assignment test case --- test-data/unit/check-redefine2.test | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index b268448267be..bc0f1a9761e6 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -121,6 +121,18 @@ def f() -> None: reveal_type(x) # N: Revealed type is "builtins.str" reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[case testRedefine2OperatorAssignment] +# flags: --allow-redefinition2 --local-partial-types +class D: pass +class C: + def __add__(self, x: C) -> D: ... + +c = C() +if int(): + c += C() + reveal_type(c) # N: Revealed type is "__main__.D" +reveal_type(c) # N: Revealed type is "Union[__main__.C, __main__.D]" + [case testRedefine2ImportFrom] # flags: --allow-redefinition2 --local-partial-types if int(): From 106577d3c479e19d0aaccbaf91cb6b5f2e57170c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Feb 2025 16:50:46 +0000 Subject: [PATCH 31/80] WIP add break/continue tests (failing) --- test-data/unit/check-redefine2.test | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index bc0f1a9761e6..12358249b40f 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -443,6 +443,31 @@ def f1() -> None: reveal_type(x) # N: Revealed type is "builtins.list[builtins.int]" [builtins fixtures/list.pyi] +[case testRedefine2BreakAndContinue] +# flags: --allow-redefinition2 --local-partial-types +def b() -> None: + while int(): + x = "" + if int(): + x = 1 + break + reveal_type(x) # N: Revealed type is "builtins.str" + x = None + reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" + +def c() -> None: + x = 0 + while int(): + reveal_type(x) # N: Revealed type is "builtins.int" \ + # N: Revealed type is "Union[builtins.int, builtins.str, None]" + if int(): + x = "" + continue + else: + x = None + reveal_type(x) # N: Revealed type is "None" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, None]" + [case testRedefine2Underscore] # flags: --allow-redefinition2 --local-partial-types def f() -> None: From ea6f13b13fb32ead56baf65d1ffd84d3c1c53926 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Feb 2025 16:55:50 +0000 Subject: [PATCH 32/80] WIP del test (failing) --- test-data/unit/check-redefine2.test | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 12358249b40f..caae444d08d7 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -398,6 +398,32 @@ def f3(x) -> None: reveal_type(x) # N: Revealed type is "Any" reveal_type(x) # N: Revealed type is "Any" +[case tetRedefine2Del] +# flags: --allow-redefinition2 --local-partial-types +def f1() -> None: + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" + del x + reveal_type(x) # N: Revealed type is "" + x = 0 + reveal_type(x) # N: Revealed type is "builtins.int" + +def f2() -> None: + if int(): + x = 0 + del x + else: + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" + +def f3() -> None: + if int(): + x = 0 + else: + x = "" + del x + reveal_type(x) # N: Revealed type is "builtins.str" + [case testRedefine2WhileLoopSimple] # flags: --allow-redefinition2 --local-partial-types def f() -> None: From 71deb242b329ef4a3b9dbde73fcbac3d3fbfd093 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 7 Feb 2025 10:32:37 +0000 Subject: [PATCH 33/80] Add return test (failing) --- test-data/unit/check-redefine2.test | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index caae444d08d7..49c9b6d4b48b 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -469,6 +469,24 @@ def f1() -> None: reveal_type(x) # N: Revealed type is "builtins.list[builtins.int]" [builtins fixtures/list.pyi] +[case testRedefine2Return] +# flags: --allow-redefinition2 --local-partial-types +def f1() -> None: + if int(): + x = 0 + return + else: + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" + +def f2() -> None: + if int(): + x = "" + else: + x = 0 + return + reveal_type(x) # N: Revealed type is "builtins.str" + [case testRedefine2BreakAndContinue] # flags: --allow-redefinition2 --local-partial-types def b() -> None: From b3cc74a814ab79dd0b38aff7e9e986bfa6b07af8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 7 Feb 2025 11:50:07 +0000 Subject: [PATCH 34/80] Fix binder issues --- mypy/binder.py | 1 + mypy/checker.py | 4 ++++ test-data/unit/check-redefine2.test | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mypy/binder.py b/mypy/binder.py index 384bdca728b2..cb352089a32e 100644 --- a/mypy/binder.py +++ b/mypy/binder.py @@ -39,6 +39,7 @@ class CurrentType(NamedTuple): class Frame: """A Frame represents a specific point in the execution of a program. + It carries information about the current types of expressions at that point, arising either from assignments to those expressions or the result of isinstance checks and other type narrowing diff --git a/mypy/checker.py b/mypy/checker.py index 825d16afd5bc..ce1ca5f215d1 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4367,6 +4367,8 @@ def infer_variable_type( init_type = strip_type(init_type) self.set_inferred_type(name, lvalue, init_type) + if self.options.allow_redefinition2: + self.binder.assign_type(lvalue, init_type, init_type) def infer_partial_type(self, name: Var, lvalue: Lvalue, init_type: Type) -> bool: init_type = get_proper_type(init_type) @@ -4535,6 +4537,8 @@ def check_simple_assignment( self.widened_vars.append(inferred.name) self.set_inferred_type(inferred, lvalue, lvalue_type) self.binder.put(lvalue, rvalue_type) + # TODO: hack, maybe integrate into put? + self.binder.declarations[literal_hash(lvalue)] = lvalue_type if ( isinstance(get_proper_type(lvalue_type), UnionType) # Skip literal types, as they have special logic (for better errors). diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 49c9b6d4b48b..ee178c41a8bf 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -422,7 +422,7 @@ def f3() -> None: else: x = "" del x - reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "builtins.int" [case testRedefine2WhileLoopSimple] # flags: --allow-redefinition2 --local-partial-types From 172bcf2032d508c9f4925c29e8a5f6516590debc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 10 Feb 2025 14:23:21 +0000 Subject: [PATCH 35/80] Various binder fixes and test updates --- mypy/binder.py | 11 ++- mypy/checker.py | 38 ++++---- mypy/checkexpr.py | 7 +- mypy/literals.py | 4 + mypy/semanal.py | 3 +- test-data/unit/check-redefine2.test | 134 ++++++++++++++++++++++++++-- 6 files changed, 166 insertions(+), 31 deletions(-) diff --git a/mypy/binder.py b/mypy/binder.py index cb352089a32e..d9a9c4175640 100644 --- a/mypy/binder.py +++ b/mypy/binder.py @@ -7,7 +7,7 @@ from typing_extensions import TypeAlias as _TypeAlias from mypy.erasetype import remove_instance_last_known_values -from mypy.literals import Key, literal, literal_hash, subkeys +from mypy.literals import Key, literal, literal_hash, subkeys, is_member_literal_hash from mypy.nodes import Expression, IndexExpr, MemberExpr, NameExpr, RefExpr, TypeInfo, Var from mypy.subtypes import is_same_type, is_subtype from mypy.typeops import make_simplified_union @@ -98,7 +98,9 @@ class A: # This maps an expression to a list of bound types for every item in the union type. type_assignments: Assigns | None = None - def __init__(self) -> None: + def __init__(self, bind_all: bool) -> None: + self.bind_all = bind_all + # Each frame gets an increasing, distinct id. self.next_id = 1 @@ -227,12 +229,15 @@ def update_from_options(self, frames: list[Frame]) -> bool: for key in keys: current_value = self._get(key) resulting_values = [f.types.get(key, current_value) for f in frames] - if any(x is None for x in resulting_values): + old_semantics = not self.bind_all or is_member_literal_hash(key) + if old_semantics and any(x is None for x in resulting_values): # We didn't know anything about key before # (current_value must be None), and we still don't # know anything about key in at least one possible frame. continue + resulting_values = [x for x in resulting_values if x is not None] + if all_reachable and all( x is not None and not x.from_assignment for x in resulting_values ): diff --git a/mypy/checker.py b/mypy/checker.py index ce1ca5f215d1..7e10d4a620b9 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -378,7 +378,7 @@ def __init__( self.plugin = plugin self.tscope = Scope() self.scope = CheckerScope(tree) - self.binder = ConditionalTypeBinder() + self.binder = ConditionalTypeBinder(bind_all=options.allow_redefinition2) self.globals = tree.names self.return_types = [] self.dynamic_funcs = [] @@ -433,7 +433,7 @@ def reset(self) -> None: # TODO: verify this is still actually worth it over creating new checkers self.partial_reported.clear() self.module_refs.clear() - self.binder = ConditionalTypeBinder() + self.binder = ConditionalTypeBinder(bind_all=self.options.allow_redefinition2) self._type_maps[1:] = [] self._type_maps[0].clear() self.temp_type_map = None @@ -1198,7 +1198,7 @@ def check_func_def( original_typ = typ for item, typ in expanded: old_binder = self.binder - self.binder = ConditionalTypeBinder() + self.binder = ConditionalTypeBinder(bind_all=self.options.allow_redefinition2) with self.binder.top_frame_context(): defn.expanded.append(item) @@ -2571,7 +2571,7 @@ def visit_class_def(self, defn: ClassDef) -> None: self.fail(message_registry.CANNOT_INHERIT_FROM_FINAL.format(base.name), defn) with self.tscope.class_scope(defn.info), self.enter_partial_types(is_class=True): old_binder = self.binder - self.binder = ConditionalTypeBinder() + self.binder = ConditionalTypeBinder(bind_all=self.options.allow_redefinition2) with self.binder.top_frame_context(): with self.scope.push_class(defn.info): self.accept(defn.defs) @@ -3295,7 +3295,7 @@ def check_assignment( lvalue_type = make_optional_type(lvalue_type) self.set_inferred_type(lvalue.node, lvalue, lvalue_type) - rvalue_type = self.check_simple_assignment( + rvalue_type, lvalue_type = self.check_simple_assignment( lvalue_type, rvalue, context=rvalue, inferred=inferred, lvalue=lvalue) inferred = None @@ -4503,11 +4503,11 @@ def check_simple_assignment( *, notes: list[str] | None = None, lvalue: Expression | None = None, - inferred: Var| None = None, - ) -> Type: + inferred: Var | None = None, + ) -> tuple[Type, Type | None]: if self.is_stub and isinstance(rvalue, EllipsisExpr): # '...' is always a valid initializer in a stub. - return AnyType(TypeOfAny.special_form) + return AnyType(TypeOfAny.special_form), lvalue_type else: always_allow_any = lvalue_type is not None and not isinstance( get_proper_type(lvalue_type), AnyType @@ -4578,7 +4578,7 @@ def check_simple_assignment( f"{lvalue_name} has type", notes=notes, ) - return rvalue_type + return rvalue_type, lvalue_type def check_member_assignment( self, @@ -4605,7 +4605,7 @@ def check_member_assignment( if (isinstance(instance_type, FunctionLike) and instance_type.is_type_obj()) or isinstance( instance_type, TypeType ): - rvalue_type = self.check_simple_assignment(attribute_type, rvalue, context) + rvalue_type, _ = self.check_simple_assignment(attribute_type, rvalue, context) return rvalue_type, attribute_type, True with self.msg.filter_errors(filter_deprecated=True): @@ -4616,7 +4616,7 @@ def check_member_assignment( if not isinstance(attribute_type, Instance): # TODO: support __set__() for union types. - rvalue_type = self.check_simple_assignment(attribute_type, rvalue, context) + rvalue_type, _ = self.check_simple_assignment(attribute_type, rvalue, context) return rvalue_type, attribute_type, use_binder mx = MemberContext( @@ -4635,7 +4635,7 @@ def check_member_assignment( # the return type of __get__. This doesn't match the python semantics, # (which allow you to override the descriptor with any value), but preserves # the type of accessing the attribute (even after the override). - rvalue_type = self.check_simple_assignment(get_type, rvalue, context) + rvalue_type, _ = self.check_simple_assignment(get_type, rvalue, context) return rvalue_type, get_type, use_binder dunder_set = attribute_type.type.get_method("__set__") @@ -4711,7 +4711,7 @@ def check_member_assignment( # and '__get__' type is narrower than '__set__', then we invoke the binder to narrow type # by this assignment. Technically, this is not safe, but in practice this is # what a user expects. - rvalue_type = self.check_simple_assignment(set_type, rvalue, context) + rvalue_type, _ = self.check_simple_assignment(set_type, rvalue, context) infer = is_subtype(rvalue_type, get_type) and is_subtype(get_type, set_type) return rvalue_type if infer else set_type, get_type, infer @@ -4741,6 +4741,14 @@ def check_indexed_assignment( if isinstance(res_type, UninhabitedType) and not res_type.ambiguous: self.binder.unreachable() + def replace_partial_type(self, var: Var, new_type: Type, partial_types: dict[Var, Context]) -> None: + var.type = new_type + del partial_types[var] + if self.options.allow_redefinition2: + n = NameExpr(var.name) + n.node = var + self.binder.assign_type(n, new_type, new_type) + def try_infer_partial_type_from_indexed_assignment( self, lvalue: IndexExpr, rvalue: Expression ) -> None: @@ -4777,8 +4785,8 @@ def try_infer_partial_type_from_indexed_assignment( and not is_equivalent(value_type, var.type.value_type) ) ): - var.type = self.named_generic_type(typename, [key_type, value_type]) - del partial_types[var] + new_type = self.named_generic_type(typename, [key_type, value_type]) + self.replace_partial_type(var, new_type, partial_types) def type_requires_usage(self, typ: Type) -> tuple[str, ErrorCode] | None: """Some types require usage in all cases. The classic example is diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index cc138e571021..9b7b1c14d338 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1117,8 +1117,7 @@ def try_infer_partial_type(self, e: CallExpr) -> None: typ = self.try_infer_partial_value_type_from_call(e, callee.name, var) # Var may be deleted from partial_types in try_infer_partial_value_type_from_call if typ is not None and var in partial_types: - var.type = typ - del partial_types[var] + self.chk.replace_partial_type(var, typ, partial_types) elif isinstance(callee.expr, IndexExpr) and isinstance(callee.expr.base, RefExpr): # Call 'x[y].method(...)'; may infer type of 'x' if it's a partial defaultdict. if callee.expr.analyzed is not None: @@ -1140,8 +1139,8 @@ def try_infer_partial_type(self, e: CallExpr) -> None: # Store inferred partial type. assert partial_type.type is not None typename = partial_type.type.fullname - var.type = self.chk.named_generic_type(typename, [key_type, value_type]) - del partial_types[var] + new_type = self.chk.named_generic_type(typename, [key_type, value_type]) + self.chk.replace_partial_type(var, new_type, partial_types) def get_partial_var(self, ref: RefExpr) -> tuple[Var, dict[Var, Context]] | None: var = ref.node diff --git a/mypy/literals.py b/mypy/literals.py index 5b0c46f4bee8..1b924d1b8931 100644 --- a/mypy/literals.py +++ b/mypy/literals.py @@ -163,6 +163,10 @@ def extract_var_from_literal_hash(key: Key) -> Var | None: return None +def is_member_literal_hash(key: Key) -> bool: + return key[0] == "Member" + + class _Hasher(ExpressionVisitor[Optional[Key]]): def visit_int_expr(self, e: IntExpr) -> Key: return ("Literal", e.value) diff --git a/mypy/semanal.py b/mypy/semanal.py index 9c72779009f3..1020d0196a30 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3764,7 +3764,8 @@ def process_type_annotation(self, s: AssignmentStmt) -> None: # decide here what to infer: int or Literal[42]. safe_literal_inference = self.type.mro[1].get(ref_expr.name) is None if safe_literal_inference and ref_expr.is_inferred_def: - s.type = self.analyze_simple_literal_type(s.rvalue, s.is_final_def) + if not self.options.allow_redefinition2 or s.is_final_def: + s.type = self.analyze_simple_literal_type(s.rvalue, s.is_final_def) if s.type: # Store type into nodes. for lvalue in s.lvalues: diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index ee178c41a8bf..4c00509f9e44 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -34,12 +34,69 @@ def f2() -> None: x = None reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" -def f3() -> None: +[case testRedefine2UninitializedCodePath1] +# flags: --allow-redefinition2 --local-partial-types +def f1() -> None: if int(): x = 0 + reveal_type(x) # N: Revealed type is "builtins.int" + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" + +[case testRedefine2UninitializedCodePath2] +# flags: --allow-redefinition2 --local-partial-types +from typing import Union + +def f1() -> None: + if int(): + x: Union[int, str] = 0 + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" x = "" reveal_type(x) # N: Revealed type is "builtins.str" +[case testRedefine2UninitializedCodePath3] +# flags: --allow-redefinition2 --local-partial-types +from typing import Union + +def f1() -> None: + if int(): + x = 0 + elif int(): + x = "" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +[case testRedefine2UninitializedCodePath4] +# flags: --allow-redefinition2 --local-partial-types +from typing import Union + +def f1() -> None: + if int(): + x: Union[int, str] = 0 + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +[case testRedefine2UninitializedCodePath5] +# flags: --allow-redefinition2 --local-partial-types +from typing import Union + +def f1() -> None: + x = 0 + if int(): + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" + x = None + reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" + +[case testRedefine2UninitializedCodePath6] +# flags: --allow-redefinition2 --local-partial-types +from typing import Union + +x: Union[str, None] + +def f1() -> None: + if x is not None: + reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "Union[builtins.str, None]" + [case testRedefine2GlobalVariableSimple] # flags: --allow-redefinition2 --local-partial-types if int(): @@ -114,13 +171,27 @@ def f1() -> None: [case testRedefine2AssignmentExpression] # flags: --allow-redefinition2 --local-partial-types -def f() -> None: +def f1() -> None: if x := int(): reveal_type(x) # N: Revealed type is "builtins.int" elif x := str(): reveal_type(x) # N: Revealed type is "builtins.str" reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +def f2() -> None: + if x := int(): + reveal_type(x) # N: Revealed type is "builtins.int" + elif x := str(): + reveal_type(x) # N: Revealed type is "builtins.str" + else: + pass + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +def f3() -> None: + if (x := int()) or (x := str()): + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + [case testRedefine2OperatorAssignment] # flags: --allow-redefinition2 --local-partial-types class D: pass @@ -133,7 +204,7 @@ if int(): reveal_type(c) # N: Revealed type is "__main__.D" reveal_type(c) # N: Revealed type is "Union[__main__.C, __main__.D]" -[case testRedefine2ImportFrom] +[case testRedefine2ImportFrom-xfail] # flags: --allow-redefinition2 --local-partial-types if int(): from m import x @@ -209,9 +280,9 @@ else: z = "" reveal_type(z) # N: Revealed type is "Union[None, builtins.int, builtins.str]" -[case testRedefine2OptionalTypeForInstanceVariabls] +[case testRedefine2PartialTypeForInstanceVariable] # flags: --allow-redefinition2 --local-partial-types -class C: +class C1: def __init__(self) -> None: self.x = None if int(): @@ -219,6 +290,50 @@ class C: reveal_type(self.x) # N: Revealed type is "builtins.int" reveal_type(self.x) # N: Revealed type is "Union[builtins.int, None]" +reveal_type(C1().x) # N: Revealed type is "Union[builtins.int, None]" + +class C2: + def __init__(self) -> None: + self.x = [] + for i in [1, 2]: + self.x.append(i) + reveal_type(self.x) # N: Revealed type is "builtins.list[builtins.int]" + +reveal_type(C2().x) # N: Revealed type is "builtins.list[builtins.int]" + +class C3: + def __init__(self) -> None: + self.x = None + if int(): + self.x = 1 + else: + self.x = "" # E: Incompatible types in assignment (expression has type "str", variable has type "Optional[int]") + reveal_type(self.x) # N: Revealed type is "Union[builtins.int, None]" + +reveal_type(C3().x) # N: Revealed type is "Union[builtins.int, None]" + +class C4: + def __init__(self) -> None: + self.x = [] + if int(): + self.x = [""] + reveal_type(self.x) # N: Revealed type is "builtins.list[builtins.str]" + +reveal_type(C4().x) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/list.pyi] + +[case testZZZ] +# flags: --local-partial-types +class C: + def __init__(self) -> None: + self.x = None + if int(): + self.x = 1 + + reveal_type(self.x) # N: Revealed type is "Union[builtins.int, None]" + +reveal_type(C().x) # N: Revealed type is "Union[builtins.int, None]" + [case testRedefine2PartialGenericTypes] # flags: --allow-redefinition2 --local-partial-types def f1() -> None: @@ -432,13 +547,13 @@ def f() -> None: reveal_type(x) # N: Revealed type is "builtins.str" x = 0 reveal_type(x) # N: Revealed type is "builtins.int" - reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int]" + reveal_type(x) # N: Revealed type is "builtins.int" while int(): x = None reveal_type(x) # N: Revealed type is "None" x = b"" reveal_type(x) # N: Revealed type is "builtins.bytes" - reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int, None, builtins.bytes]" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.bytes]" x = [1] reveal_type(x) # N: Revealed type is "builtins.list[builtins.int]" @@ -611,11 +726,14 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" reveal_type(y) # N: Revealed type is "Union[builtins.str, builtins.int]" +[builtins fixtures/for.pyi] +[case testRedefine2ForLoop1] +# flags: --allow-redefinition2 --local-partial-types def l() -> list[int]: return [] -def f3() -> None: +def f1() -> None: x = "" for x in l(): reveal_type(x) # N: Revealed type is "builtins.int" From 9353132e1f05eb346fbba05867e998b1f92c3de0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 10 Feb 2025 14:26:52 +0000 Subject: [PATCH 36/80] Add default arg test case --- test-data/unit/check-redefine2.test | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 4c00509f9e44..55797c877646 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -118,6 +118,16 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[case testRedefine2DefaultArgument] +# flags: --allow-redefinition2 --local-partial-types +from typing import Optional + +def f(x: Optional[str] = None) -> None: + reveal_type(x) # N: Revealed type is "Union[builtins.str, None]" + if x is None: + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" + [case testRedefine2ClassBody] # flags: --allow-redefinition2 --local-partial-types class C: From 13b7aa36c5fbb6b53638f87162d9ac65e62d189d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 10 Feb 2025 15:42:18 +0000 Subject: [PATCH 37/80] Add max iterations check to processing loops --- mypy/checker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/checker.py b/mypy/checker.py index 7e10d4a620b9..c767abad9c6b 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -618,6 +618,8 @@ def accept_loop( partials_old = partials_new widened_old = widened_new iter += 1 + if iter == 20: + raise RuntimeError(f"Too many iterations when checking a loop") # If necessary, reset the modified options and make up for the postponed error checks: self.options.warn_unreachable = warn_unreachable From f1223dfad5252b5c7ca17e734dd57c5259bd6c03 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 10 Feb 2025 15:43:24 +0000 Subject: [PATCH 38/80] Fix try statement within loop --- mypy/checker.py | 8 +++-- test-data/unit/check-redefine2.test | 47 +++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index c767abad9c6b..1fc4e19a79e7 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -5112,8 +5112,12 @@ def visit_try_without_finally(self, s: TryStmt, try_frame: bool) -> None: # try/except block. source = var.name if isinstance(var.node, Var): - var.node.type = DeletedType(source=source) - self.binder.cleanse(var) + new_type = DeletedType(source=source) + var.node.type = new_type + if self.options.allow_redefinition2: + self.binder.assign_type(var, new_type, new_type) + if not self.options.allow_redefinition2: + self.binder.cleanse(var) if s.else_body: self.accept(s.else_body) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 55797c877646..689b376018b5 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -594,6 +594,53 @@ def f1() -> None: reveal_type(x) # N: Revealed type is "builtins.list[builtins.int]" [builtins fixtures/list.pyi] +[case testRedefine2WhileLoopComplex1] +# flags: --allow-redefinition2 --local-partial-types + +def f1() -> None: + while True: + try: + pass + except Exception as e: + continue +[builtins fixtures/exception.pyi] + +[case testRedefine2WhileLoopComplex2] +# flags: --allow-redefinition2 --local-partial-types + +class C: + def __enter__(self) -> str: ... + def __exit__(self, *args) -> str: ... + +def f1() -> None: + while True: + with C() as x: + continue + +def f2() -> None: + while True: + from m import y + if int(): + continue + +[file m.py] +y = "" +[builtins fixtures/tuple.pyi] + +[case testRedefine2WhileLoopComplex3] +# flags: --allow-redefinition2 --local-partial-types --python-version 3.10 + +def f1() -> None: + while True: + x = object() + match x: + case str(y): + pass + case int(): + pass + if int(): + continue + [case testRedefine2Return] # flags: --allow-redefinition2 --local-partial-types def f1() -> None: From d4445dee5ba2eb1982d7f46695f5864c3a34f126 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 10 Feb 2025 15:58:34 +0000 Subject: [PATCH 39/80] Fix indexed literals --- mypy/binder.py | 4 ++-- mypy/literals.py | 4 ---- test-data/unit/check-redefine2.test | 10 ++++++++++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/mypy/binder.py b/mypy/binder.py index d9a9c4175640..fc68a3aef47f 100644 --- a/mypy/binder.py +++ b/mypy/binder.py @@ -7,7 +7,7 @@ from typing_extensions import TypeAlias as _TypeAlias from mypy.erasetype import remove_instance_last_known_values -from mypy.literals import Key, literal, literal_hash, subkeys, is_member_literal_hash +from mypy.literals import Key, literal, literal_hash, subkeys, extract_var_from_literal_hash from mypy.nodes import Expression, IndexExpr, MemberExpr, NameExpr, RefExpr, TypeInfo, Var from mypy.subtypes import is_same_type, is_subtype from mypy.typeops import make_simplified_union @@ -229,7 +229,7 @@ def update_from_options(self, frames: list[Frame]) -> bool: for key in keys: current_value = self._get(key) resulting_values = [f.types.get(key, current_value) for f in frames] - old_semantics = not self.bind_all or is_member_literal_hash(key) + old_semantics = not self.bind_all or extract_var_from_literal_hash(key) is None if old_semantics and any(x is None for x in resulting_values): # We didn't know anything about key before # (current_value must be None), and we still don't diff --git a/mypy/literals.py b/mypy/literals.py index 1b924d1b8931..5b0c46f4bee8 100644 --- a/mypy/literals.py +++ b/mypy/literals.py @@ -163,10 +163,6 @@ def extract_var_from_literal_hash(key: Key) -> Var | None: return None -def is_member_literal_hash(key: Key) -> bool: - return key[0] == "Member" - - class _Hasher(ExpressionVisitor[Optional[Key]]): def visit_int_expr(self, e: IntExpr) -> Key: return ("Literal", e.value) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 689b376018b5..ec23d75678c3 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -797,6 +797,16 @@ def f1() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int]" [builtins fixtures/for.pyi] +[case testRedefine2ForLoop2] +# flags: --allow-redefinition2 --local-partial-types +from typing import Any + +def f(a: Any) -> None: + for d in a: + if isinstance(d["x"], str): + return +[builtins fixtures/isinstance.pyi] + [case testRedefine2ForStatementIndexNarrowing] # flags: --allow-redefinition2 --local-partial-types from typing_extensions import TypedDict From b3b1e9ef2d0fba6680980d96062d9a7aa2390330 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 10 Feb 2025 17:58:21 +0000 Subject: [PATCH 40/80] Put types of parameters and annotated variables to the binder --- mypy/checker.py | 12 +++++++++ test-data/unit/check-redefine2.test | 39 +++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 1fc4e19a79e7..5c453f551f94 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1388,6 +1388,16 @@ def check_func_def( new_frame = self.binder.push_frame() new_frame.types[key] = narrowed_type self.binder.declarations[key] = old_binder.declarations[key] + + if self.options.allow_redefinition2 and not self.is_stub: + # Add formal argument types to the binder. + for arg in defn.arguments: + # TODO: Add these directly using a fast path + n = NameExpr("") + v = arg.variable + n.node = v + self.binder.assign_type(n, v.type, v.type) + with self.scope.push_function(defn): # We suppress reachability warnings for empty generator functions # (return; yield) which have a "yield" that's unreachable by definition @@ -3333,6 +3343,8 @@ def check_assignment( and not self.options.allow_redefinition2 # TODO WHAT ): lvalue.node.type = remove_instance_last_known_values(lvalue_type) + elif self.options.allow_redefinition2: + self.binder.assign_type(lvalue, lvalue_type, lvalue_type) elif index_lvalue: self.check_indexed_assignment(index_lvalue, rvalue, lvalue) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index ec23d75678c3..587cae74e673 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -118,16 +118,26 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" -[case testRedefine2DefaultArgument] +[case testRedefine2ParameterTypes] # flags: --allow-redefinition2 --local-partial-types from typing import Optional -def f(x: Optional[str] = None) -> None: +def f1(x: Optional[str] = None) -> None: reveal_type(x) # N: Revealed type is "Union[builtins.str, None]" if x is None: x = "" reveal_type(x) # N: Revealed type is "builtins.str" +def f2(*args: str, **kwargs: int) -> None: + reveal_type(args) # N: Revealed type is "builtins.tuple[builtins.str, ...]" + reveal_type(kwargs) # N: Revealed type is "builtins.dict[builtins.str, builtins.int]" + +class C: + def m(self) -> None: + reveal_type(self) # N: Revealed type is "__main__.C" +[builtins fixtures/dict.pyi] + + [case testRedefine2ClassBody] # flags: --allow-redefinition2 --local-partial-types class C: @@ -853,3 +863,28 @@ def f3() -> None: for x in [1]: x = "" reveal_type(x) # N: Revealed type is "builtins.str" + +[case testRedefine2VariableAnnotatedInLoop] +# flags: --allow-redefinition2 --local-partial-types --enable-error-code=redundant-expr +from typing import Optional + +def f1() -> None: + e: Optional[str] = None + for x in ["a"]: + if e is None and int(): + e = x + continue + elif e is not None and int(): + break + reveal_type(e) # N: Revealed type is "Union[builtins.str, None]" + reveal_type(e) # N: Revealed type is "Union[builtins.str, None]" + +def f2(e: Optional[str]) -> None: + for x in ["a"]: + if e is None and int(): + e = x + continue + elif e is not None and int(): + break + reveal_type(e) # N: Revealed type is "Union[builtins.str, None]" + reveal_type(e) # N: Revealed type is "Union[builtins.str, None]" From 06ed1e9c6929a840eb657b1d978ccf0fe99ad581 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 10 Feb 2025 18:12:42 +0000 Subject: [PATCH 41/80] Fix self check --- mypy/checker.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 5c453f551f94..61ae87fa9c01 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1393,10 +1393,11 @@ def check_func_def( # Add formal argument types to the binder. for arg in defn.arguments: # TODO: Add these directly using a fast path - n = NameExpr("") v = arg.variable - n.node = v - self.binder.assign_type(n, v.type, v.type) + if v.type is not None: + n = NameExpr("") + n.node = v + self.binder.assign_type(n, v.type, v.type) with self.scope.push_function(defn): # We suppress reachability warnings for empty generator functions @@ -3343,7 +3344,7 @@ def check_assignment( and not self.options.allow_redefinition2 # TODO WHAT ): lvalue.node.type = remove_instance_last_known_values(lvalue_type) - elif self.options.allow_redefinition2: + elif self.options.allow_redefinition2 and lvalue_type is not None: self.binder.assign_type(lvalue, lvalue_type, lvalue_type) elif index_lvalue: @@ -4552,7 +4553,9 @@ def check_simple_assignment( self.set_inferred_type(inferred, lvalue, lvalue_type) self.binder.put(lvalue, rvalue_type) # TODO: hack, maybe integrate into put? - self.binder.declarations[literal_hash(lvalue)] = lvalue_type + lit = literal_hash(lvalue) + if lit is not None: + self.binder.declarations[lit] = lvalue_type if ( isinstance(get_proper_type(lvalue_type), UnionType) # Skip literal types, as they have special logic (for better errors). From 9f48a19d844af40f7027e897798ab6f6e7d7acbc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 10 Feb 2025 18:18:55 +0000 Subject: [PATCH 42/80] Update tests --- test-data/unit/check-redefine2.test | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 587cae74e673..51c9ffcab244 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -805,6 +805,17 @@ def f1() -> None: for x in l(): reveal_type(x) # N: Revealed type is "builtins.int" reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int]" + +def f2() -> None: + for x in [1, 2]: + x = [x] + reveal_type(x) # N: Revealed type is "builtins.list[builtins.int]" + +def f3() -> None: + for x in [1, 2]: + if int(): + x = "x" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" [builtins fixtures/for.pyi] [case testRedefine2ForLoop2] @@ -832,6 +843,13 @@ for a in ("hourly", "daily"): reveal_type(a.upper()) # N: Revealed type is "builtins.str" c = a reveal_type(c) # N: Revealed type is "builtins.str" + a = "monthly" + reveal_type(a) # N: Revealed type is "Union[Literal['hourly']?, Literal['daily']?]" + a = "yearly" + reveal_type(a) # N: Revealed type is "Union[Literal['hourly']?, Literal['daily']?]" + a = 1 + reveal_type(a) # N: Revealed type is "builtins.int" +reveal_type(a) # N: Revealed type is "builtins.int" b: str for b in ("hourly", "daily"): From f5a191503e1d34bf4c7e192a5c57ff6f1eec7685 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Feb 2025 13:29:07 +0000 Subject: [PATCH 43/80] Improve repr of Var nodes --- mypy/nodes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 6487ee4b745c..7616922be3fa 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1073,7 +1073,8 @@ def fullname(self) -> str: return self._fullname def __repr__(self) -> str: - return f"" + name = self.fullname or self.name + return f"" def accept(self, visitor: NodeVisitor[T]) -> T: return visitor.visit_var(self) From 6ca4d76b809c796b13f843fa3a63979a7cb05714 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Feb 2025 13:30:55 +0000 Subject: [PATCH 44/80] Fix match statements by reusing dummy Var node --- mypy/checker.py | 14 ++++++++------ mypy/nodes.py | 4 +++- test-data/unit/check-redefine2.test | 13 ++++++++++++- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 61ae87fa9c01..223bd231538c 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1395,7 +1395,7 @@ def check_func_def( # TODO: Add these directly using a fast path v = arg.variable if v.type is not None: - n = NameExpr("") + n = NameExpr(v.name) n.node = v self.binder.assign_type(n, v.type, v.type) @@ -5521,11 +5521,13 @@ def visit_match_stmt(self, s: MatchStmt) -> None: # Create a dummy subject expression to handle cases where a match statement's subject # is not a literal value. This lets us correctly narrow types and check exhaustivity # This is hack! - id = s.subject.callee.fullname if isinstance(s.subject.callee, RefExpr) else "" - name = "dummy-match-" + id - v = Var(name) - named_subject = NameExpr(name) - named_subject.node = v + if s.subject_dummy is None: + id = s.subject.callee.fullname if isinstance(s.subject.callee, RefExpr) else "" + name = "dummy-match-" + id + v = Var(name) + s.subject_dummy = NameExpr(name) + s.subject_dummy.node = v + named_subject = s.subject_dummy else: named_subject = s.subject diff --git a/mypy/nodes.py b/mypy/nodes.py index 7616922be3fa..678c28f14684 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1638,11 +1638,12 @@ def accept(self, visitor: StatementVisitor[T]) -> T: class MatchStmt(Statement): - __slots__ = ("subject", "patterns", "guards", "bodies") + __slots__ = ("subject", "subject_dummy", "patterns", "guards", "bodies") __match_args__ = ("subject", "patterns", "guards", "bodies") subject: Expression + subject_dummy: NameExpr | None patterns: list[Pattern] guards: list[Expression | None] bodies: list[Block] @@ -1657,6 +1658,7 @@ def __init__( super().__init__() assert len(patterns) == len(guards) == len(bodies) self.subject = subject + self.subject_dummy = None self.patterns = patterns self.guards = guards self.bodies = bodies diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 51c9ffcab244..64924901cf41 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -637,7 +637,7 @@ def f2() -> None: y = "" [builtins fixtures/tuple.pyi] -[case testRedefine2WhileLoopComplex3] +[case testRedefine2LoopWithMatch] # flags: --allow-redefinition2 --local-partial-types --python-version 3.10 def f1() -> None: @@ -651,6 +651,17 @@ def f1() -> None: if int(): continue +def f2() -> None: + for x in [""]: + match str(): + case "a": + y = "" + case "b": + y = 1 + return + reveal_type(y) # N: Revealed type is "builtins.str" +[builtins fixtures/list.pyi] + [case testRedefine2Return] # flags: --allow-redefinition2 --local-partial-types def f1() -> None: From 9fde15f4c41eced75c49c19c833c662f9e285964 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Feb 2025 14:32:44 +0000 Subject: [PATCH 45/80] Fix crash from inferring union with partial type item --- mypy/checker.py | 2 +- test-data/unit/check-redefine2.test | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 223bd231538c..9b67d2b62203 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4548,7 +4548,7 @@ def check_simple_assignment( new_inferred = rvalue_type if not is_same_type(inferred.type, new_inferred): lvalue_type = make_simplified_union([inferred.type, new_inferred]) - if not is_same_type(lvalue_type, inferred.type): + if not is_same_type(lvalue_type, inferred.type) and not isinstance(inferred.type, PartialType): self.widened_vars.append(inferred.name) self.set_inferred_type(inferred, lvalue, lvalue_type) self.binder.put(lvalue, rvalue_type) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 64924901cf41..800e279c8443 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -917,3 +917,20 @@ def f2(e: Optional[str]) -> None: break reveal_type(e) # N: Revealed type is "Union[builtins.str, None]" reveal_type(e) # N: Revealed type is "Union[builtins.str, None]" + +[case testRedefine2LoopAndPartialTypesSpecialCase] +# flags: --allow-redefinition2 --local-partial-types +def f() -> list[str]: + a = [] # type: ignore + o = [] + for line in ["x"]: + if int(): + continue + if int(): + a = [] + if int(): + a.append(line) + else: + o.append(line) + return o +[builtins fixtures/list.pyi] From abf4eac160cde01a5a6e8b726b7f7aa7acf4f84e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Feb 2025 17:35:44 +0000 Subject: [PATCH 46/80] Add tests --- test-data/unit/check-redefine2.test | 65 +++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 800e279c8443..35d12fdbe316 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -760,8 +760,73 @@ def f1() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" reveal_type(e) # N: Revealed type is "" reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +def f2() -> None: + try: + x = 1 + if int(): + x = "" + return + except Exception: + # TODO: Too wide + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + # TODO: Too wide + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +def f3() -> None: + try: + x = 1 + if int(): + x = "" + return + finally: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" \ + # N: Revealed type is "builtins.int" + reveal_type(x) # N: Revealed type is "builtins.int" + +def f4() -> None: + while int(): + try: + x = 1 + if int(): + x = "" + break + if int(): + while int(): + if int(): + x = None + break + finally: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, None]" \ + # N: Revealed type is "Union[builtins.int, None]" + reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" [builtins fixtures/exception.pyi] +[case testRedefine2RaiseStatement] +# flags: --allow-redefinition2 --local-partial-types +def f1() -> None: + if int(): + x = "" + elif int(): + x = None + raise Exception() + else: + x = 1 + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int]" + +def f2() -> None: + try: + x = 1 + if int(): + x = "" + raise Exception() + reveal_type(x) # N: Revealed type is "builtins.int" + except Exception: + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/exception.pyi] + + [case testRedefine2MultipleAssignment] # flags: --allow-redefinition2 --local-partial-types def f1() -> None: From 71cdb613ddeee83c6637dd474070c45fe11645f4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Feb 2025 17:41:11 +0000 Subject: [PATCH 47/80] Add test cases for "del" --- test-data/unit/check-redefine2.test | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 35d12fdbe316..5ef76b0c561f 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -559,6 +559,23 @@ def f3() -> None: del x reveal_type(x) # N: Revealed type is "builtins.int" +def f4() -> None: + while int(): + if int(): + x: int = 0 + else: + del x + reveal_type(x) # N: Revealed type is "builtins.int" + +def f5() -> None: + while int(): + if int(): + x = 0 + else: + del x + continue + x = "" + reveal_type(x) # N: Revealed type is "builtins.str" [case testRedefine2WhileLoopSimple] # flags: --allow-redefinition2 --local-partial-types def f() -> None: From 69fca07aee18f1a9a8f48ca9b5765746121da821 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 20 Feb 2025 14:35:03 +0000 Subject: [PATCH 48/80] Rename flag to --allow-redefinition-new --- mypy/build.py | 2 +- mypy/checker.py | 28 ++++----- mypy/main.py | 10 ++-- mypy/options.py | 5 +- mypy/semanal.py | 4 +- test-data/unit/check-python310.test | 2 +- test-data/unit/check-redefine2.test | 92 ++++++++++++++--------------- 7 files changed, 72 insertions(+), 71 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index a43b0e7a72b5..aff8fa045e0e 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -2240,7 +2240,7 @@ def semantic_analysis_pass1(self) -> None: # TODO: Do this while constructing the AST? self.tree.names = SymbolTable() if not self.tree.is_stub: - if not self.options.allow_redefinition2: + if not self.options.allow_redefinition_new: # Always perform some low-key variable renaming when assignments can't # widen inferred types self.tree.accept(LimitedVariableRenameVisitor()) diff --git a/mypy/checker.py b/mypy/checker.py index 9b67d2b62203..3973e844a658 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -378,7 +378,7 @@ def __init__( self.plugin = plugin self.tscope = Scope() self.scope = CheckerScope(tree) - self.binder = ConditionalTypeBinder(bind_all=options.allow_redefinition2) + self.binder = ConditionalTypeBinder(bind_all=options.allow_redefinition_new) self.globals = tree.names self.return_types = [] self.dynamic_funcs = [] @@ -433,7 +433,7 @@ def reset(self) -> None: # TODO: verify this is still actually worth it over creating new checkers self.partial_reported.clear() self.module_refs.clear() - self.binder = ConditionalTypeBinder(bind_all=self.options.allow_redefinition2) + self.binder = ConditionalTypeBinder(bind_all=self.options.allow_redefinition_new) self._type_maps[1:] = [] self._type_maps[0].clear() self.temp_type_map = None @@ -1200,7 +1200,7 @@ def check_func_def( original_typ = typ for item, typ in expanded: old_binder = self.binder - self.binder = ConditionalTypeBinder(bind_all=self.options.allow_redefinition2) + self.binder = ConditionalTypeBinder(bind_all=self.options.allow_redefinition_new) with self.binder.top_frame_context(): defn.expanded.append(item) @@ -1389,7 +1389,7 @@ def check_func_def( new_frame.types[key] = narrowed_type self.binder.declarations[key] = old_binder.declarations[key] - if self.options.allow_redefinition2 and not self.is_stub: + if self.options.allow_redefinition_new and not self.is_stub: # Add formal argument types to the binder. for arg in defn.arguments: # TODO: Add these directly using a fast path @@ -2584,7 +2584,7 @@ def visit_class_def(self, defn: ClassDef) -> None: self.fail(message_registry.CANNOT_INHERIT_FROM_FINAL.format(base.name), defn) with self.tscope.class_scope(defn.info), self.enter_partial_types(is_class=True): old_binder = self.binder - self.binder = ConditionalTypeBinder(bind_all=self.options.allow_redefinition2) + self.binder = ConditionalTypeBinder(bind_all=self.options.allow_redefinition_new) with self.binder.top_frame_context(): with self.scope.push_class(defn.info): self.accept(defn.defs) @@ -3341,10 +3341,10 @@ def check_assignment( and lvalue.node.is_inferred and lvalue.node.is_index_var and lvalue_type is not None - and not self.options.allow_redefinition2 # TODO WHAT + and not self.options.allow_redefinition_new # TODO WHAT ): lvalue.node.type = remove_instance_last_known_values(lvalue_type) - elif self.options.allow_redefinition2 and lvalue_type is not None: + elif self.options.allow_redefinition_new and lvalue_type is not None: self.binder.assign_type(lvalue, lvalue_type, lvalue_type) elif index_lvalue: @@ -4311,7 +4311,7 @@ def check_lvalue(self, lvalue: Lvalue) -> tuple[Type | None, IndexExpr | None, V self.store_type(lvalue, lvalue_type) elif isinstance(lvalue, NameExpr): lvalue_type = self.expr_checker.analyze_ref_expr(lvalue, lvalue=True) - if isinstance(lvalue.node, Var) and lvalue.node.is_inferred and self.options.allow_redefinition2: + if isinstance(lvalue.node, Var) and lvalue.node.is_inferred and self.options.allow_redefinition_new: inferred = lvalue.node self.store_type(lvalue, lvalue_type) elif isinstance(lvalue, (TupleExpr, ListExpr)): @@ -4382,12 +4382,12 @@ def infer_variable_type( init_type = strip_type(init_type) self.set_inferred_type(name, lvalue, init_type) - if self.options.allow_redefinition2: + if self.options.allow_redefinition_new: self.binder.assign_type(lvalue, init_type, init_type) def infer_partial_type(self, name: Var, lvalue: Lvalue, init_type: Type) -> bool: init_type = get_proper_type(init_type) - if isinstance(init_type, NoneType) and (isinstance(lvalue, MemberExpr) or not self.options.allow_redefinition2): + if isinstance(init_type, NoneType) and (isinstance(lvalue, MemberExpr) or not self.options.allow_redefinition_new): partial_type = PartialType(None, name) elif isinstance(init_type, Instance): fullname = init_type.type.fullname @@ -4761,7 +4761,7 @@ def check_indexed_assignment( def replace_partial_type(self, var: Var, new_type: Type, partial_types: dict[Var, Context]) -> None: var.type = new_type del partial_types[var] - if self.options.allow_redefinition2: + if self.options.allow_redefinition_new: n = NameExpr(var.name) n.node = var self.binder.assign_type(n, new_type, new_type) @@ -5129,9 +5129,9 @@ def visit_try_without_finally(self, s: TryStmt, try_frame: bool) -> None: if isinstance(var.node, Var): new_type = DeletedType(source=source) var.node.type = new_type - if self.options.allow_redefinition2: + if self.options.allow_redefinition_new: self.binder.assign_type(var, new_type, new_type) - if not self.options.allow_redefinition2: + if not self.options.allow_redefinition_new: self.binder.cleanse(var) if s.else_body: self.accept(s.else_body) @@ -8570,7 +8570,7 @@ def is_valid_inferred_type(typ: Type, # type could either be NoneType or an Optional type, depending on # the context. This resolution happens in leave_partial_types when # we pop a partial types scope. - return is_lvalue_final or (not is_lvalue_member and options.allow_redefinition2) + return is_lvalue_final or (not is_lvalue_member and options.allow_redefinition_new) elif isinstance(proper_type, UninhabitedType): return False return not typ.accept(InvalidInferredTypes()) diff --git a/mypy/main.py b/mypy/main.py index c4cedd31368a..b3cb02f7878c 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -93,8 +93,8 @@ def main( stdout, stderr, options.hide_error_codes, hide_success=bool(options.output) ) - if options.allow_redefinition2 and not options.local_partial_types: - fail("error: --local-partial-types must be used if using --allow-redefinition2", stderr, options) + if options.allow_redefinition_new and not options.local_partial_types: + fail("error: --local-partial-types must be used if using --allow-redefinition-new", stderr, options) if options.install_types and (stdout is not sys.stdout or stderr is not sys.stderr): # Since --install-types performs user input, we want regular stdout and stderr. @@ -859,15 +859,15 @@ def add_invertible_flag( "--allow-redefinition", default=False, strict_flag=False, - help="Allow unconditional variable redefinition with a new type", + help="Allow restricted, unconditional variable redefinition with a new type", group=strictness_group, ) add_invertible_flag( - "--allow-redefinition2", + "--allow-redefinition-new", default=False, strict_flag=False, - help="Allow variable redefinition with a new type (including conditional)", + help="Allow flexible variable redefinition with a new type", group=strictness_group, ) diff --git a/mypy/options.py b/mypy/options.py index 59096bb77354..9eef53872ad2 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -219,8 +219,9 @@ def __init__(self) -> None: # and the same nesting level as the initialization self.allow_redefinition = False - # TODO - self.allow_redefinition2 = False + # Allow flexible variable redefinition with an arbitrary type, in different + # blocks and and at different nesting levels + self.allow_redefinition_new = False # Prohibit equality, identity, and container checks for non-overlapping types. # This makes 1 == '1', 1 in ['1'], and 1 is '1' errors. diff --git a/mypy/semanal.py b/mypy/semanal.py index 1020d0196a30..56755df59a78 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3764,7 +3764,7 @@ def process_type_annotation(self, s: AssignmentStmt) -> None: # decide here what to infer: int or Literal[42]. safe_literal_inference = self.type.mro[1].get(ref_expr.name) is None if safe_literal_inference and ref_expr.is_inferred_def: - if not self.options.allow_redefinition2 or s.is_final_def: + if not self.options.allow_redefinition_new or s.is_final_def: s.type = self.analyze_simple_literal_type(s.rvalue, s.is_final_def) if s.type: # Store type into nodes. @@ -4336,7 +4336,7 @@ def analyze_name_lvalue( else: lvalue.fullname = lvalue.name if self.is_func_scope(): - if unmangle(name) == "_" and not self.options.allow_redefinition2: + if unmangle(name) == "_" and not self.options.allow_redefinition_new: # Special case for assignment to local named '_': always infer 'Any'. typ = AnyType(TypeOfAny.special_form) self.store_declared_types(lvalue, typ) diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 6ab438174672..37b713c06036 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -2554,7 +2554,7 @@ def f(t: T) -> None: [builtins fixtures/tuple.pyi] [case testRedefine2MatchBasics] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f1(x: int | str | list[bytes]) -> None: match x: diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 5ef76b0c561f..7cea889da4a4 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -1,7 +1,7 @@ -- Test cases for the redefinition of variable with a different type (new version). [case testRedefine2LocalWithDifferentType] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f() -> None: x = 0 reveal_type(x) # N: Revealed type is "builtins.int" @@ -9,7 +9,7 @@ def f() -> None: reveal_type(x) # N: Revealed type is "builtins.str" [case testRedefine2ConditionalLocalWithDifferentType] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f() -> None: if int(): x = 0 @@ -19,7 +19,7 @@ def f() -> None: reveal_type(x) # N: Revealed type is "builtins.str" [case testRedefine2MergeConditionalLocal1] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f1() -> None: if int(): x = 0 @@ -35,7 +35,7 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" [case testRedefine2UninitializedCodePath1] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f1() -> None: if int(): x = 0 @@ -44,7 +44,7 @@ def f1() -> None: reveal_type(x) # N: Revealed type is "builtins.str" [case testRedefine2UninitializedCodePath2] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types from typing import Union def f1() -> None: @@ -55,7 +55,7 @@ def f1() -> None: reveal_type(x) # N: Revealed type is "builtins.str" [case testRedefine2UninitializedCodePath3] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types from typing import Union def f1() -> None: @@ -66,7 +66,7 @@ def f1() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" [case testRedefine2UninitializedCodePath4] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types from typing import Union def f1() -> None: @@ -75,7 +75,7 @@ def f1() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" [case testRedefine2UninitializedCodePath5] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types from typing import Union def f1() -> None: @@ -87,7 +87,7 @@ def f1() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" [case testRedefine2UninitializedCodePath6] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types from typing import Union x: Union[str, None] @@ -98,7 +98,7 @@ def f1() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.str, None]" [case testRedefine2GlobalVariableSimple] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types if int(): x = 0 reveal_type(x) # N: Revealed type is "builtins.int" @@ -119,7 +119,7 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" [case testRedefine2ParameterTypes] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types from typing import Optional def f1(x: Optional[str] = None) -> None: @@ -139,7 +139,7 @@ class C: [case testRedefine2ClassBody] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types class C: if int(): x = 0 @@ -152,7 +152,7 @@ class C: reveal_type(C.x) # N: Revealed type is "Union[builtins.int, builtins.str]" [case testRedefine2NestedFunctionBasics] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f1() -> None: if int(): x = 0 @@ -177,7 +177,7 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" [case testRedefine2LambdaBasics] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f1() -> None: x = 0 if int(): @@ -190,7 +190,7 @@ def f1() -> None: reveal_type(f) # N: Revealed type is "def () -> Union[builtins.int, builtins.str]" [case testRedefine2AssignmentExpression] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f1() -> None: if x := int(): reveal_type(x) # N: Revealed type is "builtins.int" @@ -213,7 +213,7 @@ def f3() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" [case testRedefine2OperatorAssignment] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types class D: pass class C: def __add__(self, x: C) -> D: ... @@ -225,7 +225,7 @@ if int(): reveal_type(c) # N: Revealed type is "Union[__main__.C, __main__.D]" [case testRedefine2ImportFrom-xfail] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types if int(): from m import x else: @@ -244,7 +244,7 @@ x = 1 y = "" [case testRedefine2Import] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types if int(): import m else: @@ -260,7 +260,7 @@ y = "" [builtins fixtures/module.pyi] [case testRedefine2OptionalTypesSimple] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f1() -> None: x = None if int(): @@ -301,7 +301,7 @@ else: reveal_type(z) # N: Revealed type is "Union[None, builtins.int, builtins.str]" [case testRedefine2PartialTypeForInstanceVariable] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types class C1: def __init__(self) -> None: self.x = None @@ -355,7 +355,7 @@ class C: reveal_type(C().x) # N: Revealed type is "Union[builtins.int, None]" [case testRedefine2PartialGenericTypes] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f1() -> None: a = [] a.append(1) @@ -405,7 +405,7 @@ def f6() -> None: [builtins fixtures/list.pyi] [case testRedefine2FinalLiteral] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types from typing_extensions import Final, Literal x: Final = "foo" @@ -414,7 +414,7 @@ a: Literal["foo"] = x [builtins fixtures/tuple.pyi] [case testRedefine2AnnotatedVariable] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types from typing import Optional def f1() -> None: @@ -448,7 +448,7 @@ class C: reveal_type(self.x) # N: Revealed type is "builtins.str" [case testRedefine2AnyType1] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def a(): pass def f1() -> None: @@ -509,7 +509,7 @@ def f7() -> None: reveal_type(x) # N: Revealed type is "builtins.int" [case testRedefine2AnyType2] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types from typing import Any def f1() -> None: @@ -534,7 +534,7 @@ def f3(x) -> None: reveal_type(x) # N: Revealed type is "Any" [case tetRedefine2Del] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f1() -> None: x = "" reveal_type(x) # N: Revealed type is "builtins.str" @@ -577,7 +577,7 @@ def f5() -> None: x = "" reveal_type(x) # N: Revealed type is "builtins.str" [case testRedefine2WhileLoopSimple] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f() -> None: while int(): x = "" @@ -595,7 +595,7 @@ def f() -> None: reveal_type(x) # N: Revealed type is "builtins.list[builtins.int]" [case testRedefine2WhileLoopOptional] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f1() -> None: x = None while int(): @@ -613,7 +613,7 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "Union[None, builtins.str]" [case testRedefine2WhileLoopPartialType] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f1() -> None: x = [] while int(): @@ -622,7 +622,7 @@ def f1() -> None: [builtins fixtures/list.pyi] [case testRedefine2WhileLoopComplex1] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f1() -> None: while True: @@ -633,7 +633,7 @@ def f1() -> None: [builtins fixtures/exception.pyi] [case testRedefine2WhileLoopComplex2] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types class C: def __enter__(self) -> str: ... @@ -655,7 +655,7 @@ y = "" [builtins fixtures/tuple.pyi] [case testRedefine2LoopWithMatch] -# flags: --allow-redefinition2 --local-partial-types --python-version 3.10 +# flags: --allow-redefinition-new --local-partial-types --python-version 3.10 def f1() -> None: while True: @@ -680,7 +680,7 @@ def f2() -> None: [builtins fixtures/list.pyi] [case testRedefine2Return] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f1() -> None: if int(): x = 0 @@ -698,7 +698,7 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "builtins.str" [case testRedefine2BreakAndContinue] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def b() -> None: while int(): x = "" @@ -723,7 +723,7 @@ def c() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, None]" [case testRedefine2Underscore] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f() -> None: if int(): _ = 0 @@ -734,7 +734,7 @@ def f() -> None: reveal_type(_) # N: Revealed type is "Union[builtins.int, builtins.str]" [case testRedefine2WithStatement] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types class C: def __enter__(self) -> int: ... def __exit__(self, x, y, z): ... @@ -758,7 +758,7 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" [case testRedefine2TryStatement] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types class E(Exception): pass def g(): ... @@ -820,7 +820,7 @@ def f4() -> None: [builtins fixtures/exception.pyi] [case testRedefine2RaiseStatement] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f1() -> None: if int(): x = "" @@ -845,7 +845,7 @@ def f2() -> None: [case testRedefine2MultipleAssignment] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f1() -> None: x, y = 1, "" reveal_type(x) # N: Revealed type is "builtins.int" @@ -867,7 +867,7 @@ def f2() -> None: reveal_type(y) # N: Revealed type is "Union[builtins.str, builtins.int]" [case testRedefine2ForLoopBasics] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f1() -> None: for x in [1]: reveal_type(x) # N: Revealed type is "builtins.int" @@ -889,7 +889,7 @@ def f2() -> None: [builtins fixtures/for.pyi] [case testRedefine2ForLoop1] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def l() -> list[int]: return [] @@ -912,7 +912,7 @@ def f3() -> None: [builtins fixtures/for.pyi] [case testRedefine2ForLoop2] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types from typing import Any def f(a: Any) -> None: @@ -922,7 +922,7 @@ def f(a: Any) -> None: [builtins fixtures/isinstance.pyi] [case testRedefine2ForStatementIndexNarrowing] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types from typing_extensions import TypedDict class X(TypedDict): @@ -951,7 +951,7 @@ for b in ("hourly", "daily"): [builtins fixtures/for.pyi] [case testRedefine2ForLoopIndexWidening] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f1() -> None: for x in [1]: @@ -976,7 +976,7 @@ def f3() -> None: reveal_type(x) # N: Revealed type is "builtins.str" [case testRedefine2VariableAnnotatedInLoop] -# flags: --allow-redefinition2 --local-partial-types --enable-error-code=redundant-expr +# flags: --allow-redefinition-new --local-partial-types --enable-error-code=redundant-expr from typing import Optional def f1() -> None: @@ -1001,7 +1001,7 @@ def f2(e: Optional[str]) -> None: reveal_type(e) # N: Revealed type is "Union[builtins.str, None]" [case testRedefine2LoopAndPartialTypesSpecialCase] -# flags: --allow-redefinition2 --local-partial-types +# flags: --allow-redefinition-new --local-partial-types def f() -> list[str]: a = [] # type: ignore o = [] From 5ee186853320b3302b0e8f8346853130456c302a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 20 Feb 2025 14:41:42 +0000 Subject: [PATCH 49/80] Rename tests --- test-data/unit/check-python310.test | 2 +- test-data/unit/check-redefine2.test | 92 ++++++++++++++--------------- 2 files changed, 47 insertions(+), 47 deletions(-) diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 37b713c06036..66d34b63919b 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -2553,7 +2553,7 @@ def f(t: T) -> None: reveal_type(k) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.K]" [builtins fixtures/tuple.pyi] -[case testRedefine2MatchBasics] +[case testNewRedefineMatchBasics] # flags: --allow-redefinition-new --local-partial-types def f1(x: int | str | list[bytes]) -> None: diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 7cea889da4a4..637e6fe5ab19 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -1,6 +1,6 @@ -- Test cases for the redefinition of variable with a different type (new version). -[case testRedefine2LocalWithDifferentType] +[case testNewRedefineLocalWithDifferentType] # flags: --allow-redefinition-new --local-partial-types def f() -> None: x = 0 @@ -8,7 +8,7 @@ def f() -> None: x = '' reveal_type(x) # N: Revealed type is "builtins.str" -[case testRedefine2ConditionalLocalWithDifferentType] +[case testNewRedefineConditionalLocalWithDifferentType] # flags: --allow-redefinition-new --local-partial-types def f() -> None: if int(): @@ -18,7 +18,7 @@ def f() -> None: x = '' reveal_type(x) # N: Revealed type is "builtins.str" -[case testRedefine2MergeConditionalLocal1] +[case testNewRedefineMergeConditionalLocal1] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: if int(): @@ -34,7 +34,7 @@ def f2() -> None: x = None reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" -[case testRedefine2UninitializedCodePath1] +[case testNewRedefineUninitializedCodePath1] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: if int(): @@ -43,7 +43,7 @@ def f1() -> None: x = "" reveal_type(x) # N: Revealed type is "builtins.str" -[case testRedefine2UninitializedCodePath2] +[case testNewRedefineUninitializedCodePath2] # flags: --allow-redefinition-new --local-partial-types from typing import Union @@ -54,7 +54,7 @@ def f1() -> None: x = "" reveal_type(x) # N: Revealed type is "builtins.str" -[case testRedefine2UninitializedCodePath3] +[case testNewRedefineUninitializedCodePath3] # flags: --allow-redefinition-new --local-partial-types from typing import Union @@ -65,7 +65,7 @@ def f1() -> None: x = "" reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" -[case testRedefine2UninitializedCodePath4] +[case testNewRedefineUninitializedCodePath4] # flags: --allow-redefinition-new --local-partial-types from typing import Union @@ -74,7 +74,7 @@ def f1() -> None: x: Union[int, str] = 0 reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" -[case testRedefine2UninitializedCodePath5] +[case testNewRedefineUninitializedCodePath5] # flags: --allow-redefinition-new --local-partial-types from typing import Union @@ -86,7 +86,7 @@ def f1() -> None: x = None reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" -[case testRedefine2UninitializedCodePath6] +[case testNewRedefineUninitializedCodePath6] # flags: --allow-redefinition-new --local-partial-types from typing import Union @@ -97,7 +97,7 @@ def f1() -> None: reveal_type(x) # N: Revealed type is "builtins.str" reveal_type(x) # N: Revealed type is "Union[builtins.str, None]" -[case testRedefine2GlobalVariableSimple] +[case testNewRedefineGlobalVariableSimple] # flags: --allow-redefinition-new --local-partial-types if int(): x = 0 @@ -118,7 +118,7 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" -[case testRedefine2ParameterTypes] +[case testNewRedefineParameterTypes] # flags: --allow-redefinition-new --local-partial-types from typing import Optional @@ -138,7 +138,7 @@ class C: [builtins fixtures/dict.pyi] -[case testRedefine2ClassBody] +[case testNewRedefineClassBody] # flags: --allow-redefinition-new --local-partial-types class C: if int(): @@ -151,7 +151,7 @@ class C: reveal_type(C.x) # N: Revealed type is "Union[builtins.int, builtins.str]" -[case testRedefine2NestedFunctionBasics] +[case testNewRedefineNestedFunctionBasics] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: if int(): @@ -176,7 +176,7 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" -[case testRedefine2LambdaBasics] +[case testNewRedefineLambdaBasics] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: x = 0 @@ -189,7 +189,7 @@ def f1() -> None: f = lambda: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" reveal_type(f) # N: Revealed type is "def () -> Union[builtins.int, builtins.str]" -[case testRedefine2AssignmentExpression] +[case testNewRedefineAssignmentExpression] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: if x := int(): @@ -212,7 +212,7 @@ def f3() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" -[case testRedefine2OperatorAssignment] +[case testNewRedefineOperatorAssignment] # flags: --allow-redefinition-new --local-partial-types class D: pass class C: @@ -224,7 +224,7 @@ if int(): reveal_type(c) # N: Revealed type is "__main__.D" reveal_type(c) # N: Revealed type is "Union[__main__.C, __main__.D]" -[case testRedefine2ImportFrom-xfail] +[case testNewRedefineImportFrom-xfail] # flags: --allow-redefinition-new --local-partial-types if int(): from m import x @@ -243,7 +243,7 @@ reveal_type(y) # N: Revealed type is "Union[builtins.str, builtins.int]" x = 1 y = "" -[case testRedefine2Import] +[case testNewRedefineImport] # flags: --allow-redefinition-new --local-partial-types if int(): import m @@ -259,7 +259,7 @@ x = 1 y = "" [builtins fixtures/module.pyi] -[case testRedefine2OptionalTypesSimple] +[case testNewRedefineOptionalTypesSimple] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: x = None @@ -300,7 +300,7 @@ else: z = "" reveal_type(z) # N: Revealed type is "Union[None, builtins.int, builtins.str]" -[case testRedefine2PartialTypeForInstanceVariable] +[case testNewRedefinePartialTypeForInstanceVariable] # flags: --allow-redefinition-new --local-partial-types class C1: def __init__(self) -> None: @@ -354,7 +354,7 @@ class C: reveal_type(C().x) # N: Revealed type is "Union[builtins.int, None]" -[case testRedefine2PartialGenericTypes] +[case testNewRedefinePartialGenericTypes] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: a = [] @@ -404,7 +404,7 @@ def f6() -> None: reveal_type(a) # N: Revealed type is "builtins.list[builtins.str]" [builtins fixtures/list.pyi] -[case testRedefine2FinalLiteral] +[case testNewRedefineFinalLiteral] # flags: --allow-redefinition-new --local-partial-types from typing_extensions import Final, Literal @@ -413,7 +413,7 @@ reveal_type(x) # N: Revealed type is "Literal['foo']?" a: Literal["foo"] = x [builtins fixtures/tuple.pyi] -[case testRedefine2AnnotatedVariable] +[case testNewRedefineAnnotatedVariable] # flags: --allow-redefinition-new --local-partial-types from typing import Optional @@ -447,7 +447,7 @@ class C: self.x = "" reveal_type(self.x) # N: Revealed type is "builtins.str" -[case testRedefine2AnyType1] +[case testNewRedefineAnyType1] # flags: --allow-redefinition-new --local-partial-types def a(): pass @@ -508,7 +508,7 @@ def f7() -> None: x = a() reveal_type(x) # N: Revealed type is "builtins.int" -[case testRedefine2AnyType2] +[case testNewRedefineAnyType2] # flags: --allow-redefinition-new --local-partial-types from typing import Any @@ -533,7 +533,7 @@ def f3(x) -> None: reveal_type(x) # N: Revealed type is "Any" reveal_type(x) # N: Revealed type is "Any" -[case tetRedefine2Del] +[case tetNewRedefineDel] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: x = "" @@ -576,7 +576,7 @@ def f5() -> None: continue x = "" reveal_type(x) # N: Revealed type is "builtins.str" -[case testRedefine2WhileLoopSimple] +[case testNewRedefineWhileLoopSimple] # flags: --allow-redefinition-new --local-partial-types def f() -> None: while int(): @@ -594,7 +594,7 @@ def f() -> None: x = [1] reveal_type(x) # N: Revealed type is "builtins.list[builtins.int]" -[case testRedefine2WhileLoopOptional] +[case testNewRedefineWhileLoopOptional] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: x = None @@ -612,7 +612,7 @@ def f2() -> None: x = "" reveal_type(x) # N: Revealed type is "Union[None, builtins.str]" -[case testRedefine2WhileLoopPartialType] +[case testNewRedefineWhileLoopPartialType] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: x = [] @@ -621,7 +621,7 @@ def f1() -> None: reveal_type(x) # N: Revealed type is "builtins.list[builtins.int]" [builtins fixtures/list.pyi] -[case testRedefine2WhileLoopComplex1] +[case testNewRedefineWhileLoopComplex1] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: @@ -632,7 +632,7 @@ def f1() -> None: continue [builtins fixtures/exception.pyi] -[case testRedefine2WhileLoopComplex2] +[case testNewRedefineWhileLoopComplex2] # flags: --allow-redefinition-new --local-partial-types class C: @@ -654,7 +654,7 @@ def f2() -> None: y = "" [builtins fixtures/tuple.pyi] -[case testRedefine2LoopWithMatch] +[case testNewRedefineLoopWithMatch] # flags: --allow-redefinition-new --local-partial-types --python-version 3.10 def f1() -> None: @@ -679,7 +679,7 @@ def f2() -> None: reveal_type(y) # N: Revealed type is "builtins.str" [builtins fixtures/list.pyi] -[case testRedefine2Return] +[case testNewRedefineReturn] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: if int(): @@ -697,7 +697,7 @@ def f2() -> None: return reveal_type(x) # N: Revealed type is "builtins.str" -[case testRedefine2BreakAndContinue] +[case testNewRedefineBreakAndContinue] # flags: --allow-redefinition-new --local-partial-types def b() -> None: while int(): @@ -722,7 +722,7 @@ def c() -> None: reveal_type(x) # N: Revealed type is "None" reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, None]" -[case testRedefine2Underscore] +[case testNewRedefineUnderscore] # flags: --allow-redefinition-new --local-partial-types def f() -> None: if int(): @@ -733,7 +733,7 @@ def f() -> None: reveal_type(_) # N: Revealed type is "builtins.str" reveal_type(_) # N: Revealed type is "Union[builtins.int, builtins.str]" -[case testRedefine2WithStatement] +[case testNewRedefineWithStatement] # flags: --allow-redefinition-new --local-partial-types class C: def __enter__(self) -> int: ... @@ -757,7 +757,7 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "builtins.str" reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" -[case testRedefine2TryStatement] +[case testNewRedefineTryStatement] # flags: --allow-redefinition-new --local-partial-types class E(Exception): pass @@ -819,7 +819,7 @@ def f4() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" [builtins fixtures/exception.pyi] -[case testRedefine2RaiseStatement] +[case testNewRedefineRaiseStatement] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: if int(): @@ -844,7 +844,7 @@ def f2() -> None: [builtins fixtures/exception.pyi] -[case testRedefine2MultipleAssignment] +[case testNewRedefineMultipleAssignment] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: x, y = 1, "" @@ -866,7 +866,7 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" reveal_type(y) # N: Revealed type is "Union[builtins.str, builtins.int]" -[case testRedefine2ForLoopBasics] +[case testNewRedefineForLoopBasics] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: for x in [1]: @@ -888,7 +888,7 @@ def f2() -> None: reveal_type(y) # N: Revealed type is "Union[builtins.str, builtins.int]" [builtins fixtures/for.pyi] -[case testRedefine2ForLoop1] +[case testNewRedefineForLoop1] # flags: --allow-redefinition-new --local-partial-types def l() -> list[int]: return [] @@ -911,7 +911,7 @@ def f3() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" [builtins fixtures/for.pyi] -[case testRedefine2ForLoop2] +[case testNewRedefineForLoop2] # flags: --allow-redefinition-new --local-partial-types from typing import Any @@ -921,7 +921,7 @@ def f(a: Any) -> None: return [builtins fixtures/isinstance.pyi] -[case testRedefine2ForStatementIndexNarrowing] +[case testNewRedefineForStatementIndexNarrowing] # flags: --allow-redefinition-new --local-partial-types from typing_extensions import TypedDict @@ -950,7 +950,7 @@ for b in ("hourly", "daily"): reveal_type(b.upper()) # N: Revealed type is "builtins.str" [builtins fixtures/for.pyi] -[case testRedefine2ForLoopIndexWidening] +[case testNewRedefineForLoopIndexWidening] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: @@ -975,7 +975,7 @@ def f3() -> None: x = "" reveal_type(x) # N: Revealed type is "builtins.str" -[case testRedefine2VariableAnnotatedInLoop] +[case testNewRedefineVariableAnnotatedInLoop] # flags: --allow-redefinition-new --local-partial-types --enable-error-code=redundant-expr from typing import Optional @@ -1000,7 +1000,7 @@ def f2(e: Optional[str]) -> None: reveal_type(e) # N: Revealed type is "Union[builtins.str, None]" reveal_type(e) # N: Revealed type is "Union[builtins.str, None]" -[case testRedefine2LoopAndPartialTypesSpecialCase] +[case testNewRedefineLoopAndPartialTypesSpecialCase] # flags: --allow-redefinition-new --local-partial-types def f() -> list[str]: a = [] # type: ignore From b8a106d0ebfb1f5800c9a1d635698415c25e8e56 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 20 Feb 2025 15:01:08 +0000 Subject: [PATCH 50/80] Suppress from --help, since this is experimental --- mypy/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/main.py b/mypy/main.py index b3cb02f7878c..2a2181783eaa 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -867,7 +867,7 @@ def add_invertible_flag( "--allow-redefinition-new", default=False, strict_flag=False, - help="Allow flexible variable redefinition with a new type", + help=argparse.SUPPRESS, # This is still very experimental group=strictness_group, ) From bf75cc44ae4968cddf521bf98c1b6458ca70e6b3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 20 Feb 2025 15:11:51 +0000 Subject: [PATCH 51/80] Add and update tests --- test-data/unit/check-redefine2.test | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 637e6fe5ab19..cc275f714463 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -406,7 +406,7 @@ def f6() -> None: [case testNewRedefineFinalLiteral] # flags: --allow-redefinition-new --local-partial-types -from typing_extensions import Final, Literal +from typing import Final, Literal x: Final = "foo" reveal_type(x) # N: Revealed type is "Literal['foo']?" @@ -923,7 +923,7 @@ def f(a: Any) -> None: [case testNewRedefineForStatementIndexNarrowing] # flags: --allow-redefinition-new --local-partial-types -from typing_extensions import TypedDict +from typing import TypedDict class X(TypedDict): hourly: int @@ -949,6 +949,7 @@ for b in ("hourly", "daily"): reveal_type(b) # N: Revealed type is "builtins.str" reveal_type(b.upper()) # N: Revealed type is "builtins.str" [builtins fixtures/for.pyi] +[typing fixtures/typing-full.pyi] [case testNewRedefineForLoopIndexWidening] # flags: --allow-redefinition-new --local-partial-types @@ -1016,3 +1017,14 @@ def f() -> list[str]: o.append(line) return o [builtins fixtures/list.pyi] + +[case testNewRedefineFinalVariable] +# flags: --allow-redefinition-new --local-partial-types +from typing import Final + +x: Final = "foo" +x = 1 # E: Cannot assign to final name "x" + +class C: + y: Final = "foo" + y = 1 # E: Cannot assign to final name "y" From 958535321e14c3c7647c89212780dbb72866c13a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 20 Feb 2025 15:15:43 +0000 Subject: [PATCH 52/80] Minor tweaks --- mypy/binder.py | 7 +++++-- mypy/build.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mypy/binder.py b/mypy/binder.py index fc68a3aef47f..5ba047f1ea9d 100644 --- a/mypy/binder.py +++ b/mypy/binder.py @@ -99,8 +99,6 @@ class A: type_assignments: Assigns | None = None def __init__(self, bind_all: bool) -> None: - self.bind_all = bind_all - # Each frame gets an increasing, distinct id. self.next_id = 1 @@ -134,6 +132,11 @@ def __init__(self, bind_all: bool) -> None: self.break_frames: list[int] = [] self.continue_frames: list[int] = [] + # If True, initial assignment to a simple variable (e.g. "x", but not "x.y") + # is added to the binder. This allows more precise narrowing and more + # flexible inference of variable types. + self.bind_all = bind_all + def _get_id(self) -> int: self.next_id += 1 return self.next_id diff --git a/mypy/build.py b/mypy/build.py index aff8fa045e0e..355ba861385e 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -2241,7 +2241,7 @@ def semantic_analysis_pass1(self) -> None: self.tree.names = SymbolTable() if not self.tree.is_stub: if not self.options.allow_redefinition_new: - # Always perform some low-key variable renaming when assignments can't + # Perform some low-key variable renaming when assignments can't # widen inferred types self.tree.accept(LimitedVariableRenameVisitor()) if options.allow_redefinition: From 0efa85d41e9584a5fcb166994e6cf9d5d1cc726a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 20 Feb 2025 15:19:26 +0000 Subject: [PATCH 53/80] Refactor --- mypy/binder.py | 5 +++-- mypy/checker.py | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mypy/binder.py b/mypy/binder.py index 5ba047f1ea9d..f1c81eb3bfc1 100644 --- a/mypy/binder.py +++ b/mypy/binder.py @@ -28,6 +28,7 @@ get_proper_type, ) from mypy.typevars import fill_typevars_with_any +from mypy.options import Options BindableExpression: _TypeAlias = Union[IndexExpr, MemberExpr, NameExpr] @@ -98,7 +99,7 @@ class A: # This maps an expression to a list of bound types for every item in the union type. type_assignments: Assigns | None = None - def __init__(self, bind_all: bool) -> None: + def __init__(self, options: Options) -> None: # Each frame gets an increasing, distinct id. self.next_id = 1 @@ -135,7 +136,7 @@ def __init__(self, bind_all: bool) -> None: # If True, initial assignment to a simple variable (e.g. "x", but not "x.y") # is added to the binder. This allows more precise narrowing and more # flexible inference of variable types. - self.bind_all = bind_all + self.bind_all = options.allow_redefinition_new def _get_id(self) -> int: self.next_id += 1 diff --git a/mypy/checker.py b/mypy/checker.py index 3973e844a658..91222d84a349 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -378,7 +378,7 @@ def __init__( self.plugin = plugin self.tscope = Scope() self.scope = CheckerScope(tree) - self.binder = ConditionalTypeBinder(bind_all=options.allow_redefinition_new) + self.binder = ConditionalTypeBinder(options) self.globals = tree.names self.return_types = [] self.dynamic_funcs = [] @@ -433,7 +433,7 @@ def reset(self) -> None: # TODO: verify this is still actually worth it over creating new checkers self.partial_reported.clear() self.module_refs.clear() - self.binder = ConditionalTypeBinder(bind_all=self.options.allow_redefinition_new) + self.binder = ConditionalTypeBinder(self.options) self._type_maps[1:] = [] self._type_maps[0].clear() self.temp_type_map = None @@ -1200,7 +1200,7 @@ def check_func_def( original_typ = typ for item, typ in expanded: old_binder = self.binder - self.binder = ConditionalTypeBinder(bind_all=self.options.allow_redefinition_new) + self.binder = ConditionalTypeBinder(self.options) with self.binder.top_frame_context(): defn.expanded.append(item) @@ -2584,7 +2584,7 @@ def visit_class_def(self, defn: ClassDef) -> None: self.fail(message_registry.CANNOT_INHERIT_FROM_FINAL.format(base.name), defn) with self.tscope.class_scope(defn.info), self.enter_partial_types(is_class=True): old_binder = self.binder - self.binder = ConditionalTypeBinder(bind_all=self.options.allow_redefinition_new) + self.binder = ConditionalTypeBinder(self.options) with self.binder.top_frame_context(): with self.scope.push_class(defn.info): self.accept(defn.defs) From 4af080c6c862477b618fa8f7018d06e69aa60b90 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 20 Feb 2025 15:20:12 +0000 Subject: [PATCH 54/80] Black and ruff --- mypy/binder.py | 4 ++-- mypy/checker.py | 60 +++++++++++++++++++++++++++++++++---------------- mypy/main.py | 6 ++++- 3 files changed, 48 insertions(+), 22 deletions(-) diff --git a/mypy/binder.py b/mypy/binder.py index f1c81eb3bfc1..6a72d8a96f23 100644 --- a/mypy/binder.py +++ b/mypy/binder.py @@ -7,8 +7,9 @@ from typing_extensions import TypeAlias as _TypeAlias from mypy.erasetype import remove_instance_last_known_values -from mypy.literals import Key, literal, literal_hash, subkeys, extract_var_from_literal_hash +from mypy.literals import Key, extract_var_from_literal_hash, literal, literal_hash, subkeys from mypy.nodes import Expression, IndexExpr, MemberExpr, NameExpr, RefExpr, TypeInfo, Var +from mypy.options import Options from mypy.subtypes import is_same_type, is_subtype from mypy.typeops import make_simplified_union from mypy.types import ( @@ -28,7 +29,6 @@ get_proper_type, ) from mypy.typevars import fill_typevars_with_any -from mypy.options import Options BindableExpression: _TypeAlias = Union[IndexExpr, MemberExpr, NameExpr] diff --git a/mypy/checker.py b/mypy/checker.py index 91222d84a349..d0c46e9d4ca8 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -613,13 +613,17 @@ def accept_loop( self.accept(body) partials_new = sum(len(pts.map) for pts in self.partial_types) widened_new = len(self.widened_vars) - if (partials_new == partials_old) and not self.binder.last_pop_changed and (widened_new == widened_old or iter > 1): + if ( + (partials_new == partials_old) + and not self.binder.last_pop_changed + and (widened_new == widened_old or iter > 1) + ): break partials_old = partials_new widened_old = widened_new iter += 1 if iter == 20: - raise RuntimeError(f"Too many iterations when checking a loop") + raise RuntimeError("Too many iterations when checking a loop") # If necessary, reset the modified options and make up for the postponed error checks: self.options.warn_unreachable = warn_unreachable @@ -3242,7 +3246,9 @@ def check_assignment( return var = lvalue_type.var - if is_valid_inferred_type(rvalue_type, self.options, is_lvalue_final=var.is_final): + if is_valid_inferred_type( + rvalue_type, self.options, is_lvalue_final=var.is_final + ): partial_types = self.find_partial_types(var) if partial_types is not None: if not self.current_node_deferred: @@ -3309,7 +3315,8 @@ def check_assignment( self.set_inferred_type(lvalue.node, lvalue, lvalue_type) rvalue_type, lvalue_type = self.check_simple_assignment( - lvalue_type, rvalue, context=rvalue, inferred=inferred, lvalue=lvalue) + lvalue_type, rvalue, context=rvalue, inferred=inferred, lvalue=lvalue + ) inferred = None # Special case: only non-abstract non-protocol classes can be assigned to @@ -3341,7 +3348,7 @@ def check_assignment( and lvalue.node.is_inferred and lvalue.node.is_index_var and lvalue_type is not None - and not self.options.allow_redefinition_new # TODO WHAT + and not self.options.allow_redefinition_new # TODO WHAT ): lvalue.node.type = remove_instance_last_known_values(lvalue_type) elif self.options.allow_redefinition_new and lvalue_type is not None: @@ -3427,7 +3434,9 @@ def try_infer_partial_generic_type_from_assignment( rvalue_type = self.expr_checker.accept(rvalue) rvalue_type = get_proper_type(rvalue_type) if isinstance(rvalue_type, Instance): - if rvalue_type.type == typ.type and is_valid_inferred_type(rvalue_type, self.options): + if rvalue_type.type == typ.type and is_valid_inferred_type( + rvalue_type, self.options + ): var.type = rvalue_type del partial_types[var] elif isinstance(rvalue_type, AnyType): @@ -4311,7 +4320,11 @@ def check_lvalue(self, lvalue: Lvalue) -> tuple[Type | None, IndexExpr | None, V self.store_type(lvalue, lvalue_type) elif isinstance(lvalue, NameExpr): lvalue_type = self.expr_checker.analyze_ref_expr(lvalue, lvalue=True) - if isinstance(lvalue.node, Var) and lvalue.node.is_inferred and self.options.allow_redefinition_new: + if ( + isinstance(lvalue.node, Var) + and lvalue.node.is_inferred + and self.options.allow_redefinition_new + ): inferred = lvalue.node self.store_type(lvalue, lvalue_type) elif isinstance(lvalue, (TupleExpr, ListExpr)): @@ -4353,9 +4366,12 @@ def infer_variable_type( if isinstance(init_type, DeletedType): self.msg.deleted_as_rvalue(init_type, context) elif ( - not is_valid_inferred_type(init_type, self.options, - is_lvalue_final=name.is_final, - is_lvalue_member=isinstance(lvalue, MemberExpr)) + not is_valid_inferred_type( + init_type, + self.options, + is_lvalue_final=name.is_final, + is_lvalue_member=isinstance(lvalue, MemberExpr), + ) and not self.no_partial_types ): # We cannot use the type of the initialization expression for full type @@ -4387,7 +4403,9 @@ def infer_variable_type( def infer_partial_type(self, name: Var, lvalue: Lvalue, init_type: Type) -> bool: init_type = get_proper_type(init_type) - if isinstance(init_type, NoneType) and (isinstance(lvalue, MemberExpr) or not self.options.allow_redefinition_new): + if isinstance(init_type, NoneType) and ( + isinstance(lvalue, MemberExpr) or not self.options.allow_redefinition_new + ): partial_type = PartialType(None, name) elif isinstance(init_type, Instance): fullname = init_type.type.fullname @@ -4535,8 +4553,9 @@ def check_simple_assignment( rvalue, type_context=type_context, always_allow_any=always_allow_any ) if ( - lvalue_type is not None and type_context is None and - not is_valid_inferred_type(rvalue_type, self.options) # TODO + lvalue_type is not None + and type_context is None + and not is_valid_inferred_type(rvalue_type, self.options) # TODO ): rvalue_type = self.expr_checker.accept( rvalue, type_context=lvalue_type, always_allow_any=always_allow_any @@ -4548,7 +4567,9 @@ def check_simple_assignment( new_inferred = rvalue_type if not is_same_type(inferred.type, new_inferred): lvalue_type = make_simplified_union([inferred.type, new_inferred]) - if not is_same_type(lvalue_type, inferred.type) and not isinstance(inferred.type, PartialType): + if not is_same_type(lvalue_type, inferred.type) and not isinstance( + inferred.type, PartialType + ): self.widened_vars.append(inferred.name) self.set_inferred_type(inferred, lvalue, lvalue_type) self.binder.put(lvalue, rvalue_type) @@ -4758,7 +4779,9 @@ def check_indexed_assignment( if isinstance(res_type, UninhabitedType) and not res_type.ambiguous: self.binder.unreachable() - def replace_partial_type(self, var: Var, new_type: Type, partial_types: dict[Var, Context]) -> None: + def replace_partial_type( + self, var: Var, new_type: Type, partial_types: dict[Var, Context] + ) -> None: var.type = new_type del partial_types[var] if self.options.allow_redefinition_new: @@ -8548,10 +8571,9 @@ def _find_inplace_method(inst: Instance, method: str, operator: str) -> str | No return None -def is_valid_inferred_type(typ: Type, - options: Options, - is_lvalue_final: bool = False, - is_lvalue_member: bool = False) -> bool: +def is_valid_inferred_type( + typ: Type, options: Options, is_lvalue_final: bool = False, is_lvalue_member: bool = False +) -> bool: """Is an inferred type valid and needs no further refinement? Examples of invalid types include the None type (when we are not assigning diff --git a/mypy/main.py b/mypy/main.py index 2a2181783eaa..f5abbdba8361 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -94,7 +94,11 @@ def main( ) if options.allow_redefinition_new and not options.local_partial_types: - fail("error: --local-partial-types must be used if using --allow-redefinition-new", stderr, options) + fail( + "error: --local-partial-types must be used if using --allow-redefinition-new", + stderr, + options, + ) if options.install_types and (stdout is not sys.stdout or stderr is not sys.stderr): # Since --install-types performs user input, we want regular stdout and stderr. From c478514e0d99a0b1f855abd84f7ec6b56d9473cd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 20 Feb 2025 15:31:39 +0000 Subject: [PATCH 55/80] More tweaks --- mypy/checker.py | 9 +++++++-- mypy/main.py | 2 +- mypy/semanal.py | 2 ++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index d0c46e9d4ca8..c35e066ece65 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4321,9 +4321,9 @@ def check_lvalue(self, lvalue: Lvalue) -> tuple[Type | None, IndexExpr | None, V elif isinstance(lvalue, NameExpr): lvalue_type = self.expr_checker.analyze_ref_expr(lvalue, lvalue=True) if ( - isinstance(lvalue.node, Var) + self.options.allow_redefinition_new + and isinstance(lvalue.node, Var) and lvalue.node.is_inferred - and self.options.allow_redefinition_new ): inferred = lvalue.node self.store_type(lvalue, lvalue_type) @@ -4406,6 +4406,8 @@ def infer_partial_type(self, name: Var, lvalue: Lvalue, init_type: Type) -> bool if isinstance(init_type, NoneType) and ( isinstance(lvalue, MemberExpr) or not self.options.allow_redefinition_new ): + # When using --allow-redefinition-new, None types aren't special + # when inferring simple variable types. partial_type = PartialType(None, name) elif isinstance(init_type, Instance): fullname = init_type.type.fullname @@ -4782,9 +4784,12 @@ def check_indexed_assignment( def replace_partial_type( self, var: Var, new_type: Type, partial_types: dict[Var, Context] ) -> None: + """Replace the partial type of var with a non-partial type.""" var.type = new_type del partial_types[var] if self.options.allow_redefinition_new: + # When using --allow-redefinition-new, binder tracks all types of + # simple variables. n = NameExpr(var.name) n.node = var self.binder.assign_type(n, new_type, new_type) diff --git a/mypy/main.py b/mypy/main.py index f5abbdba8361..ad836a5ddc19 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -95,7 +95,7 @@ def main( if options.allow_redefinition_new and not options.local_partial_types: fail( - "error: --local-partial-types must be used if using --allow-redefinition-new", + "error: --local-partial-types must be enabled if using --allow-redefinition-new", stderr, options, ) diff --git a/mypy/semanal.py b/mypy/semanal.py index 56755df59a78..07ff0c305f49 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4338,6 +4338,8 @@ def analyze_name_lvalue( if self.is_func_scope(): if unmangle(name) == "_" and not self.options.allow_redefinition_new: # Special case for assignment to local named '_': always infer 'Any'. + # This isn't needed with --allow-redefinition-new, since arbitrary + # types can be assigned to '_' anyway. typ = AnyType(TypeOfAny.special_form) self.store_declared_types(lvalue, typ) if is_final and self.is_final_redefinition(kind, name): From fb9421fc3f8524fdca584a7a365029a883f37a25 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 21 Feb 2025 14:43:55 +0000 Subject: [PATCH 56/80] Prevent wideding of variable defined in outer scope --- mypy/checker.py | 29 +++++++++++++++++------------ test-data/unit/check-redefine2.test | 12 ++++++++++++ 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index c35e066ece65..1c011aa948d9 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4567,18 +4567,23 @@ def check_simple_assignment( new_inferred = remove_instance_last_known_values(rvalue_type) else: new_inferred = rvalue_type - if not is_same_type(inferred.type, new_inferred): - lvalue_type = make_simplified_union([inferred.type, new_inferred]) - if not is_same_type(lvalue_type, inferred.type) and not isinstance( - inferred.type, PartialType - ): - self.widened_vars.append(inferred.name) - self.set_inferred_type(inferred, lvalue, lvalue_type) - self.binder.put(lvalue, rvalue_type) - # TODO: hack, maybe integrate into put? - lit = literal_hash(lvalue) - if lit is not None: - self.binder.declarations[lit] = lvalue_type + if isinstance(lvalue, NameExpr) and not is_same_type(inferred.type, new_inferred): + # Should we widen the inferred type or the lvalue? + different_scopes = ( + self.scope.top_function() is not None and lvalue.kind != LDEF + ) + if not different_scopes: + lvalue_type = make_simplified_union([inferred.type, new_inferred]) + if not is_same_type(lvalue_type, inferred.type) and not isinstance( + inferred.type, PartialType + ): + self.widened_vars.append(inferred.name) + self.set_inferred_type(inferred, lvalue, lvalue_type) + self.binder.put(lvalue, rvalue_type) + # TODO: hack, maybe integrate into put? + lit = literal_hash(lvalue) + if lit is not None: + self.binder.declarations[lit] = lvalue_type if ( isinstance(get_proper_type(lvalue_type), UnionType) # Skip literal types, as they have special logic (for better errors). diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index cc275f714463..8aee3166fe00 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -118,6 +118,18 @@ def f2() -> None: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" +[case testNewRedefineGlobalVariableNoneInit] +# flags: --allow-redefinition-new --local-partial-types +x = None + +def f() -> None: + global x + reveal_type(x) # N: Revealed type is "None" + x = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "None") + reveal_type(x) # N: Revealed type is "None" + +reveal_type(x) # N: Revealed type is "None" + [case testNewRedefineParameterTypes] # flags: --allow-redefinition-new --local-partial-types from typing import Optional From a29a3dcfc08716e0060e01f4d2ef48515724305d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 21 Feb 2025 15:16:30 +0000 Subject: [PATCH 57/80] Add comment --- mypy/checker.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy/checker.py b/mypy/checker.py index 1c011aa948d9..f4da87e33ac5 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -596,6 +596,9 @@ def accept_loop( # Check for potential decreases in the number of partial types so as not to stop the # iteration too early: partials_old = sum(len(pts.map) for pts in self.partial_types) + # Check if assignment widened the inferred type of a variable; in this case we + # need to iterate again (we only do one extra iteration, since this could go + # on without bound otherwise) widened_old = len(self.widened_vars) # Disable error types that we cannot safely identify in intermediate iteration steps: From ff7557bbeb5c9e180205e90dac4aac76a2512f8c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 21 Feb 2025 15:19:28 +0000 Subject: [PATCH 58/80] Fix issue --- mypy/checker.py | 1 - test-data/unit/check-redefine2.test | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index f4da87e33ac5..5578b5203793 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3351,7 +3351,6 @@ def check_assignment( and lvalue.node.is_inferred and lvalue.node.is_index_var and lvalue_type is not None - and not self.options.allow_redefinition_new # TODO WHAT ): lvalue.node.type = remove_instance_last_known_values(lvalue_type) elif self.options.allow_redefinition_new and lvalue_type is not None: diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 8aee3166fe00..e425ca029ed2 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -949,9 +949,9 @@ for a in ("hourly", "daily"): c = a reveal_type(c) # N: Revealed type is "builtins.str" a = "monthly" - reveal_type(a) # N: Revealed type is "Union[Literal['hourly']?, Literal['daily']?]" + reveal_type(a) # N: Revealed type is "builtins.str" a = "yearly" - reveal_type(a) # N: Revealed type is "Union[Literal['hourly']?, Literal['daily']?]" + reveal_type(a) # N: Revealed type is "builtins.str" a = 1 reveal_type(a) # N: Revealed type is "builtins.int" reveal_type(a) # N: Revealed type is "builtins.int" From 10b6082461169651ce0c59afd581dcbc88a69f21 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 21 Feb 2025 15:37:37 +0000 Subject: [PATCH 59/80] Some polish --- mypy/checker.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 5578b5203793..450b9ee4be2b 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4559,18 +4559,21 @@ def check_simple_assignment( if ( lvalue_type is not None and type_context is None - and not is_valid_inferred_type(rvalue_type, self.options) # TODO + and not is_valid_inferred_type(rvalue_type, self.options) ): + # Inference in an empty type context didn't produce a valid type, so + # try using lvalue type as context instead. rvalue_type = self.expr_checker.accept( rvalue, type_context=lvalue_type, always_allow_any=always_allow_any ) - if inferred is not None and lvalue is not None and inferred.type is not None: + if isinstance(lvalue, NameExpr) and inferred is not None and inferred.type is not None: if not inferred.is_final: new_inferred = remove_instance_last_known_values(rvalue_type) else: new_inferred = rvalue_type - if isinstance(lvalue, NameExpr) and not is_same_type(inferred.type, new_inferred): - # Should we widen the inferred type or the lvalue? + if not is_same_type(inferred.type, new_inferred): + # Should we widen the inferred type or the lvalue? We can only widen + # a variable type if the variable was defined in the current function. different_scopes = ( self.scope.top_function() is not None and lvalue.kind != LDEF ) @@ -4579,6 +4582,7 @@ def check_simple_assignment( if not is_same_type(lvalue_type, inferred.type) and not isinstance( inferred.type, PartialType ): + # Widen the type to the union of original and new type. self.widened_vars.append(inferred.name) self.set_inferred_type(inferred, lvalue, lvalue_type) self.binder.put(lvalue, rvalue_type) From e8789e066921b009b2ed3186136192b675a0e00f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 21 Feb 2025 15:40:32 +0000 Subject: [PATCH 60/80] Don't widen final variables --- mypy/checker.py | 12 +++++++----- test-data/unit/check-redefine2.test | 11 +++++++++-- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 450b9ee4be2b..4f1dc9bbb835 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4566,11 +4566,13 @@ def check_simple_assignment( rvalue_type = self.expr_checker.accept( rvalue, type_context=lvalue_type, always_allow_any=always_allow_any ) - if isinstance(lvalue, NameExpr) and inferred is not None and inferred.type is not None: - if not inferred.is_final: - new_inferred = remove_instance_last_known_values(rvalue_type) - else: - new_inferred = rvalue_type + if ( + isinstance(lvalue, NameExpr) + and inferred is not None + and inferred.type is not None + and not inferred.is_final + ): + new_inferred = remove_instance_last_known_values(rvalue_type) if not is_same_type(inferred.type, new_inferred): # Should we widen the inferred type or the lvalue? We can only widen # a variable type if the variable was defined in the current function. diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index e425ca029ed2..a5d8f33f0fd5 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -423,6 +423,11 @@ from typing import Final, Literal x: Final = "foo" reveal_type(x) # N: Revealed type is "Literal['foo']?" a: Literal["foo"] = x + +class B: + x: Final = "bar" + a: Literal["bar"] = x +reveal_type(B.x) # N: Revealed type is "Literal['bar']?" [builtins fixtures/tuple.pyi] [case testNewRedefineAnnotatedVariable] @@ -1035,8 +1040,10 @@ def f() -> list[str]: from typing import Final x: Final = "foo" -x = 1 # E: Cannot assign to final name "x" +x = 1 # E: Cannot assign to final name "x" \ + # E: Incompatible types in assignment (expression has type "int", variable has type "str") class C: y: Final = "foo" - y = 1 # E: Cannot assign to final name "y" + y = 1 # E: Cannot assign to final name "y" \ + # E: Incompatible types in assignment (expression has type "int", variable has type "str") From 9494ddc0714420ec4e086a6801c60e31568947c5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 21 Feb 2025 16:03:35 +0000 Subject: [PATCH 61/80] Fix incremental mode --- mypy/options.py | 1 + test-data/unit/check-incremental.test | 11 +++++++++++ test-data/unit/check-redefine2.test | 20 ++++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/mypy/options.py b/mypy/options.py index 9eef53872ad2..27b583722568 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -22,6 +22,7 @@ class BuildType: PER_MODULE_OPTIONS: Final = { # Please keep this list sorted "allow_redefinition", + "allow_redefinition_new", "allow_untyped_globals", "always_false", "always_true", diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 0c7e67e5444d..26ef6cb589ed 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -6855,3 +6855,14 @@ from .lib import NT [builtins fixtures/tuple.pyi] [out] [out2] + +[case testNewRedefineAffectsCache] +# flags: --local-partial-types --allow-redefinition-new +# flags2: --local-partial-types +# flags3: --local-partial-types --allow-redefinition-new +x = 0 +if int(): + x = "" +[out] +[out2] +main:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index a5d8f33f0fd5..c7a150e7bd42 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -1047,3 +1047,23 @@ class C: y: Final = "foo" y = 1 # E: Cannot assign to final name "y" \ # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testNewRedefineEnableUsingComment] +# flags: --local-partial-types +import a +import b + +[file a.py] +# mypy: allow-redefinition-new +if int(): + x = 0 +else: + x = "" +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +[file b.py] +if int(): + x = 0 +else: + x = "" # E: Incompatible types in assignment (expression has type "str", variable has type "int") +reveal_type(x) # N: Revealed type is "builtins.int" From fed6d4535bb93b5b7139c9d9eeabbf0a83ff91f8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 21 Feb 2025 16:04:44 +0000 Subject: [PATCH 62/80] Simplify --- mypy/semanal.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 07ff0c305f49..f87478ef63de 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3764,8 +3764,7 @@ def process_type_annotation(self, s: AssignmentStmt) -> None: # decide here what to infer: int or Literal[42]. safe_literal_inference = self.type.mro[1].get(ref_expr.name) is None if safe_literal_inference and ref_expr.is_inferred_def: - if not self.options.allow_redefinition_new or s.is_final_def: - s.type = self.analyze_simple_literal_type(s.rvalue, s.is_final_def) + s.type = self.analyze_simple_literal_type(s.rvalue, s.is_final_def) if s.type: # Store type into nodes. for lvalue in s.lvalues: From aaefd4b60a3c71c0ba239a838ae01456af2da998 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 21 Feb 2025 16:11:34 +0000 Subject: [PATCH 63/80] Detect invalid per-module options --- mypy/semanal.py | 7 +++++++ test-data/unit/check-redefine2.test | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/mypy/semanal.py b/mypy/semanal.py index f87478ef63de..e301ac1e5120 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -658,6 +658,13 @@ def refresh_partial( def refresh_top_level(self, file_node: MypyFile) -> None: """Reanalyze a stale module top-level in fine-grained incremental mode.""" + if self.options.allow_redefinition_new and not self.options.local_partial_types: + n = TempNode(AnyType(TypeOfAny.special_form)) + n.line = 1 + n.column = 0 + n.end_line = 1 + n.end_column = 0 + self.fail("--local-partial-types must be enabled if using --allow-redefinition-new", n) self.recurse_into_functions = False self.add_implicit_module_attrs(file_node) for d in file_node.defs: diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index c7a150e7bd42..6ef9af0d1a10 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -1067,3 +1067,22 @@ if int(): else: x = "" # E: Incompatible types in assignment (expression has type "str", variable has type "int") reveal_type(x) # N: Revealed type is "builtins.int" + +[case testNewRedefineWithoutLocalPartialTypes] +import a +import b + +[file a.py] +# mypy: local-partial-types, allow-redefinition-new +x = 0 +if int(): + x = "" + +[file b.py] +# mypy: allow-redefinition-new +x = 0 +if int(): + x = "" + +[out] +tmp/b.py:1: error: --local-partial-types must be enabled if using --allow-redefinition-new From d7c35e499068590074eba08e1f3f41ff51b89871 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 24 Feb 2025 14:58:59 +0000 Subject: [PATCH 64/80] Fix typo --- mypy/checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 4f1dc9bbb835..74662fca213a 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4577,7 +4577,7 @@ def check_simple_assignment( # Should we widen the inferred type or the lvalue? We can only widen # a variable type if the variable was defined in the current function. different_scopes = ( - self.scope.top_function() is not None and lvalue.kind != LDEF + self.scope.top_level_function() is not None and lvalue.kind != LDEF ) if not different_scopes: lvalue_type = make_simplified_union([inferred.type, new_inferred]) From 5bd6defcc6e2eb0f47e59f4bee35a51db5a27725 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 24 Feb 2025 17:05:13 +0000 Subject: [PATCH 65/80] Skip failing match statement test on 3.9 --- test-data/unit/check-python310.test | 25 +++++++++++++++++++++++++ test-data/unit/check-redefine2.test | 25 ------------------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 66d34b63919b..92fde5615816 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -2565,3 +2565,28 @@ def f1(x: int | str | list[bytes]) -> None: case [y]: reveal_type(y) # N: Revealed type is "builtins.bytes" reveal_type(y) # N: Revealed type is "Union[builtins.str, builtins.bytes]" + +[case testNewRedefineLoopWithMatch] +# flags: --allow-redefinition-new --local-partial-types + +def f1() -> None: + while True: + x = object() + match x: + case str(y): + pass + case int(): + pass + if int(): + continue + +def f2() -> None: + for x in [""]: + match str(): + case "a": + y = "" + case "b": + y = 1 + return + reveal_type(y) # N: Revealed type is "builtins.str" +[builtins fixtures/list.pyi] diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 6ef9af0d1a10..2380ccb510a1 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -671,31 +671,6 @@ def f2() -> None: y = "" [builtins fixtures/tuple.pyi] -[case testNewRedefineLoopWithMatch] -# flags: --allow-redefinition-new --local-partial-types --python-version 3.10 - -def f1() -> None: - while True: - x = object() - match x: - case str(y): - pass - case int(): - pass - if int(): - continue - -def f2() -> None: - for x in [""]: - match str(): - case "a": - y = "" - case "b": - y = 1 - return - reveal_type(y) # N: Revealed type is "builtins.str" -[builtins fixtures/list.pyi] - [case testNewRedefineReturn] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: From 3e38e7d24ec06675d5b8b8df0b18c2e25d62f3c6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 27 Feb 2025 11:14:52 +0000 Subject: [PATCH 66/80] Don't infer infinitely complex types in loops --- mypy/checker.py | 2 +- test-data/unit/check-redefine2.test | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 74662fca213a..7dbceb23414b 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -618,7 +618,7 @@ def accept_loop( widened_new = len(self.widened_vars) if ( (partials_new == partials_old) - and not self.binder.last_pop_changed + and (not self.binder.last_pop_changed or iter > 3) and (widened_new == widened_old or iter > 1) ): break diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 2380ccb510a1..67eaca6e0456 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -1061,3 +1061,16 @@ if int(): [out] tmp/b.py:1: error: --local-partial-types must be enabled if using --allow-redefinition-new + +[case testNewRedefineNestedLoopInfiniteExpansion] +# flags: --allow-redefinition-new --local-partial-types +def a(): ... + +def f() -> None: + while int(): + x = a() + + while int(): + x = [x] + + reveal_type(x) # N: Revealed type is "Union[Any, builtins.list[Any], builtins.list[Union[Any, builtins.list[Any]]], builtins.list[Union[Any, builtins.list[Any], builtins.list[Union[Any, builtins.list[Any]]]]], builtins.list[Union[Any, builtins.list[Any], builtins.list[Union[Any, builtins.list[Any]]], builtins.list[Union[Any, builtins.list[Any], builtins.list[Union[Any, builtins.list[Any]]]]]]]]" From 65f4785b5bc4e4b9dec85dd9be97c8e2c232660e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Mar 2025 12:55:04 +0000 Subject: [PATCH 67/80] WIP add failing test cases --- test-data/unit/check-redefine2.test | 35 +++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 67eaca6e0456..eec529199681 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -1074,3 +1074,38 @@ def f() -> None: x = [x] reveal_type(x) # N: Revealed type is "Union[Any, builtins.list[Any], builtins.list[Union[Any, builtins.list[Any]]], builtins.list[Union[Any, builtins.list[Any], builtins.list[Union[Any, builtins.list[Any]]]]], builtins.list[Union[Any, builtins.list[Any], builtins.list[Union[Any, builtins.list[Any]]], builtins.list[Union[Any, builtins.list[Any], builtins.list[Union[Any, builtins.list[Any]]]]]]]]" + +[case testNewRedefinePartialNoneEmptyList] +# flags: --allow-redefinition-new --local-partial-types +def func() -> None: + l = None + + if int(): + l = [] + l.append(1) + +[case testNewRedefineNarrowingSpecialCase] +# flags: --allow-redefinition-new --local-partial-types --warn-unreachable +from typing import Any, Union + +def get() -> Union[tuple[Any, Any], tuple[None, None]]: ... + +def func() -> None: + x, _ = get() + reveal_type(x) # N: Revealed type is "Union[Any, None]" + if x and int(): + ... + reveal_type(x) # N: Revealed type is "Union[Any, None]" + if x and int(): + ... +[builtins fixtures/tuple.pyi] + +[case testNewRedefineAsteriskAssignment] +# flags: --allow-redefinition-new --local-partial-types --warn-unreachable + +def blah() -> tuple[int]: + return (42,) + +def main() -> None: + x, *_ = blah() # E: Need type annotation for "_" (hint: "_: list[] = ...") +[builtins fixtures/tuple.pyi] From 179b846e56aaf2f50353f0bb8429c3469b3550ef Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Mar 2025 13:42:25 +0000 Subject: [PATCH 68/80] Avoid inferring a partial type for "_" --- mypy/checker.py | 2 +- test-data/unit/check-redefine2.test | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 7dbceb23414b..7b23aa29b2e8 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4380,7 +4380,7 @@ def infer_variable_type( # inference (it's not specific enough), but we might be able to give # partial type which will be made more specific later. A partial type # gets generated in assignment like 'x = []' where item type is not known. - if not self.infer_partial_type(name, lvalue, init_type): + if name.name != "_" and not self.infer_partial_type(name, lvalue, init_type): self.msg.need_annotation_for_var(name, context, self.options.python_version) self.set_inference_error_fallback_type(name, lvalue, init_type) elif ( diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index eec529199681..88fcbcc7a95f 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -1100,12 +1100,23 @@ def func() -> None: ... [builtins fixtures/tuple.pyi] -[case testNewRedefineAsteriskAssignment] +[case testNewRedefinePartialTypeForUnderscore] # flags: --allow-redefinition-new --local-partial-types --warn-unreachable -def blah() -> tuple[int]: +def t() -> tuple[int]: return (42,) -def main() -> None: - x, *_ = blah() # E: Need type annotation for "_" (hint: "_: list[] = ...") +def f1() -> None: + # Underscore is slightly special to preserve backward compatibility + x, *_ = t() + reveal_type(x) # N: Revealed type is "builtins.int" + +def f2() -> None: + x, *y = t() # E: Need type annotation for "y" (hint: "y: List[] = ...") + +def f3() -> None: + x, _ = 1, [] + +def f4() -> None: + a, b = 1, [] # E: Need type annotation for "b" (hint: "b: List[] = ...") [builtins fixtures/tuple.pyi] From 2b711e2451eb79578afb82b3e4168c03bda852af Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Mar 2025 14:04:25 +0000 Subject: [PATCH 69/80] Fix binder issue with unions containing Any --- mypy/binder.py | 2 +- test-data/unit/check-redefine2.test | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mypy/binder.py b/mypy/binder.py index 6a72d8a96f23..e139843d0b53 100644 --- a/mypy/binder.py +++ b/mypy/binder.py @@ -288,7 +288,7 @@ def update_from_options(self, frames: list[Frame]) -> bool: # still equivalent to such type). if isinstance(type, UnionType): type = collapse_variadic_union(type) - if isinstance(type, ProperType) and isinstance(type, UnionType): + if old_semantics and isinstance(type, ProperType) and isinstance(type, UnionType): # Simplify away any extra Any's that were added to the declared # type when popping a frame. simplified = UnionType.make_union( diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 88fcbcc7a95f..1d6390c77b24 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -1090,18 +1090,18 @@ from typing import Any, Union def get() -> Union[tuple[Any, Any], tuple[None, None]]: ... -def func() -> None: +def f() -> None: x, _ = get() reveal_type(x) # N: Revealed type is "Union[Any, None]" if x and int(): - ... + reveal_type(x) # N: Revealed type is "Any" reveal_type(x) # N: Revealed type is "Union[Any, None]" if x and int(): - ... + reveal_type(x) # N: Revealed type is "Any" [builtins fixtures/tuple.pyi] [case testNewRedefinePartialTypeForUnderscore] -# flags: --allow-redefinition-new --local-partial-types --warn-unreachable +# flags: --allow-redefinition-new --local-partial-types def t() -> tuple[int]: return (42,) From 14524fecf2fbeadafa782a120d9eeab9ebbbc73b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Mar 2025 14:40:09 +0000 Subject: [PATCH 70/80] Improve error message --- mypy/binder.py | 6 +++++- mypy/checker.py | 6 ++++++ test-data/unit/check-redefine2.test | 4 +++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/mypy/binder.py b/mypy/binder.py index e139843d0b53..599b7452e6d9 100644 --- a/mypy/binder.py +++ b/mypy/binder.py @@ -288,7 +288,11 @@ def update_from_options(self, frames: list[Frame]) -> bool: # still equivalent to such type). if isinstance(type, UnionType): type = collapse_variadic_union(type) - if old_semantics and isinstance(type, ProperType) and isinstance(type, UnionType): + if ( + old_semantics + and isinstance(type, ProperType) + and isinstance(type, UnionType) + ): # Simplify away any extra Any's that were added to the declared # type when popping a frame. simplified = UnionType.make_union( diff --git a/mypy/checker.py b/mypy/checker.py index 7b23aa29b2e8..13b3d29ebba8 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4566,6 +4566,12 @@ def check_simple_assignment( rvalue_type = self.expr_checker.accept( rvalue, type_context=lvalue_type, always_allow_any=always_allow_any ) + if not is_valid_inferred_type(rvalue_type, self.options) and inferred is not None: + self.msg.need_annotation_for_var( + inferred, context, self.options.python_version + ) + rvalue_type = rvalue_type.accept(SetNothingToAny()) + if ( isinstance(lvalue, NameExpr) and inferred is not None diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 1d6390c77b24..b87353dee012 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -1081,8 +1081,10 @@ def func() -> None: l = None if int(): - l = [] + l = [] # E: Need type annotation for "l" l.append(1) + reveal_type(l) # N: Revealed type is "Union[None, builtins.list[Any]]" +[builtins fixtures/list.pyi] [case testNewRedefineNarrowingSpecialCase] # flags: --allow-redefinition-new --local-partial-types --warn-unreachable From 37098d63e53c249b3ac27be89378483689d5272f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Mar 2025 15:57:22 +0000 Subject: [PATCH 71/80] Address review --- mypy/binder.py | 7 ++++++- mypy/checker.py | 3 ++- test-data/unit/check-redefine2.test | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/mypy/binder.py b/mypy/binder.py index 599b7452e6d9..d3482d1dad4f 100644 --- a/mypy/binder.py +++ b/mypy/binder.py @@ -135,7 +135,7 @@ def __init__(self, options: Options) -> None: # If True, initial assignment to a simple variable (e.g. "x", but not "x.y") # is added to the binder. This allows more precise narrowing and more - # flexible inference of variable types. + # flexible inference of variable types (--allow-redefinition-new). self.bind_all = options.allow_redefinition_new def _get_id(self) -> int: @@ -233,6 +233,11 @@ def update_from_options(self, frames: list[Frame]) -> bool: for key in keys: current_value = self._get(key) resulting_values = [f.types.get(key, current_value) for f in frames] + # Keys can be narrowed using two different semantics. The new semantics + # is enabled for plain variables when bind_all is true, and it allows + # variable types to be widened using subsequent assignments. This is + # tricky to support for instance attributes (primarily due to deferrals), + # so we don't use it for them. old_semantics = not self.bind_all or extract_var_from_literal_hash(key) is None if old_semantics and any(x is None for x in resulting_values): # We didn't know anything about key before diff --git a/mypy/checker.py b/mypy/checker.py index 13b3d29ebba8..f51babe41fc2 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4594,7 +4594,8 @@ def check_simple_assignment( self.widened_vars.append(inferred.name) self.set_inferred_type(inferred, lvalue, lvalue_type) self.binder.put(lvalue, rvalue_type) - # TODO: hack, maybe integrate into put? + # TODO: A bit hacky, maybe add a binder method that does put and + # updates declaration? lit = literal_hash(lvalue) if lit is not None: self.binder.declarations[lit] = lvalue_type diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index b87353dee012..d2f1f9117fef 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -34,6 +34,23 @@ def f2() -> None: x = None reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" +[case testNewRedefineMergeConditionalLocal2] +# flags: --allow-redefinition-new --local-partial-types +def nested_ifs() -> None: + if int(): + if int(): + x = 0 + else: + x = '' + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + else: + if int(): + x = None + else: + x = b"" + reveal_type(x) # N: Revealed type is "Union[None, builtins.bytes]" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str, None, builtins.bytes]" + [case testNewRedefineUninitializedCodePath1] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: From 808f6e3664ffe4d2fdf060a0dc6e41f32c380af4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Mar 2025 16:02:26 +0000 Subject: [PATCH 72/80] Address more review comments --- test-data/unit/check-redefine2.test | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index d2f1f9117fef..2d6432a15dd8 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -371,18 +371,6 @@ class C4: reveal_type(C4().x) # N: Revealed type is "builtins.list[builtins.str]" [builtins fixtures/list.pyi] -[case testZZZ] -# flags: --local-partial-types -class C: - def __init__(self) -> None: - self.x = None - if int(): - self.x = 1 - - reveal_type(self.x) # N: Revealed type is "Union[builtins.int, None]" - -reveal_type(C().x) # N: Revealed type is "Union[builtins.int, None]" - [case testNewRedefinePartialGenericTypes] # flags: --allow-redefinition-new --local-partial-types def f1() -> None: @@ -794,10 +782,9 @@ def f2() -> None: x = "" return except Exception: - # TODO: Too wide reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" - # TODO: Too wide - reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + return + reveal_type(x) # N: Revealed type is "builtins.int" def f3() -> None: try: From 43179616968e664cbf367c3b058c40f11f2e2879 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Mar 2025 16:08:58 +0000 Subject: [PATCH 73/80] Update comment --- mypy/checker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index f51babe41fc2..9a56d1382f67 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4580,8 +4580,8 @@ def check_simple_assignment( ): new_inferred = remove_instance_last_known_values(rvalue_type) if not is_same_type(inferred.type, new_inferred): - # Should we widen the inferred type or the lvalue? We can only widen - # a variable type if the variable was defined in the current function. + # Should we widen the inferred type or the lvalue? Variables defined + # at module level or class bodies can't be widened in functions. different_scopes = ( self.scope.top_level_function() is not None and lvalue.kind != LDEF ) From 460ebe4b6dc47b0391e7b514a1befd15500efa1f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Mar 2025 16:22:48 +0000 Subject: [PATCH 74/80] Update comments --- mypy/checker.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 9a56d1382f67..52a99fed2082 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -316,7 +316,8 @@ class TypeChecker(NodeVisitor[None], CheckerPluginInterface): # Vars for which partial type errors are already reported # (to avoid logically duplicate errors with different error context). partial_reported: set[Var] - # Short names of Var nodes whose previous inferred type has been widened via assignment + # Short names of Var nodes whose previous inferred type has been widened via assignment. + # NOTE: The names might not be unique, they are only for debugging purposes. widened_vars: list[str] globals: SymbolTable modules: dict[str, MypyFile] @@ -616,6 +617,12 @@ def accept_loop( self.accept(body) partials_new = sum(len(pts.map) for pts in self.partial_types) widened_new = len(self.widened_vars) + # Perform multiple iterations if something changed that might affect + # inferred types. Also limit the number of iterations. The limits are + # somewhat arbitrary, but they were chosen to 1) avoid slowdown from + # multiple iterations in common cases and 2) support common, valid use + # cases. Limits are needed since otherwise we could infer infinitely + # complex types. if ( (partials_new == partials_old) and (not self.binder.last_pop_changed or iter > 3) @@ -3320,6 +3327,8 @@ def check_assignment( rvalue_type, lvalue_type = self.check_simple_assignment( lvalue_type, rvalue, context=rvalue, inferred=inferred, lvalue=lvalue ) + # The above call may update inferred variable type. Prevent further + # inference. inferred = None # Special case: only non-abstract non-protocol classes can be assigned to From 2c6a1447ad9b6502dded4b375f929493e1174e80 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Mar 2025 16:29:09 +0000 Subject: [PATCH 75/80] Address more comments --- mypy/checker.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 52a99fed2082..a86e51e5c660 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1406,7 +1406,7 @@ def check_func_def( if self.options.allow_redefinition_new and not self.is_stub: # Add formal argument types to the binder. for arg in defn.arguments: - # TODO: Add these directly using a fast path + # TODO: Add these directly using a fast path (possibly "put") v = arg.variable if v.type is not None: n = NameExpr(v.name) @@ -3304,7 +3304,8 @@ def check_assignment( # unpleasant, and a generalization of this would # be an improvement! if ( - is_literal_none(rvalue) + not self.options.allow_redefinition_new + and is_literal_none(rvalue) and isinstance(lvalue, NameExpr) and lvalue.kind == LDEF and isinstance(lvalue.node, Var) From daf9c750c0f106c3d1eadd3105c3d9f2001b0b02 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Mar 2025 16:44:46 +0000 Subject: [PATCH 76/80] Add TODO comments --- mypy/checker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/checker.py b/mypy/checker.py index a86e51e5c660..69c8323ecc17 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3364,6 +3364,7 @@ def check_assignment( ): lvalue.node.type = remove_instance_last_known_values(lvalue_type) elif self.options.allow_redefinition_new and lvalue_type is not None: + # TODO: Can we use put() here? self.binder.assign_type(lvalue, lvalue_type, lvalue_type) elif index_lvalue: @@ -5188,6 +5189,7 @@ def visit_try_without_finally(self, s: TryStmt, try_frame: bool) -> None: new_type = DeletedType(source=source) var.node.type = new_type if self.options.allow_redefinition_new: + # TODO: Should we use put() here? self.binder.assign_type(var, new_type, new_type) if not self.options.allow_redefinition_new: self.binder.cleanse(var) From 69fd839512f0f67f08a917a677d784cf95f805df Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 12 Mar 2025 17:15:50 +0000 Subject: [PATCH 77/80] TEMPORARY: Enable --allow-redefinition-new and --local-partial-types This lets us see the impact on mypy primer. --- mypy/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/options.py b/mypy/options.py index 27b583722568..4fbd35206eb1 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -222,7 +222,7 @@ def __init__(self) -> None: # Allow flexible variable redefinition with an arbitrary type, in different # blocks and and at different nesting levels - self.allow_redefinition_new = False + self.allow_redefinition_new = True # Prohibit equality, identity, and container checks for non-overlapping types. # This makes 1 == '1', 1 in ['1'], and 1 is '1' errors. @@ -361,7 +361,7 @@ def __init__(self) -> None: self.dump_deps = False self.logical_deps = False # If True, partial types can't span a module top level and a function - self.local_partial_types = False + self.local_partial_types = True # Some behaviors are changed when using Bazel (https://bazel.build). self.bazel = False # If True, export inferred types for all expressions as BuildResult.types From a5adfd78e2115478735d64ceb3b1b8ba99c829a2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 14 Mar 2025 13:26:27 +0000 Subject: [PATCH 78/80] Preserve TypedDicty type context more aggressively for backward compat --- mypy/checker.py | 9 ++++++++- test-data/unit/check-redefine2.test | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 69c8323ecc17..bae3d4d77b1a 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4560,7 +4560,7 @@ def check_simple_assignment( always_allow_any = lvalue_type is not None and not isinstance( get_proper_type(lvalue_type), AnyType ) - if inferred is None: + if inferred is None or is_typeddict_type_context(lvalue_type): type_context = lvalue_type else: type_context = None @@ -9233,3 +9233,10 @@ def _ambiguous_enum_variants(types: list[Type]) -> set[str]: else: result.add("") return result + + +def is_typeddict_type_context(lvalue_type: Type | None) -> bool: + if lvalue_type is None: + return False + lvalue_proper = get_proper_type(lvalue_type) + return isinstance(lvalue_proper, TypedDictType) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 2d6432a15dd8..61947ceff4bc 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -1126,3 +1126,17 @@ def f3() -> None: def f4() -> None: a, b = 1, [] # E: Need type annotation for "b" (hint: "b: List[] = ...") [builtins fixtures/tuple.pyi] + +[case testNewRedefineUseInferredTypedDictTypeForContext] +# flags: --allow-redefinition-new --local-partial-types +from typing import TypedDict + +class TD(TypedDict): + x: int + +def f() -> None: + td = TD(x=1) + if int(): + td = {"x": 5} + reveal_type(td) # N: Revealed type is "TypedDict('__main__.TD', {'x': builtins.int})" +[typing fixtures/typing-typeddict.pyi] From 2dba308d9a5637dd7550b2ec33c01f12d00981ea Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 14 Mar 2025 13:35:14 +0000 Subject: [PATCH 79/80] Backward compatibility fix --- mypy/checkexpr.py | 8 ++++++++ test-data/unit/check-redefine2.test | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 9b7b1c14d338..4a2904a2ef13 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -5786,6 +5786,14 @@ def check_for_comp(self, e: GeneratorExpr | DictionaryComprehension) -> None: _, sequence_type = self.chk.analyze_async_iterable_item_type(sequence) else: _, sequence_type = self.chk.analyze_iterable_item_type(sequence) + if ( + isinstance(sequence_type, UninhabitedType) + and isinstance(index, NameExpr) + and index.name == "_" + ): + # To preserve backward compatibility, avoid inferring Never for "_" + sequence_type = AnyType(TypeOfAny.special_form) + self.chk.analyze_index_variables(index, sequence_type, True, e) for condition in conditions: self.accept(condition) diff --git a/test-data/unit/check-redefine2.test b/test-data/unit/check-redefine2.test index 61947ceff4bc..8ed448de7e6b 100644 --- a/test-data/unit/check-redefine2.test +++ b/test-data/unit/check-redefine2.test @@ -1140,3 +1140,10 @@ def f() -> None: td = {"x": 5} reveal_type(td) # N: Revealed type is "TypedDict('__main__.TD', {'x': builtins.int})" [typing fixtures/typing-typeddict.pyi] + +[case testNewRedefineEmptyGeneratorUsingUnderscore] +# flags: --allow-redefinition-new --local-partial-types +def f() -> None: + gen = (_ for _ in ()) + reveal_type(gen) # N: Revealed type is "typing.Generator[Any, None, None]" +[builtins fixtures/tuple.pyi] From ac852e69129dfd6f7fbaef3b1a2ef2cbe114256f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 14 Mar 2025 15:00:01 +0000 Subject: [PATCH 80/80] Fix self check --- mypy/checkexpr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 4a2904a2ef13..80471a04469c 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -5787,7 +5787,7 @@ def check_for_comp(self, e: GeneratorExpr | DictionaryComprehension) -> None: else: _, sequence_type = self.chk.analyze_iterable_item_type(sequence) if ( - isinstance(sequence_type, UninhabitedType) + isinstance(get_proper_type(sequence_type), UninhabitedType) and isinstance(index, NameExpr) and index.name == "_" ):