Python: Fix various parser issues (#6547)

* Python: Parser adaptability

* Python: Add support for PEP 695 type parameters and subscript decorators

- Implement parsing and printing for Python 3.12+ type parameters
  (TypeVar, TypeVarTuple, ParamSpec) in class and function definitions
- Add support for subscript decorators (e.g., @[property][0])
- Fix f-string debug expression trailing newline issue
- Add comprehensive tests for type parameters and decorators

Co-Authored-By: Claude Opus 4.5 <[email protected]>

* Python: Commit Python parser tests

* Python: Add type parameters support to TypeAlias

Add support for PEP 695 type parameters on type alias statements
(e.g., `type Foo[T] = list[T]`).

Co-Authored-By: Claude Opus 4.5 <[email protected]>

* Fix more parser issues

* Fix statement padding

* Python: Handle UTF-8 BOM in source files

- Detect and strip BOM character (U+FEFF) before parsing
- Store BOM presence in CompilationUnit.charset_bom_marked
- Output BOM when printing if file originally had one

Co-Authored-By: Claude Opus 4.5 <[email protected]>

* Python: Fix multi-line tuple unpacking in for loops

The parser was incorrectly determining whether parentheses belong to a
tuple or its first element. The heuristic only checked column offset,
but for multi-line tuples like:

    for (
        window,
        in_entry,
    ) in buf_out:

The tuple and first element have the same column offset (4) but are on
different lines. The fix now checks both line AND column to correctly
identify that the '(' belongs to the tuple itself.

Co-Authored-By: Claude Opus 4.5 <[email protected]>

* Fix more parser problems

* Fix more parser problems

* Fix more parser problems

* Yet more parser fixes

* Yet more parser fixes

* Unicode fixes

* pep646

---------

Co-authored-by: Claude Opus 4.5 <[email protected]>

Author: knutwannheden

rewrite-python/rewrite/src/rewrite/java/support_types.py

@@ -25,7 +25,7 @@ def prefix(self) -> Space:
 
     def is_acceptable(self, v: TreeVisitor[Any, P], p: P) -> bool:
         from .visitor import JavaVisitor
-        return isinstance(v, JavaVisitor)
+        return v.is_adaptable_to(JavaVisitor)
 
     def accept(self, v: TreeVisitor[Any, P], p: P) -> Optional[Any]:
         from .visitor import JavaVisitor

rewrite-python/rewrite/src/rewrite/parser.py

@@ -144,7 +144,7 @@ def printer(self, cursor: Cursor) -> TreeVisitor[Tree, PrintOutputCapture[P]]:
         return PrinterFactory.current().create_printer(cursor)
 
     def is_acceptable(self, v: TreeVisitor[Any, P], p: P) -> bool:
-        return isinstance(v, ParseErrorVisitor)
+        return v.is_adaptable_to(ParseErrorVisitor)
 
     def accept(self, v: TreeVisitor[Any, P], p: P) -> Optional[Any]:
         return cast(ParseErrorVisitor, v).visit_parse_error(self, p)

rewrite-python/rewrite/src/rewrite/python/_parser_visitor.py

@@ -375,6 +375,9 @@ def _visit_container(
     def visit_compilation_unit(self, cu: 'py.CompilationUnit', p: PrintOutputCapture) -> J:
         """Visit a Python compilation unit."""
         from rewrite.java.tree import Import
+        # Output UTF-8 BOM if the original file had one
+        if cu.charset_bom_marked:
+            p.append('\ufeff')
         self._before_syntax(cu, Space.Location.COMPILATION_UNIT_PREFIX, p)
 
         # Print imports
@@ -802,6 +805,8 @@ def visit_type_alias(self, alias: 'py.TypeAlias', p: PrintOutputCapture) -> J:
         self._before_syntax(alias, PySpace.Location.UNION_TYPE_PREFIX, p)
         p.append("type")
         self.visit(alias.name, p)
+        # Visit type parameters (Python 3.12+ PEP 695)
+        self._visit_container("[", alias.padding.type_parameters, JContainer.Location.TYPE_PARAMETERS, ",", "]", p)
         self._visit_left_padded("=", alias.padding.value, PyLeftPadded.Location.TYPE_ALIAS_VALUE, p)
         self._after_syntax(alias, p)
         return alias
@@ -973,6 +978,8 @@ def _visit_java(self, tree: J, p: PrintOutputCapture) -> Optional[J]:
             return self.visit_ternary(tree, p)
         elif isinstance(tree, j.Throw):
             return self.visit_throw(tree, p)
+        elif isinstance(tree, j.TypeParameter):
+            return self.visit_type_parameter(tree, p)
         elif isinstance(tree, j.Try):
             return self.visit_try(tree, p)
         elif isinstance(tree, j.Try.Resource):
@@ -1343,6 +1350,9 @@ def visit_class_declaration(self, class_decl: 'j.ClassDeclaration', p: PrintOutp
         p.append("class")
         self.visit(class_decl.name, p)
 
+        # Visit type parameters (Python 3.12+)
+        self._visit_container("[", class_decl.padding.type_parameters, JContainer.Location.TYPE_PARAMETERS, ",", "]", p)
+
         # Visit implements (base classes in Python)
         if class_decl.padding.implements:
             omit_parens = class_decl.padding.implements.markers.find_first(OmitParentheses)
@@ -1531,6 +1541,8 @@ def visit_method_declaration(self, method: 'j.MethodDeclaration', p: PrintOutput
             self.visit_modifier(mod, p)
 
         self.visit(method.name, p)
+        # Visit type parameters (Python 3.12+)
+        self._visit_container("[", method.padding.type_parameters, JContainer.Location.TYPE_PARAMETERS, ",", "]", p)
         self._visit_container("(", method.padding.parameters, JContainer.Location.METHOD_DECLARATION_PARAMETERS, ",", ")", p)
         self.visit(method.return_type_expression, p)
         self.visit(method.body, p)
@@ -1571,6 +1583,9 @@ def visit_modifier(self, mod: 'j.Modifier', p: PrintOutputCapture) -> J:
             keyword = "def"
         elif mod.type == Modifier.Type.Async:
             keyword = "async"
+        elif mod.type == Modifier.Type.LanguageExtension:
+            # Use the keyword directly (for * and ** in type parameters)
+            keyword = mod.keyword
 
         if keyword:
             for annotation in mod.annotations:
@@ -1640,6 +1655,18 @@ def visit_throw(self, throw: 'j.Throw', p: PrintOutputCapture) -> J:
         self._after_syntax(throw, p)
         return throw
 
+    def visit_type_parameter(self, type_param: 'j.TypeParameter', p: PrintOutputCapture) -> J:
+        """Visit a type parameter (Python 3.12+ PEP 695)."""
+        self._before_syntax(type_param, Space.Location.TYPE_PARAMETERS_PREFIX, p)
+        # Visit modifiers (for * and ** prefixes)
+        for mod in type_param.modifiers:
+            self.visit(mod, p)
+        self.visit(type_param.name, p)
+        # Visit bounds (for T: int style bounds in Python)
+        self._visit_container(":", type_param.padding.bounds, JContainer.Location.TYPE_BOUNDS, ",", "", p)
+        self._after_syntax(type_param, p)
+        return type_param
+
     def visit_try(self, try_: 'j.Try', p: PrintOutputCapture) -> J:
         """Visit a try statement (or with statement in Python)."""
         from rewrite.python import tree as py
@@ -1794,11 +1821,15 @@ def visit_variable_declarations(self, multi_variable: 'j.VariableDeclarations',
 
         # Visit variables
         nodes = multi_variable.padding.variables
+        is_kwonly_marker = multi_variable.markers.find_first(KeywordOnlyArguments) is not None
         for i, node in enumerate(nodes):
             # Set cursor for context in visit_variable
             self.set_cursor(Cursor(self.get_cursor(), node))
             self.visit(node.element, p)
             self._visit_markers(node.markers, p)
+            # For keyword-only args marker (bare *), print the after space before comma
+            if is_kwonly_marker:
+                self._visit_space(node.after, JRightPadded.Location.NAMED_VARIABLE.after_location, p)
             if i < len(nodes) - 1:
                 p.append(",")
             # Restore cursor

rewrite-python/rewrite/src/rewrite/python/printer.py

@@ -24,7 +24,7 @@ def accept_python(self, v: 'PythonVisitor[P]', p: P) -> Optional['J']:
 
     def is_acceptable(self, v: TreeVisitor[Any, P], p: P) -> bool:
         from .visitor import PythonVisitor
-        return isinstance(v, PythonVisitor)
+        return v.is_adaptable_to(PythonVisitor)
 
 
 class PySpace:

rewrite-python/rewrite/src/rewrite/python/support_types.py

@@ -1486,6 +1486,13 @@ def name(self) -> Identifier:
         return self._name
 
 
+    _type_parameters: Optional[JContainer[j.TypeParameter]]
+
+    @property
+    def type_parameters(self) -> Optional[List[j.TypeParameter]]:
+        return self._type_parameters.elements if self._type_parameters else None
+
+
     _value: JLeftPadded[J]
 
     @property
@@ -1504,6 +1511,10 @@ def type(self) -> Optional[JavaType]:
     class PaddingHelper:
         _t: TypeAlias
 
+        @property
+        def type_parameters(self) -> Optional[JContainer[j.TypeParameter]]:
+            return self._t._type_parameters
+
         @property
         def value(self) -> JLeftPadded[J]:
             return self._t._value

rewrite-python/rewrite/src/rewrite/python/tree.py

@@ -207,8 +207,11 @@ def parse_python_source(source: str, path: str = "<unknown>") -> dict:
         from rewrite.python._parser_visitor import ParserVisitor
         from rewrite import random_id, Markers
 
+        # Strip BOM before parsing (ParserVisitor handles it internally but ast.parse doesn't)
+        source_for_ast = source[1:] if source.startswith('\ufeff') else source
+
         # Parse using Python AST
-        tree = ast.parse(source, path)
+        tree = ast.parse(source_for_ast, path)
 
         # Convert to OpenRewrite LST
         cu = ParserVisitor(source).visit(tree)

rewrite-python/rewrite/src/rewrite/rpc/server.py

@@ -218,7 +218,9 @@ def _parse_python(self, source: str, source_path: Path) -> CompilationUnit:
         from rewrite.python._parser_visitor import ParserVisitor
 
         visitor = ParserVisitor(source)
-        tree = ast.parse(source)
+        # Strip BOM before passing to ast.parse (ParserVisitor does this internally)
+        source_for_ast = source[1:] if source.startswith('\ufeff') else source
+        tree = ast.parse(source_for_ast)
         cu = visitor.visit_Module(tree)
         return cu.replace(source_path=source_path)
 

rewrite-python/rewrite/src/rewrite/test/rewrite_test.py

@@ -177,6 +177,30 @@ def visit_markers(self, markers: Markers, p: P) -> Markers:
     def visit_marker(self, marker: Marker, p: P) -> Marker:
         return marker
 
+    def stop_after_pre_visit(self) -> None:
+        """
+        Stop visiting after pre_visit returns, preventing accept() and post_visit() from being called.
+        Call this in pre_visit when you only want to process at a high level without traversing children.
+        """
+        self._cursor.put_message("STOP_AFTER_PRE_VISIT", True)
+
+    def is_adaptable_to(self, adapt_to: Type['TreeVisitor[Any, Any]']) -> bool:
+        """
+        Check if this visitor can be adapted to the given visitor type.
+
+        A visitor is adaptable if:
+        1. It is already an instance of the target type, OR
+        2. It is a base TreeVisitor (not a language-specific visitor)
+
+        This is a simplified implementation. Full adaptation support
+        would check tree type hierarchies like Java does.
+        """
+        if isinstance(self, adapt_to):
+            return True
+        # Base TreeVisitor is adaptable to any visitor type
+        # (its tree type is Tree, which is a supertype of all tree types)
+        return type(self).__mro__[1] == TreeVisitor or type(self) == TreeVisitor
+
     def adapt(self, tree_type, visitor_type: Type[TV]) -> TV:
         # FIXME implement the visitor adapting
         return cast(TV, self)

rewrite-python/rewrite/src/rewrite/visitor.py

@@ -0,0 +1,70 @@
+from rewrite.test import RecipeSpec, python
+
+
+def test_empty():
+    # language=python
+    RecipeSpec().rewrite_run(python("a = []"))
+
+
+def test_trailing_comma():
+    # language=python
+    RecipeSpec().rewrite_run(python("a = [1, 2,  ]"))
+
+
+def test_array_subscript():
+    # language=python
+    RecipeSpec().rewrite_run(python("a = [1, 2][0]"))
+
+
+def test_array_slice():
+    # language=python
+    RecipeSpec().rewrite_run(python("a = [1, 2][0:1]"))
+
+
+def test_array_slice_no_upper():
+    # language=python
+    RecipeSpec().rewrite_run(python("a = [1, 2][0:]"))
+
+
+def test_array_slice_all_empty():
+    # language=python
+    RecipeSpec().rewrite_run(python("a = [1, 2][ :  :   ]"))
+
+
+def test_comment():
+    # language=python
+    RecipeSpec().rewrite_run(python(r'''
+        a = d[:0]
+        a = d[0:]
+        '''
+           ))
+
+
+def test_array_slice_empty_upper_and_step():
+    # language=python
+    RecipeSpec().rewrite_run(python("a = [1, 2][0::]"))
+
+
+def test_array_slice_no_lower():
+    # language=python
+    RecipeSpec().rewrite_run(python("a = [1, 2][:1]"))
+
+
+def test_array_slice_no_lower_no_upper():
+    # language=python
+    RecipeSpec().rewrite_run(python("a = [1, 2][::1]"))
+
+
+def test_array_slice_full():
+    # language=python
+    RecipeSpec().rewrite_run(python("a = [1, 2][0:1:1]"))
+
+
+def test_array_slice_tuple_index_1():
+    # language=python
+    RecipeSpec().rewrite_run(python("a = [1, 2][0,1]"))
+
+
+def test_array_slice_tuple_index_2():
+    # language=python
+    RecipeSpec().rewrite_run(python("a = [1, 2][(0,1)]"))