Support for Python 3.11: cython. WIP #939

Author: fabioz

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_collect_bytecode_info.py

@@ -669,7 +669,7 @@ def _lookahead(self):
             )
             return RESTART_FROM_LOOKAHEAD
 
-        if next_instruction.opname == 'CALL_FUNCTION':
+        if next_instruction.opname in ('CALL_FUNCTION', 'PRECALL'):
             if len(found) == next_instruction.argval + 1:
                 force_restart = False
                 delta = 0
@@ -680,7 +680,10 @@ def _lookahead(self):
                 else:
                     return None  # This is odd
 
-            del self.instructions[delta:delta + next_instruction.argval + 2]  # +2 = NAME / CALL_FUNCTION
+            del_upto = delta + next_instruction.argval + 2  # +2 = NAME / CALL_FUNCTION
+            if next_instruction.opname == 'PRECALL':
+                del_upto += 1  # Also remove the CALL right after the PRECALL.
+            del self.instructions[delta:del_upto]
 
             found = iter(found[delta:])
             call_func = next(found)
@@ -779,8 +782,12 @@ def _create_msg_part(self, instruction, tok=None, line=None):
         dec = self._decorate_jump_target
         if line is None or line in (self.BIG_LINE_INT, self.SMALL_LINE_INT):
             line = self.op_offset_to_line[instruction.offset]
+
+        argrepr = instruction.argrepr
+        if isinstance(argrepr, str) and argrepr.startswith('NULL + '):
+            argrepr = argrepr[7:]
         return _MsgPart(
-            line, tok if tok is not None else dec(instruction, instruction.argrepr))
+            line, tok if tok is not None else dec(instruction, argrepr))
 
     def _next_instruction_to_str(self, line_to_contents):
         # indent = ''
@@ -797,6 +804,9 @@ def _next_instruction_to_str(self, line_to_contents):
 
         instruction = self.instructions.pop(0)
 
+        if instruction.opname in 'RESUME':
+            return None
+
         if instruction.opname in ('LOAD_GLOBAL', 'LOAD_FAST', 'LOAD_CONST', 'LOAD_NAME'):
             next_instruction = self.instructions[0]
             if next_instruction.opname in ('STORE_FAST', 'STORE_NAME'):
@@ -855,6 +865,8 @@ def build_line_to_contents(self):
             s = self._next_instruction_to_str(line_to_contents)
             if s is RESTART_FROM_LOOKAHEAD:
                 continue
+            if s is None:
+                continue
 
             _MsgPart.add_to_line_to_contents(s, line_to_contents)
             m = self.max_line(s)

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_cython.c

@@ -704,7 +704,11 @@ cdef class PyDBFrame:
         # if DEBUG: print('frame trace_dispatch %s %s %s %s %s %s, stop: %s' % (frame.f_lineno, frame.f_code.co_name, frame.f_code.co_filename, event, constant_to_str(info.pydev_step_cmd), arg, info.pydev_step_stop))
         try:
             info.is_tracing += 1
-            line = frame.f_lineno
+
+            # TODO: This shouldn't be needed. The fact that frame.f_lineno
+            # is None seems like a bug in Python 3.11.
+            # Reported in: https://github.com/python/cpython/issues/94485
+            line = frame.f_lineno or 0  # Workaround or case where frame.f_lineno is None
             line_cache_key = (frame_cache_key, line)
 
             if main_debugger.pydb_disposed:

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_cython.pyx

@@ -571,7 +571,11 @@ def trace_dispatch(self, frame, event, arg):
         # if DEBUG: print('frame trace_dispatch %s %s %s %s %s %s, stop: %s' % (frame.f_lineno, frame.f_code.co_name, frame.f_code.co_filename, event, constant_to_str(info.pydev_step_cmd), arg, info.pydev_step_stop))
         try:
             info.is_tracing += 1
-            line = frame.f_lineno
+
+            # TODO: This shouldn't be needed. The fact that frame.f_lineno
+            # is None seems like a bug in Python 3.11.
+            # Reported in: https://github.com/python/cpython/issues/94485
+            line = frame.f_lineno or 0  # Workaround or case where frame.f_lineno is None
             line_cache_key = (frame_cache_key, line)
 
             if main_debugger.pydb_disposed:

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_frame.py

@@ -52,6 +52,12 @@ cdef extern from "code.h":
     ctypedef void freefunc(void *)
     int _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra)
     int _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra)
+    
+# TODO: Things are in a different place for Python 3.11.
+# cdef extern from "cpython/code.h":
+#     ctypedef void freefunc(void *)
+#     int _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra)
+#     int _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra)
 
 cdef extern from "Python.h":
     void Py_INCREF(object o)

src/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_frame_evaluator.pxd

@@ -16,6 +16,7 @@
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
 IS_PY36_OR_GREATER = sys.version_info > (3, 6)
+TODO_PY311 = sys.version_info > (3, 11)
 
 
 def process_args():
@@ -234,7 +235,7 @@ def build_extension(dir_name, extension_name, target_pydevd_name, force_cython,
     target_pydevd_name = extension_name
 build_extension("_pydevd_bundle", extension_name, target_pydevd_name, force_cython, extension_folder, True)
 
-if IS_PY36_OR_GREATER:
+if IS_PY36_OR_GREATER and not TODO_PY311:
     extension_name = "pydevd_frame_evaluator"
     if target_frame_eval is None:
         target_frame_eval = extension_name

src/debugpy/_vendored/pydevd/setup_pydevd_cython.py

@@ -6,8 +6,11 @@
 PYDEVD_TEST_VM = os.getenv('PYDEVD_TEST_VM', None)
 
 IS_PY36_OR_GREATER = sys.version_info[0:2] >= (3, 6)
+IS_PY311_OR_GREATER = sys.version_info[0:2] >= (3, 11)
 IS_CPYTHON = platform.python_implementation() == 'CPython'
 
+TODO_PY311 = IS_PY311_OR_GREATER  # Code which needs to be fixed in 3.11 should use this constant.
+
 IS_PY36 = False
 if sys.version_info[0] == 3 and sys.version_info[1] == 6:
     IS_PY36 = True

src/debugpy/_vendored/pydevd/tests_python/debug_constants.py

@@ -5,9 +5,13 @@
 
 import pytest
 
-from tests_python.debug_constants import IS_PY36_OR_GREATER, IS_CPYTHON, TEST_CYTHON
+from tests_python.debug_constants import IS_PY36_OR_GREATER, IS_CPYTHON, TEST_CYTHON, TODO_PY311
 
-pytestmark = pytest.mark.skipif(not IS_PY36_OR_GREATER or not IS_CPYTHON or not TEST_CYTHON, reason='Requires CPython >= 3.6')
+pytestmark = pytest.mark.skipif(
+    not IS_PY36_OR_GREATER or
+    TODO_PY311 or
+    not IS_CPYTHON or
+    not TEST_CYTHON, reason='Requires CPython >= 3.6 < 3.11')
 
 
 class _Tracer(object):

src/debugpy/_vendored/pydevd/tests_python/test_bytecode_manipulation.py

@@ -8,6 +8,7 @@
     collect_return_info, code_to_bytecode_representation
 from tests_python.debugger_unittest import IS_CPYTHON, IS_PYPY
 from _pydevd_bundle.pydevd_constants import IS_PY38_OR_GREATER, IS_JYTHON
+from tests_python.debug_constants import IS_PY311_OR_GREATER
 
 
 def _method_call_with_error():
@@ -192,7 +193,7 @@ def test_collect_try_except_info(data_regression, pyfile):
             info = collect_try_except_info(method.__code__, use_func_first_line=True)
             method_to_info[key] = sorted(str(x) for x in info)
 
-    if sys.version_info[:2] != (3, 10):
+    if sys.version_info[:2] not in ((3, 10), (3, 11)):
         data_regression.check(method_to_info)
 
     data_regression.check(method_to_info_from_source)
@@ -385,6 +386,8 @@ def try_except_with():
 
 
 def test_collect_try_except_info_async_for():
+    if IS_PY311_OR_GREATER:
+        pytest.skip('On Python 3.11 we just support collecting info from the AST.')
 
     # Not valid on Python 2.
     code_str = '''
@@ -482,7 +485,8 @@ def method4():
                 3,
                 call('tnh %s' % 1))
 
-    assert code_to_bytecode_representation(method4.__code__, use_func_first_line=True).count('\n') == 4
+    contents = code_to_bytecode_representation(method4.__code__, use_func_first_line=True)
+    assert contents.count('\n') == 4, 'Found:%s' % (contents,)
 
 
 @pytest.mark.skipif(IS_JYTHON, reason='Jython does not have bytecode support.')
@@ -537,6 +541,7 @@ def method4():
     new_repr = code_to_bytecode_representation(method4.__code__, use_func_first_line=True)
     assert 'call()' in new_repr
     assert 'call(1, 2, 3, a, b, \'x\')' in new_repr
+    assert 'NULL' not in new_repr
 
 
 @pytest.mark.skipif(IS_JYTHON, reason='Jython does not have bytecode support.')

src/debugpy/_vendored/pydevd/tests_python/test_collect_bytecode_info.py

@@ -1689,6 +1689,8 @@ def test_case_get_next_statement_targets(case_setup):
         # it's also not that bad (that line has no code in the source and
         # executing it will just set the tracing for the method).
         targets.discard(20)
+        # On Python 3.11 there's now a line 1 (which should be harmless).
+        targets.discard(1)
         expected = set((2, 3, 5, 8, 9, 10, 12, 13, 14, 15, 17, 18, 19, 21))
         assert targets == expected, 'Expected targets to be %s, was: %s' % (expected, targets)
 
@@ -3457,7 +3459,7 @@ def test_frame_eval_limitations(case_setup, filename, break_at_lines):
             hit = writer.wait_for_breakpoint_hit()
             thread_id = hit.thread_id
 
-            if IS_PY36_OR_GREATER and TEST_CYTHON:
+            if (IS_PY36_OR_GREATER and TEST_CYTHON) and not TODO_PY311:
                 assert hit.suspend_type == break_mode
             else:
                 # Before 3.6 frame eval is not available.
@@ -3496,6 +3498,7 @@ def test_step_return_my_code(case_setup):
         writer.finished_ok = True
 
 
+@pytest.mark.skipif(TODO_PY311, reason='Needs bytecode support in Python 3.11')
 def test_smart_step_into_case1(case_setup):
     with case_setup.test_file('_debugger_case_smart_step_into.py') as writer:
         line = writer.get_line_index_with_content('break here')
@@ -3518,6 +3521,7 @@ def test_smart_step_into_case1(case_setup):
         writer.finished_ok = True
 
 
+@pytest.mark.skipif(TODO_PY311, reason='Needs bytecode support in Python 3.11')
 def test_smart_step_into_case2(case_setup):
     with case_setup.test_file('_debugger_case_smart_step_into2.py') as writer:
         line = writer.get_line_index_with_content('break here')
@@ -3546,6 +3550,7 @@ def test_smart_step_into_case2(case_setup):
         writer.finished_ok = True
 
 
+@pytest.mark.skipif(TODO_PY311, reason='Needs bytecode support in Python 3.11')
 def test_smart_step_into_case3(case_setup):
     with case_setup.test_file('_debugger_case_smart_step_into3.py') as writer:
         line = writer.get_line_index_with_content('break here')