Don't crash if running under CPython debug build. Fixes #152

Author: fabioz

src/debugpy/_vendored/pydevd/.travis.yml

@@ -57,12 +57,12 @@ matrix:
         - PYDEVD_USE_CYTHON=NO
         - PYDEVD_TEST_VM=CPYTHON
 
-    # Python 3.7
+    # Python 3.8 debug
     - python: 2.7
       env:
-        - PYDEVD_PYTHON_VERSION=3.7
-        - PYDEVD_USE_CYTHON=YES
-        - PYDEVD_TEST_VM=CPYTHON
+        - PYDEVD_PYTHON_VERSION=3.8
+        - PYDEVD_TEST_VM=CPYTHON_DEBUG
+        - PYDEVD_USE_CONDA=NO
 
     - python: 3.8
       env:
@@ -124,6 +124,7 @@ install:
 # On local machine with jython: c:\bin\jython2.7.0\bin\jython.exe -Dpython.path=.;jython_test_deps/ant.jar;jython_test_deps/junit.jar -m pytest
 # On remove machine with python: c:\bin\python27\python.exe -m pytest
 script:
+  - if [[ ("$PYDEVD_TEST_VM" == "CPYTHON_DEBUG") ]]; then ./.travis/install_and_run_debug_py.sh; fi
   - if [[ ("$PYDEVD_TEST_VM" == "CPYTHON") ]]; then ./.travis/run_python_pytest.sh; fi
   - if [ "$PYDEVD_TEST_VM" == "PYPY" ]; then source activate build_env; pypy3 -m pytest -n auto; fi
   - if [ "$PYDEVD_TEST_VM" == "JYTHON" ]; then jython -Dpython.path=.:jython_test_deps/ant.jar:jython_test_deps/junit.jar -m pytest --tb=native; fi

src/debugpy/_vendored/pydevd/.travis/install_and_run_debug_py.sh

@@ -0,0 +1,35 @@
+# Build the cython extensions (to check that we don't crash when they're there in debug mode).
+python setup_cython.py build_ext --inplace
+
+curl -L https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tgz -o Python-3.8.3.tgz
+tar -xzf Python-3.8.3.tgz
+cd Python-3.8.3
+mkdir debug
+cd debug
+../configure --with-pydebug
+make
+
+curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+./python get-pip.py
+
+./python -m pip install "pytest"
+./python -m pip install "psutil"
+./python -m pip install "untangle"
+
+# Check that it worked.
+./python -c "import pytest"
+./python -c "import psutil"
+./python -c "import untangle"
+
+cd ..
+cd ..
+ls -la
+
+./Python-3.8.3/debug/python -c "import sys;assert hasattr(sys,'gettotalrefcount')"
+
+cd tests_python
+
+# Although we compiled cython, all we're checking is that we don't crash (since it was built for the release env).
+../Python-3.8.3/debug/python -m pytest test_debugger_json.py -k "test_case_json_change_breaks or test_remote_debugger_basic"
+export PYTHONPATH=..
+../Python-3.8.3/debug/python -c "import check_debug_python;check_debug_python.check() "

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_additional_thread_info.py

@@ -1,23 +1,19 @@
 # Defines which version of the PyDBAdditionalThreadInfo we'll use.
+from _pydevd_bundle.pydevd_constants import ENV_FALSE_LOWER_VALUES, USE_CYTHON_FLAG, \
+    ENV_TRUE_LOWER_VALUES
 
-import os
-use_cython = os.getenv('PYDEVD_USE_CYTHON', None)
-
-if use_cython == 'YES':
+if USE_CYTHON_FLAG in ENV_TRUE_LOWER_VALUES:
     # We must import the cython version if forcing cython
     from _pydevd_bundle.pydevd_cython_wrapper import PyDBAdditionalThreadInfo, set_additional_thread_info, _set_additional_thread_info_lock  # @UnusedImport
 
-elif use_cython == 'NO':
+elif USE_CYTHON_FLAG in ENV_FALSE_LOWER_VALUES:
     # Use the regular version if not forcing cython
     from _pydevd_bundle.pydevd_additional_thread_info_regular import PyDBAdditionalThreadInfo, set_additional_thread_info, _set_additional_thread_info_lock  # @UnusedImport @Reimport
 
-elif use_cython is None:
+else:
     # Regular: use fallback if not found (message is already given elsewhere).
     try:
         from _pydevd_bundle.pydevd_cython_wrapper import PyDBAdditionalThreadInfo, set_additional_thread_info, _set_additional_thread_info_lock
     except ImportError:
         from _pydevd_bundle.pydevd_additional_thread_info_regular import PyDBAdditionalThreadInfo, set_additional_thread_info, _set_additional_thread_info_lock  # @UnusedImport
-else:
-    raise RuntimeError('Unexpected value for PYDEVD_USE_CYTHON: %s (accepted: YES, NO)' % (use_cython,))
-
 

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py

@@ -173,7 +173,23 @@ def version_str(v):
 except AttributeError:
     PY_IMPL_NAME = ''
 
-SUPPORT_GEVENT = os.getenv('GEVENT_SUPPORT', 'False') in ('True', 'true', '1')
+ENV_TRUE_LOWER_VALUES = ('yes', 'true', '1')
+ENV_FALSE_LOWER_VALUES = ('no', 'false', '0')
+
+
+def is_true_in_env(env_key):
+    if isinstance(env_key, tuple):
+        # If a tuple, return True if any of those ends up being true.
+        for v in env_key:
+            if is_true_in_env(v):
+                return True
+        return False
+    else:
+        return os.getenv(env_key, '').lower() in ENV_TRUE_LOWER_VALUES
+
+
+# If true in env, use gevent mode.
+SUPPORT_GEVENT = is_true_in_env('GEVENT_SUPPORT')
 
 GEVENT_SUPPORT_NOT_SET_MSG = os.getenv(
     'GEVENT_SUPPORT_NOT_SET_MSG',
@@ -187,14 +203,32 @@ def version_str(v):
 
 INTERACTIVE_MODE_AVAILABLE = sys.platform in ('darwin', 'win32') or os.getenv('DISPLAY') is not None
 
-SHOW_COMPILE_CYTHON_COMMAND_LINE = os.getenv('PYDEVD_SHOW_COMPILE_CYTHON_COMMAND_LINE', 'False') == 'True'
+# If true in env, forces cython to be used (raises error if not available).
+# If false in env, disables it.
+# If not specified, uses default heuristic to determine if it should be loaded.
+USE_CYTHON_FLAG = os.getenv('PYDEVD_USE_CYTHON')
+
+# Use to disable loading the lib to set tracing to all threads (default is using heuristics based on where we're running).
+LOAD_NATIVE_LIB_FLAG = os.getenv('PYDEVD_LOAD_NATIVE_LIB', '').lower()
+
+if USE_CYTHON_FLAG is not None:
+    USE_CYTHON_FLAG = USE_CYTHON_FLAG.lower()
+    if USE_CYTHON_FLAG not in ENV_TRUE_LOWER_VALUES and USE_CYTHON_FLAG not in ENV_FALSE_LOWER_VALUES:
+        raise RuntimeError('Unexpected value for PYDEVD_USE_CYTHON: %s (enable with one of: %s, disable with one of: %s)' % (
+            USE_CYTHON_FLAG, ENV_TRUE_LOWER_VALUES, ENV_FALSE_LOWER_VALUES))
+
+else:
+    if not CYTHON_SUPPORTED:
+        USE_CYTHON_FLAG = 'no'
+
+SHOW_COMPILE_CYTHON_COMMAND_LINE = is_true_in_env('PYDEVD_SHOW_COMPILE_CYTHON_COMMAND_LINE')
 
-LOAD_VALUES_ASYNC = os.getenv('PYDEVD_LOAD_VALUES_ASYNC', 'False') == 'True'
+LOAD_VALUES_ASYNC = is_true_in_env('PYDEVD_LOAD_VALUES_ASYNC')
 DEFAULT_VALUE = "__pydevd_value_async"
 ASYNC_EVAL_TIMEOUT_SEC = 60
 NEXT_VALUE_SEPARATOR = "__pydev_val__"
 BUILTINS_MODULE_NAME = '__builtin__' if IS_PY2 else 'builtins'
-SHOW_DEBUG_INFO_ENV = os.getenv('PYCHARM_DEBUG') == 'True' or os.getenv('PYDEV_DEBUG') == 'True' or os.getenv('PYDEVD_DEBUG') == 'True'
+SHOW_DEBUG_INFO_ENV = is_true_in_env(('PYCHARM_DEBUG', 'PYDEV_DEBUG', 'PYDEVD_DEBUG'))
 
 if SHOW_DEBUG_INFO_ENV:
     # show debug info before the debugger start

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_trace_dispatch.py

@@ -2,16 +2,12 @@
 # Should give warning only here if cython is not available but supported.
 
 import os
-from _pydevd_bundle.pydevd_constants import CYTHON_SUPPORTED
+from _pydevd_bundle.pydevd_constants import USE_CYTHON_FLAG, ENV_TRUE_LOWER_VALUES, \
+    ENV_FALSE_LOWER_VALUES
 from _pydev_bundle import pydev_log
 
-use_cython = os.getenv('PYDEVD_USE_CYTHON', None)
 dirname = os.path.dirname(os.path.dirname(__file__))
 USING_CYTHON = False
-# Do not show incorrect warning for .egg files for Remote debugger
-if not CYTHON_SUPPORTED or dirname.endswith('.egg'):
-    # Do not try to import cython extensions if cython isn't supported
-    use_cython = 'NO'
 
 
 def delete_old_compiled_extensions():
@@ -35,16 +31,16 @@ def delete_old_compiled_extensions():
                        "\"%s\" and \"%s\"" % (_pydevd_bundle_ext_dir, _pydevd_frame_eval_ext_dir))
 
 
-if use_cython == 'YES':
+if USE_CYTHON_FLAG in ENV_TRUE_LOWER_VALUES:
     # We must import the cython version if forcing cython
     from _pydevd_bundle.pydevd_cython_wrapper import trace_dispatch, global_cache_skips, global_cache_frame_skips, fix_top_level_trace_and_get_trace_func
     USING_CYTHON = True
 
-elif use_cython == 'NO':
+elif USE_CYTHON_FLAG in ENV_FALSE_LOWER_VALUES:
     # Use the regular version if not forcing cython
     from _pydevd_bundle.pydevd_trace_dispatch_regular import trace_dispatch, global_cache_skips, global_cache_frame_skips, fix_top_level_trace_and_get_trace_func  # @UnusedImport
 
-elif use_cython is None:
+else:
     # Regular: use fallback if not found and give message to user
     try:
         from _pydevd_bundle.pydevd_cython_wrapper import trace_dispatch, global_cache_skips, global_cache_frame_skips, fix_top_level_trace_and_get_trace_func
@@ -63,6 +59,4 @@ def delete_old_compiled_extensions():
     except ImportError:
         from _pydevd_bundle.pydevd_trace_dispatch_regular import trace_dispatch, global_cache_skips, global_cache_frame_skips, fix_top_level_trace_and_get_trace_func  # @UnusedImport
         pydev_log.show_compile_cython_command_line()
-else:
-    raise RuntimeError('Unexpected value for PYDEVD_USE_CYTHON: %s (accepted: YES, NO)' % (use_cython,))
 

src/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_frame_eval_main.py

@@ -1,38 +1,33 @@
 import os
-import sys
 
 from _pydev_bundle import pydev_log
 from _pydevd_bundle.pydevd_trace_dispatch import USING_CYTHON
-
-IS_PY36_OR_GREATER = sys.version_info >= (3, 6)
+from _pydevd_bundle.pydevd_constants import USE_CYTHON_FLAG, ENV_FALSE_LOWER_VALUES, \
+    ENV_TRUE_LOWER_VALUES, IS_PY36_OR_GREATER
 
 frame_eval_func = None
 stop_frame_eval = None
 dummy_trace_dispatch = None
 clear_thread_local_info = None
 
-use_cython = os.getenv('PYDEVD_USE_CYTHON', None)
 USING_FRAME_EVAL = False
 
 # "NO" means we should not use frame evaluation, 'YES' we should use it (and fail if not there) and unspecified uses if possible.
-use_frame_eval = os.environ.get('PYDEVD_USE_FRAME_EVAL', None)
+use_frame_eval = os.environ.get('PYDEVD_USE_FRAME_EVAL', '').lower()
 
-if use_frame_eval == 'NO' or use_cython == 'NO' or not USING_CYTHON:
+if use_frame_eval in ENV_FALSE_LOWER_VALUES or USE_CYTHON_FLAG in ENV_FALSE_LOWER_VALUES or not USING_CYTHON:
     pass
 
-elif use_frame_eval == 'YES':
+elif use_frame_eval in ENV_TRUE_LOWER_VALUES:
     # Fail if unable to use
     from _pydevd_frame_eval.pydevd_frame_eval_cython_wrapper import frame_eval_func, stop_frame_eval, dummy_trace_dispatch, clear_thread_local_info
     USING_FRAME_EVAL = True
 
-elif use_frame_eval is None:
+else:
     # Try to use if possible
     if IS_PY36_OR_GREATER:
         try:
             from _pydevd_frame_eval.pydevd_frame_eval_cython_wrapper import frame_eval_func, stop_frame_eval, dummy_trace_dispatch, clear_thread_local_info
             USING_FRAME_EVAL = True
         except ImportError:
             pydev_log.show_compile_cython_command_line()
-
-else:
-    raise RuntimeError('Unexpected value for PYDEVD_USE_FRAME_EVAL: %s (accepted: YES, NO)' % (use_frame_eval,))

src/debugpy/_vendored/pydevd/build_tools/build.py

@@ -158,10 +158,11 @@ def build():
 
 
 if __name__ == '__main__':
-    use_cython = os.getenv('PYDEVD_USE_CYTHON', None)
-    if use_cython == 'YES':
+    use_cython = os.getenv('PYDEVD_USE_CYTHON', '').lower()
+    # Note: don't import pydevd during build (so, accept just yes/no in this case).
+    if use_cython == 'yes':
         build()
-    elif use_cython == 'NO':
+    elif use_cython == 'no':
         remove_binaries(['.pyd', '.so'])
     elif use_cython is None:
         # Regular process
@@ -170,5 +171,5 @@ def build():
             generate_cython_module()
         build()
     else:
-        raise RuntimeError('Unexpected value for PYDEVD_USE_CYTHON: %s (accepted: YES, NO)' % (use_cython,))
+        raise RuntimeError('Unexpected value for PYDEVD_USE_CYTHON: %s (accepted: yes, no)' % (use_cython,))
 

src/debugpy/_vendored/pydevd/pydevd_attach_to_process/attach_script.py

@@ -1,65 +1,5 @@
 
 
-def load_python_helper_lib():
-    import sys
-    try:
-        import ctypes
-    except ImportError:
-        ctypes = None
-
-    # Note: we cannot use import platform because it may end up importing threading,
-    # but that should be ok because at this point we can only be in CPython (other
-    # implementations wouldn't get to this point in the attach process).
-    # IS_CPYTHON = platform.python_implementation() == 'CPython'
-    IS_CPYTHON = True
-
-    import os
-    IS_64BIT_PROCESS = sys.maxsize > (2 ** 32)
-    IS_WINDOWS = sys.platform == 'win32'
-    IS_LINUX = sys.platform in ('linux', 'linux2')
-    IS_MAC = sys.platform == 'darwin'
-
-    if not IS_CPYTHON or ctypes is None or sys.version_info[:2] > (3, 7):
-        return None
-
-    if IS_WINDOWS:
-        if IS_64BIT_PROCESS:
-            suffix = 'amd64'
-        else:
-            suffix = 'x86'
-
-        filename = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'pydevd_attach_to_process', 'attach_%s.dll' % (suffix,))
-
-    elif IS_LINUX:
-        if IS_64BIT_PROCESS:
-            suffix = 'amd64'
-        else:
-            suffix = 'x86'
-
-        filename = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'pydevd_attach_to_process', 'attach_linux_%s.so' % (suffix,))
-
-    elif IS_MAC:
-        if IS_64BIT_PROCESS:
-            suffix = 'x86_64.dylib'
-        else:
-            suffix = 'x86.dylib'
-
-        filename = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'pydevd_attach_to_process', 'attach_%s' % (suffix,))
-
-    else:
-        return None
-
-    if not os.path.exists(filename):
-        return None
-
-    try:
-        # Load as pydll so that we don't release the gil.
-        lib = ctypes.pydll.LoadLibrary(filename)
-        return lib
-    except:
-        return None
-
-
 def get_main_thread_instance(threading):
     if hasattr(threading, 'main_thread'):
         return threading.main_thread()

src/debugpy/_vendored/pydevd/pydevd_tracing.py

@@ -1,6 +1,6 @@
-
 from _pydevd_bundle.pydevd_constants import get_frame, IS_CPYTHON, IS_64BIT_PROCESS, IS_WINDOWS, \
-    IS_LINUX, IS_MAC, IS_PY2, DebugInfoHolder, ForkSafeLock
+    IS_LINUX, IS_MAC, IS_PY2, DebugInfoHolder, ForkSafeLock, LOAD_NATIVE_LIB_FLAG, \
+    ENV_FALSE_LOWER_VALUES
 from _pydev_imps._pydev_saved_modules import thread, threading
 from _pydev_bundle import pydev_log, pydev_monkey
 from os.path import os
@@ -109,7 +109,7 @@ def restore_sys_set_trace_func():
 
 
 def load_python_helper_lib():
-    if not IS_CPYTHON or ctypes is None or sys.version_info[:2] > (3, 8):
+    if not IS_CPYTHON or ctypes is None or sys.version_info[:2] > (3, 8) or hasattr(sys, 'gettotalrefcount') or LOAD_NATIVE_LIB_FLAG in ENV_FALSE_LOWER_VALUES:
         return None
 
     if IS_WINDOWS:
@@ -159,8 +159,11 @@ def load_python_helper_lib():
 def set_trace_to_threads(tracing_func):
     lib = load_python_helper_lib()
     if lib is None:  # This is the case if it's not CPython.
+        pydev_log.info('Unable to load helper lib to set tracing to all threads (unsupported python vm).')
         return -1
 
+    pydev_log.info('Successfully Loaded helper lib to set tracing to all threads.')
+
     ret = 0
     set_trace_func = TracingFunctionHolder._original_tracing or sys.settrace
 

src/debugpy/_vendored/pydevd/tests_python/check_debug_python.py

@@ -0,0 +1,38 @@
+import sys
+import threading
+from _pydev_bundle import pydev_log
+
+
+def check():
+    with pydev_log.log_context(3, sys.stderr):
+        assert hasattr(sys, 'gettotalrefcount')
+        import pydevd_tracing
+
+        proceed1 = threading.Event()
+        proceed2 = threading.Event()
+
+        class SomeThread(threading.Thread):
+
+            def run(self):
+                proceed1.set()
+                proceed2.wait()
+
+        t = SomeThread()
+        t.start()
+        proceed1.wait()
+        try:
+
+            def some_func(frame, event, arg):
+                return some_func
+
+            pydevd_tracing.set_trace_to_threads(some_func)
+        finally:
+            proceed2.set()
+
+        lib = pydevd_tracing.load_python_helper_lib()
+        assert lib is None
+        print('Finished OK')
+
+
+if __name__ == '__main__':
+    check()