Fix inconsistencies in handling files. Fixes microsoft/ptvsd#1636

Author: fabioz

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_dont_trace_files.py

@@ -8,6 +8,20 @@
 LIB_FILE = 1
 PYDEV_FILE = 2
 
+DONT_TRACE_DIRS = {
+    '_pydev_bundle': PYDEV_FILE,
+    '_pydev_imps': PYDEV_FILE,
+    '_pydev_runfiles': PYDEV_FILE,
+    '_pydevd_bundle': PYDEV_FILE,
+    '_pydevd_frame_eval': PYDEV_FILE,
+    'pydev_ipython': PYDEV_FILE,
+    'pydev_sitecustomize': PYDEV_FILE,
+    'pydevd_attach_to_process': PYDEV_FILE,
+    'pydevd_concurrency_analyser': PYDEV_FILE,
+    'pydevd_plugins': PYDEV_FILE,
+    'test_pydevd_reload': PYDEV_FILE,
+}
+
 DONT_TRACE = {
     # commonly used things from the stdlib that we don't want to trace
     'Queue.py':LIB_FILE,
@@ -55,6 +69,14 @@
     'pydev_monkey_qt.py': PYDEV_FILE,
     'pydev_override.py': PYDEV_FILE,
     'pydev_run_in_console.py': PYDEV_FILE,
+    'pydev_runfiles.py': PYDEV_FILE,
+    'pydev_runfiles_coverage.py': PYDEV_FILE,
+    'pydev_runfiles_nose.py': PYDEV_FILE,
+    'pydev_runfiles_parallel.py': PYDEV_FILE,
+    'pydev_runfiles_parallel_client.py': PYDEV_FILE,
+    'pydev_runfiles_pytest2.py': PYDEV_FILE,
+    'pydev_runfiles_unittest.py': PYDEV_FILE,
+    'pydev_runfiles_xml_rpc.py': PYDEV_FILE,
     'pydev_umd.py': PYDEV_FILE,
     'pydev_versioncheck.py': PYDEV_FILE,
     'pydevconsole.py': PYDEV_FILE,

src/debugpy/_vendored/pydevd/build_tools/generate_code.py

@@ -108,6 +108,10 @@ def generate_dont_trace_files():
 LIB_FILE = 1
 PYDEV_FILE = 2
 
+DONT_TRACE_DIRS = {
+%(pydev_dirs)s
+}
+
 DONT_TRACE = {
     # commonly used things from the stdlib that we don't want to trace
     'Queue.py':LIB_FILE,
@@ -135,8 +139,13 @@ def generate_dont_trace_files():
 '''
 
     pydev_files = []
+    pydev_dirs = []
 
     for root, dirs, files in os.walk(root_dir):
+        for d in dirs:
+            if 'pydev' in d:
+                pydev_dirs.append("    '%s': PYDEV_FILE," % (d,))
+
         for d in [
             '.git',
             '.settings',
@@ -154,7 +163,6 @@ def generate_dont_trace_files():
             'test_pydevd_reload',
             'third_party',
             '__pycache__',
-            '_pydev_runfiles',
             'pydev_ipython',
             ]:
             try:
@@ -176,7 +184,10 @@ def generate_dont_trace_files():
                     ):
                     pydev_files.append("    '%s': PYDEV_FILE," % (f,))
 
-    contents = template % (dict(pydev_files='\n'.join(sorted(pydev_files))))
+    contents = template % (dict(
+        pydev_files='\n'.join(sorted(pydev_files)),
+        pydev_dirs='\n'.join(sorted(pydev_dirs)),
+    ))
     assert 'pydevd.py' in contents
     assert 'pydevd_dont_trace.py' in contents
     with open(os.path.join(root_dir, '_pydevd_bundle', 'pydevd_dont_trace_files.py'), 'w') as stream:

src/debugpy/_vendored/pydevd/pydevd.py

@@ -43,7 +43,7 @@
     ForkSafeLock)
 from _pydevd_bundle.pydevd_defaults import PydevdCustomization  # Note: import alias used on pydev_monkey.
 from _pydevd_bundle.pydevd_custom_frames import CustomFramesContainer, custom_frames_container_init
-from _pydevd_bundle.pydevd_dont_trace_files import DONT_TRACE, PYDEV_FILE, LIB_FILE
+from _pydevd_bundle.pydevd_dont_trace_files import DONT_TRACE, PYDEV_FILE, LIB_FILE, DONT_TRACE_DIRS
 from _pydevd_bundle.pydevd_extension_api import DebuggerEventHandler
 from _pydevd_bundle.pydevd_frame_utils import add_exception_to_frame, remove_exception_from_frame
 from _pydevd_bundle.pydevd_net_command_factory_xml import NetCommandFactory
@@ -605,6 +605,7 @@ def new_trace_dispatch(frame, event, arg):
         self.collect_return_info = collect_return_info
         self.get_exception_breakpoint = get_exception_breakpoint
         self._dont_trace_get_file_type = DONT_TRACE.get
+        self._dont_trace_dirs_get_file_type = DONT_TRACE_DIRS.get
         self.PYDEV_FILE = PYDEV_FILE
         self.LIB_FILE = LIB_FILE
 
@@ -750,7 +751,25 @@ def _internal_get_file_type(self, abs_real_path_and_basename):
         if abs_real_path_and_basename[0].startswith(('<builtin', '<attrs')):
             # In PyPy "<builtin> ..." can appear and should be ignored for the user.
             return self.PYDEV_FILE
-        return self._dont_trace_get_file_type(basename)
+        file_type = self._dont_trace_get_file_type(basename)
+        if file_type is not None:
+            return file_type
+
+        if basename.startswith('__init__.py'):
+            # i.e.: ignore the __init__ files inside pydevd (the other
+            # files are ignored just by their name).
+            abs_path = abs_real_path_and_basename[0]
+            i = max(abs_path.rfind('/'), abs_path.rfind('\\'))
+            if i:
+                abs_path = abs_path[0:i]
+                i = max(abs_path.rfind('/'), abs_path.rfind('\\'))
+                if i:
+                    dirname = abs_path[i + 1:]
+                    # At this point, something as:
+                    # "my_path\_pydev_runfiles\__init__.py"
+                    # is now  "_pydev_runfiles".
+                    return self._dont_trace_dirs_get_file_type(dirname)
+        return None
 
     def dont_trace_external_files(self, abs_path):
         '''
@@ -837,6 +856,7 @@ def get_file_type(self, frame, abs_real_path_and_basename=None, _cache_file_type
             if file_type is None:
                 if self.dont_trace_external_files(abs_real_path_and_basename[0]):
                     file_type = PYDEV_FILE
+
             _cache_file_type[cache_key] = file_type
             return file_type
 

src/debugpy/_vendored/pydevd/pydevd_file_utils.py

@@ -747,10 +747,15 @@ def _norm_file_to_client(filename, cache=norm_filename_to_client_container):
 
 # For given file f returns tuple of its absolute path, real path and base name
 def get_abs_path_real_path_and_base_from_file(
-        f, NORM_PATHS_AND_BASE_CONTAINER=NORM_PATHS_AND_BASE_CONTAINER):
+        filename, NORM_PATHS_AND_BASE_CONTAINER=NORM_PATHS_AND_BASE_CONTAINER):
     try:
-        return NORM_PATHS_AND_BASE_CONTAINER[f]
+        return NORM_PATHS_AND_BASE_CONTAINER[filename]
     except:
+        f = filename
+        if not f:
+            # i.e.: it's possible that the user compiled code with an empty string (consider
+            # it as <string> in this case).
+            f = '<string>'
         if _NormPaths is None:  # Interpreter shutdown
             i = max(f.rfind('/'), f.rfind('\\'))
             return (f, f, f[i + 1:])
@@ -770,7 +775,7 @@ def get_abs_path_real_path_and_base_from_file(
             i = max(f.rfind('/'), f.rfind('\\'))
             base = f[i + 1:]
         ret = abs_path, real_path, base
-        NORM_PATHS_AND_BASE_CONTAINER[f] = ret
+        NORM_PATHS_AND_BASE_CONTAINER[filename] = ret
         return ret
 
 
@@ -784,8 +789,14 @@ def get_abs_path_real_path_and_base_from_frame(frame):
             # files from eggs in Python 2.7 have paths like build/bdist.linux-x86_64/egg/<path-inside-egg>
             f = frame.f_globals['__file__']
 
-        if get_abs_path_real_path_and_base_from_file is None:  # Interpreter shutdown
-            return f
+        if get_abs_path_real_path_and_base_from_file is None:
+            # Interpreter shutdown
+            if not f:
+                # i.e.: it's possible that the user compiled code with an empty string (consider
+                # it as <string> in this case).
+                f = '<string>'
+            i = max(f.rfind('/'), f.rfind('\\'))
+            return f, f, f[i + 1:]
 
         ret = get_abs_path_real_path_and_base_from_file(f)
         # Also cache based on the frame.f_code.co_filename (if we had it inside build/bdist it can make a difference).

src/debugpy/_vendored/pydevd/tests_python/resources/my_code/my_code_on_entry.py

@@ -0,0 +1 @@
+print('my code on entry')

src/debugpy/_vendored/pydevd/tests_python/resources/not_my_code/empty_file.py

@@ -0,0 +1,7 @@
+if __name__ == '__main__':
+    import sys
+    import os
+    sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+
+    from my_code import my_code_on_entry
+    print('TEST SUCEEDED')

src/debugpy/_vendored/pydevd/tests_python/resources/not_my_code/main_on_entry.py

@@ -0,0 +1,7 @@
+if __name__ == '__main__':
+    import sys
+    import os
+    sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+
+    import empty_file
+    print('TEST SUCEEDED')

src/debugpy/_vendored/pydevd/tests_python/resources/not_my_code/main_on_entry2.py

@@ -134,7 +134,8 @@ def wait_for_thread_stopped(self, reason='breakpoint', line=None, file=None, nam
                 if isinstance(path, bytes):
                     path = path.decode('utf-8')
 
-            assert path.endswith(file)
+            if not path.endswith(file):
+                raise AssertionError('Expected path: %s to end with: %s' % (path, file))
         if name is not None:
             assert json_hit.stack_trace_response.body.stackFrames[0]['name'] == name
         if line is not None:
@@ -3758,6 +3759,50 @@ def update_command_line_args(self, args):
         writer.finished_ok = True
 
 
+def test_stop_on_entry(case_setup):
+    with case_setup.test_file('not_my_code/main_on_entry.py') as writer:
+        json_facade = JsonFacade(writer)
+        json_facade.write_launch(
+            justMyCode=False,
+            stopOnEntry=True,
+            rules=[
+                {'path': '**/not_my_code/**', 'include':False},
+            ]
+        )
+
+        json_facade.write_make_initial_run()
+        json_facade.wait_for_thread_stopped(
+            'entry',
+            file=(
+                # We need to match the end with the proper slash.
+                'my_code/__init__.py',
+                'my_code\\__init__.py'
+            )
+        )
+        json_facade.write_continue()
+        writer.finished_ok = True
+
+
+def test_stop_on_entry2(case_setup):
+    with case_setup.test_file('not_my_code/main_on_entry2.py') as writer:
+        json_facade = JsonFacade(writer)
+        json_facade.write_launch(
+            justMyCode=False,
+            stopOnEntry=True,
+            rules=[
+                {'path': '**/main_on_entry2.py', 'include':False},
+            ]
+        )
+
+        json_facade.write_make_initial_run()
+        json_facade.wait_for_thread_stopped(
+            'entry',
+            file='empty_file.py'
+        )
+        json_facade.write_continue()
+        writer.finished_ok = True
+
+
 @pytest.mark.parametrize('val', [True, False])
 def test_debug_options(case_setup, val):
     with case_setup.test_file('_debugger_case_debug_options.py') as writer: