Merge pull request #1650 from zooba/issue-1648

zooba · web-flow · commit 120e45035173 · 2016-09-22T12:04:26.000-07:00
Fixes #1648 Fix debug-attach and profiling for Python 3.6
diff --git a/Python/Product/Core/Core.csproj b/Python/Product/Core/Core.csproj
@@ -77,6 +77,14 @@
       <Project>{89d51398-a003-44ba-b1b2-cfc6f8396d7e}</Project>
       <Name>Microsoft.PythonTools.BuildTasks</Name>
     </ProjectReference>
+    <ProjectReference Include="..\DebuggerHelper\DebuggerHelper.vcxproj">
+      <Project>{25956dfa-17a2-4109-b9e5-d46cce1ed52f}</Project>
+      <Name>DebuggerHelper</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\DebuggerHelper\DebuggerHelperX86.vcxproj">
+      <Project>{a2a795f7-27d0-4801-88da-95b368f070ad}</Project>
+      <Name>DebuggerHelperX86</Name>
+    </ProjectReference>
     <ProjectReference Include="..\Debugger\Debugger.csproj">
       <Project>{DECC7971-FA58-4DB0-9561-BFFADD393BBD}</Project>
       <Name>Microsoft.PythonTools.Debugger</Name>
@@ -93,6 +101,14 @@
       <Project>{3814d9db-10e6-4478-bd98-6c5840612af8}</Project>
       <Name>Microsoft.PythonTools.ProjectWizards</Name>
     </ProjectReference>
+    <ProjectReference Include="..\PyDebugAttach\PyDebugAttach.vcxproj">
+      <Project>{ac19caa0-5c69-4b20-8a18-d4b6b65f22b8}</Project>
+      <Name>PyDebugAttach</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\PyDebugAttach\PyDebugAttachX86.vcxproj">
+      <Project>{70e7eb43-81d3-4aa0-9870-0b304732aff2}</Project>
+      <Name>PyDebugAttachX86</Name>
+    </ProjectReference>
     <ProjectReference Include="..\PythonTools\PythonTools.csproj">
       <Project>{fa7be5f5-e04f-4613-b7ac-70ce10d1bb68}</Project>
       <Name>Microsoft.PythonTools</Name>
diff --git a/Python/Product/PyDebugAttach/PyDebugAttach.cpp b/Python/Product/PyDebugAttach/PyDebugAttach.cpp
@@ -722,8 +722,8 @@ long GetPythonThreadId(PythonVersion version, PyThreadState* curThread) {
         threadId = ((PyThreadState_25_27*)curThread)->thread_id;
     } else if (PyThreadState_30_33::IsFor(version)) {
         threadId = ((PyThreadState_30_33*)curThread)->thread_id;
-    } else if (PyThreadState_34_35::IsFor(version)) {
-        threadId = ((PyThreadState_34_35*)curThread)->thread_id;
+    } else if (PyThreadState_34_36::IsFor(version)) {
+        threadId = ((PyThreadState_34_36*)curThread)->thread_id;
     }
     return threadId;
 }
@@ -855,17 +855,14 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) {
         auto getThreadTls = (PyThread_get_key_value*)GetProcAddress(module, "PyThread_get_key_value");
         auto setThreadTls = (PyThread_set_key_value*)GetProcAddress(module, "PyThread_set_key_value");
         auto delThreadTls = (PyThread_delete_key_value*)GetProcAddress(module, "PyThread_delete_key_value");
-        auto pyGilStateEnsure = (PyGILState_EnsureFunc*)GetProcAddress(module, "PyGILState_Ensure");
-        auto pyGilStateRelease = (PyGILState_ReleaseFunc*)GetProcAddress(module, "PyGILState_Release");
         auto PyCFrame_Type = (PyTypeObject*)GetProcAddress(module, "PyCFrame_Type");
 
         if (addPendingCall == nullptr || curPythonThread == nullptr || interpHead == nullptr || gilEnsure == nullptr || gilRelease == nullptr || threadHead == nullptr ||
             initThreads == nullptr || releaseLock == nullptr || threadsInited == nullptr || threadNext == nullptr || threadSwap == nullptr ||
             pyDictNew == nullptr || pyCompileString == nullptr || pyEvalCode == nullptr || getDictItem == nullptr || call == nullptr ||
             getBuiltins == nullptr || dictSetItem == nullptr || intFromLong == nullptr || pyErrRestore == nullptr || pyErrFetch == nullptr ||
             errOccurred == nullptr || pyImportMod == nullptr || pyGetAttr == nullptr || pyNone == nullptr || pySetAttr == nullptr || boolFromLong == nullptr ||
-            getThreadTls == nullptr || setThreadTls == nullptr || delThreadTls == nullptr ||
-            pyGilStateEnsure == nullptr || pyGilStateRelease == nullptr) {
+            getThreadTls == nullptr || setThreadTls == nullptr || delThreadTls == nullptr) {
                 // we're missing some APIs, we cannot attach.
                 connInfo.ReportError(ConnError_PythonNotFound);
                 return false;
@@ -975,13 +972,13 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) {
                             // Py_InitThreads to bring up multi-threading.
                             // Some context here: http://bugs.python.org/issue11329
                             // http://pytools.codeplex.com/workitem/834
-                            gilState = pyGilStateEnsure();
+                            gilState = gilEnsure();
                         }
                         initThreads();
 
                         if (version >= PythonVersion_32) {
                             // we will release the GIL here
-                            pyGilStateRelease(gilState);
+                            gilRelease(gilState);
                         } else {
                             releaseLock();
                         }
@@ -1149,8 +1146,8 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) {
                             frame = ((PyThreadState_25_27*)curThread)->frame;
                         } else if (PyThreadState_30_33::IsFor(version)) {
                             frame = ((PyThreadState_30_33*)curThread)->frame;
-                        } else if (PyThreadState_34_35::IsFor(version)) {
-                            frame = ((PyThreadState_34_35*)curThread)->frame;
+                        } else if (PyThreadState_34_36::IsFor(version)) {
+                            frame = ((PyThreadState_34_36*)curThread)->frame;
                         }
 
                         auto threadObj = PyObjectHolder(isDebug, call(new_thread.ToPython(), pyThreadId.ToPython(), pyTrue, frame, NULL));
@@ -1342,8 +1339,8 @@ int TraceGeneral(int interpreterId, PyObject *obj, PyFrameObject *frame, int wha
             ((PyThreadState_25_27*)curThread)->c_tracefunc(((PyThreadState_25_27*)curThread)->c_traceobj, frame, what, arg);
         } else if (PyThreadState_30_33::IsFor(version)) {
             ((PyThreadState_30_33*)curThread)->c_tracefunc(((PyThreadState_30_33*)curThread)->c_traceobj, frame, what, arg);
-        } else if (PyThreadState_34_35::IsFor(version)) {
-            ((PyThreadState_34_35*)curThread)->c_tracefunc(((PyThreadState_34_35*)curThread)->c_traceobj, frame, what, arg);
+        } else if (PyThreadState_34_36::IsFor(version)) {
+            ((PyThreadState_34_36*)curThread)->c_tracefunc(((PyThreadState_34_36*)curThread)->c_traceobj, frame, what, arg);
         }
     }
     return 0;
@@ -1387,8 +1384,8 @@ void SetInitialTraceFunc(DWORD interpreterId, PyThreadState *thread) {
         gilstate_counter = ((PyThreadState_25_27*)thread)->gilstate_counter;
     } else if (PyThreadState_30_33::IsFor(version)) {
         gilstate_counter = ((PyThreadState_30_33*)thread)->gilstate_counter;
-    } else if (PyThreadState_34_35::IsFor(version)) {
-        gilstate_counter = ((PyThreadState_34_35*)thread)->gilstate_counter;
+    } else if (PyThreadState_34_36::IsFor(version)) {
+        gilstate_counter = ((PyThreadState_34_36*)thread)->gilstate_counter;
     }
 
     if (gilstate_counter == 1) {
diff --git a/Python/Product/VsPyProf/PythonApi.cpp b/Python/Product/VsPyProf/PythonApi.cpp
@@ -88,7 +88,7 @@ VsPyProf* VsPyProf::Create(HMODULE pythonModule) {
                 }
 
                 if ((major == 2 && (minor >= 4 && minor <= 7)) ||
-                    (major == 3 && (minor >= 0 && minor <= 5))) {
+                    (major == 3 && (minor >= 0 && minor <= 6))) {
                         return new VsPyProf(pythonModule,
                             major,
                             minor,
@@ -134,6 +134,8 @@ bool VsPyProf::GetUserToken(PyFrameObject* frameObj, DWORD_PTR& func, DWORD_PTR&
             filename = ((PyCodeObject30_32*)codeObj)->co_filename;
         } else if (PyCodeObject33_35::IsFor(MajorVersion, MinorVersion)) {
             filename = ((PyCodeObject33_35*)codeObj)->co_filename;
+        } else if (PyCodeObject36::IsFor(MajorVersion, MinorVersion)) {
+            filename = ((PyCodeObject36*)codeObj)->co_filename;
         }
         module = (DWORD_PTR)filename;
 
@@ -210,6 +212,9 @@ bool VsPyProf::GetUserToken(PyFrameObject* frameObj, DWORD_PTR& func, DWORD_PTR&
             } else if (PyCodeObject33_35::IsFor(MajorVersion, MinorVersion)) {
                 RegisterName(func, ((PyCodeObject33_35*)codeObj)->co_name, &moduleName);
                 lineno = ((PyCodeObject33_35*)codeObj)->co_firstlineno;
+            } else if (PyCodeObject36::IsFor(MajorVersion, MinorVersion)) {
+                RegisterName(func, ((PyCodeObject36*)codeObj)->co_name, &moduleName);
+                lineno = ((PyCodeObject36*)codeObj)->co_firstlineno;
             }
 
             // give the profiler the line number of this function
@@ -242,6 +247,9 @@ wstring VsPyProf::GetClassNameFromFrame(PyFrameObject* frameObj, PyObject *codeO
         } else if (PyCodeObject33_35::IsFor(MajorVersion, MinorVersion)) {
             argCount = ((PyCodeObject33_35*)codeObj)->co_argcount;
             argNames = (PyTupleObject*)((PyCodeObject33_35*)codeObj)->co_varnames;
+        } else if (PyCodeObject36::IsFor(MajorVersion, MinorVersion)) {
+            argCount = ((PyCodeObject36*)codeObj)->co_argcount;
+            argNames = (PyTupleObject*)((PyCodeObject36*)codeObj)->co_varnames;
         }
 
         if (argCount != 0 && argNames->ob_type == PyTuple_Type) {
@@ -252,8 +260,8 @@ wstring VsPyProf::GetClassNameFromFrame(PyFrameObject* frameObj, PyObject *codeO
                 PyObject* self = nullptr;
                 if (PyFrameObject25_33::IsFor(MajorVersion, MinorVersion)) {
                     self = ((PyFrameObject25_33*)frameObj)->f_localsplus[0];
-                } else if (PyFrameObject34_35::IsFor(MajorVersion, MinorVersion)) {
-                    self = ((PyFrameObject34_35*)frameObj)->f_localsplus[0];
+                } else if (PyFrameObject34_36::IsFor(MajorVersion, MinorVersion)) {
+                    self = ((PyFrameObject34_36*)frameObj)->f_localsplus[0];
                 }
                 return GetClassNameFromSelf(self, codeObj);
             }
@@ -281,6 +289,8 @@ wstring VsPyProf::GetClassNameFromSelf(PyObject* self, PyObject *codeObj) {
             nameObj = ((PyCodeObject30_32*)codeObj)->co_name;
         } else if (PyCodeObject33_35::IsFor(MajorVersion, MinorVersion)) {
             nameObj = ((PyCodeObject33_35*)codeObj)->co_name;
+        } else if (PyCodeObject36::IsFor(MajorVersion, MinorVersion)) {
+            nameObj = ((PyCodeObject36*)codeObj)->co_name;
         }
         GetNameAscii(nameObj, codeName);
 
diff --git a/Python/Product/VsPyProf/python.h b/Python/Product/VsPyProf/python.h
@@ -28,7 +28,8 @@ enum PythonVersion {
     PythonVersion_32 = 0x0302,
     PythonVersion_33 = 0x0303,
     PythonVersion_34 = 0x0304,
-    PythonVersion_35 = 0x0305
+    PythonVersion_35 = 0x0305,
+    PythonVersion_36 = 0x0306
 };
 
 
@@ -144,7 +145,38 @@ class PyCodeObject33_35 : public PyObject {
     }
 };
 
-// 2.5 - 3.1
+// 3.6
+class PyCodeObject36 : public PyObject {
+public:
+    int co_argcount;            /* #arguments, except *args */
+    int co_kwonlyargcount;      /* #keyword only arguments */
+    int co_nlocals;             /* #local variables */
+    int co_stacksize;           /* #entries needed for evaluation stack */
+    int co_flags;               /* CO_..., see below */
+    int co_firstlineno;         /* first source line number */
+    PyObject *co_code;          /* instruction opcodes */
+    PyObject *co_consts;        /* list (constants used) */
+    PyObject *co_names;         /* list of strings (names used) */
+    PyObject *co_varnames;      /* tuple of strings (local variable names) */
+    PyObject *co_freevars;      /* tuple of strings (free variable names) */
+    PyObject *co_cellvars;      /* tuple of strings (cell variable names) */
+    /* The rest doesn't count for hash or comparisons */
+    unsigned char *co_cell2arg; /* Maps cell vars which are arguments. */
+    PyObject *co_filename;      /* unicode (where it was loaded from) */
+    PyObject *co_name;          /* unicode (name, for reference) */
+    PyObject *co_lnotab;        /* string (encoding addr<->lineno mapping) */
+    void *co_zombieframe;       /* for optimization only (see frameobject.c) */
+
+    static bool IsFor(int majorVersion, int minorVersion) {
+        return majorVersion == 3 && minorVersion >= 6;
+    }
+
+    static bool IsFor(PythonVersion version) {
+        return version >= PythonVersion_36;
+    }
+};
+
+// 2.5 - 3.6
 class PyFunctionObject : public PyObject {
 public:
     PyObject *func_code;    /* A code object */
@@ -175,7 +207,7 @@ typedef struct {
     long hash;          /* Hash value; -1 if not set */
 } PyUnicodeObject;
 
-// 2.4 - 3.5 compatible
+// 2.4 - 3.6 compatible
 class PyFrameObject : public PyVarObject {
 public:
     PyFrameObject *f_back;  /* previous frame, or NULL */
@@ -216,7 +248,7 @@ class PyFrameObject25_33 : public PyFrameObject {
     }
 };
 
-class PyFrameObject34_35 : public PyFrameObject {
+class PyFrameObject34_36 : public PyFrameObject {
 public:
     /* Borrowed reference to a generator, or NULL */
     PyObject *f_gen;
@@ -231,14 +263,14 @@ class PyFrameObject34_35 : public PyFrameObject {
     PyObject *f_localsplus[1];    /* locals+stack, dynamically sized */
 
     static bool IsFor(int majorVersion, int minorVersion) {
-        return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 5;
+        return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 6;
     }
 };
 
 
 typedef void (*destructor)(PyObject *);
 
-// 2.4 - 3.5
+// 2.4 - 3.6
 class PyMethodDef {
 public:
     char    *ml_name;    /* The name of the built-in function/method */
@@ -261,7 +293,7 @@ class PyTypeObject : public PyVarObject {
     void* tp_setattr;
     union {
         void* tp_compare; /* 2.4 - 3.4 */
-        void* tp_as_async; /* 3.5 */
+        void* tp_as_async; /* 3.5 - 3.6 */
     };
     void* tp_repr;
 
@@ -331,7 +363,7 @@ class PyTypeObject : public PyVarObject {
     unsigned int tp_version_tag;
 };
 
-// 2.4 - 3.5
+// 2.4 - 3.6
 class PyTupleObject : public PyVarObject {
 public:
     PyObject *ob_item[1];
@@ -342,7 +374,7 @@ class PyTupleObject : public PyVarObject {
      */
 };
 
-// 2.4 - 3.5
+// 2.4 - 3.6
 class PyCFunctionObject : public PyObject {
 public:
     PyMethodDef *m_ml;      /* Description of the C function to call */
@@ -473,7 +505,7 @@ class PyThreadState_30_33 : public PyThreadState {
     }
 };
 
-class PyThreadState_34_35 : public PyThreadState {
+class PyThreadState_34_36 : public PyThreadState {
 public:
     PyThreadState *prev;
     PyThreadState *next;
@@ -513,11 +545,11 @@ class PyThreadState_34_35 : public PyThreadState {
 
     /* XXX signal handlers should also be here */
     static bool IsFor(int majorVersion, int minorVersion) {
-        return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 5;
+        return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 6;
     }
 
     static bool IsFor(PythonVersion version) {
-        return version >= PythonVersion_34 && version <= PythonVersion_35;
+        return version >= PythonVersion_34 && version <= PythonVersion_36;
     }
 };
 
@@ -570,6 +602,7 @@ static PythonVersion GetPythonVersion(HMODULE hMod) {
                 case '3': return PythonVersion_33;
                 case '4': return PythonVersion_34;
                 case '5': return PythonVersion_35;
+                case '6': return PythonVersion_36;
                 }
             }
         }
diff --git a/Python/Tests/Core/DebugReplEvaluatorTests.cs b/Python/Tests/Core/DebugReplEvaluatorTests.cs
@@ -116,7 +116,7 @@ public void ErrorInInput() {
             Assert.AreEqual(@"Traceback (most recent call last):
   File ""<debug input>"", line 1, in <module>
 NameError: name 'does_not_exist' is not defined
-", _window.Error);
+".Replace("\r\n", "\n"), _window.Error.Replace("\r\n", "\n"));
         }
 
         [TestMethod, Priority(3)]
@@ -434,6 +434,21 @@ internal override PythonVersion Version {
         }
     }
 
+    [TestClass]
+    public class DebugReplEvaluatorTests36 : DebugReplEvaluatorTests {
+        [ClassInitialize]
+        public static new void DoDeployment(TestContext context) {
+            AssertListener.Initialize();
+            PythonTestData.Deploy();
+        }
+
+        internal override PythonVersion Version {
+            get {
+                return PythonPaths.Python36 ?? PythonPaths.Python36_x64;
+            }
+        }
+    }
+
     [TestClass]
     public class DebugReplEvaluatorTests27 : DebugReplEvaluatorTests {
         [ClassInitialize]
diff --git a/Python/Tests/DebuggerTests/AttachTests.cs b/Python/Tests/DebuggerTests/AttachTests.cs
@@ -807,7 +807,10 @@ public void AttachPtvsd() {
             string script = TestData.GetPath(@"TestData\DebuggerProject\AttachPtvsd.py");
             var psi = new ProcessStartInfo(Version.InterpreterPath, PtvsdInterpreterArguments + " \"" + script + "\"") {
                 WorkingDirectory = TestData.GetPath(),
-                UseShellExecute = false
+                UseShellExecute = false,
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                CreateNoWindow = true
             };
 
             var p = Process.Start(psi);
@@ -822,7 +825,7 @@ public void AttachPtvsd() {
                         break;
                     } catch (SocketException) {
                         // Failed to connect - the process might have not started yet, so keep trying a few more times.
-                        if (i >= 5) {
+                        if (i >= 5 || p.HasExited) {
                             throw;
                         }
                     }
@@ -862,6 +865,8 @@ public void AttachPtvsd() {
                     DetachProcess(proc);
                 }
             } finally {
+                Console.WriteLine(p.StandardOutput.ReadToEnd());
+                Console.WriteLine(p.StandardError.ReadToEnd());
                 DisposeProcess(p);
             }
         }
@@ -1046,6 +1051,28 @@ internal override PythonVersion Version {
         }
     }
 
+    [TestClass]
+    public class AttachTests36 : AttachTests {
+        internal override PythonVersion Version {
+            get {
+                return PythonPaths.Python36;
+            }
+        }
+
+        public override void AttachNewThread_PyThreadState_New() {
+            // PyEval_AcquireLock deprecated in 3.2
+        }
+    }
+
+    [TestClass]
+    public class AttachTests36_x64 : AttachTests35 {
+        internal override PythonVersion Version {
+            get {
+                return PythonPaths.Python36_x64;
+            }
+        }
+    }
+
     [TestClass]
     public class AttachTests25 : AttachTests {
         internal override PythonVersion Version {
diff --git a/Python/Tests/DebuggerTests/DebuggerTests.cs b/Python/Tests/DebuggerTests/DebuggerTests.cs
diff --git a/Python/Tests/ProfilingUITests/ProfilingTests.cs b/Python/Tests/ProfilingUITests/ProfilingTests.cs
