Skip to content

Commit

Permalink
Merge pull request #1650 from zooba/issue-1648
Browse files Browse the repository at this point in the history 
Fixes #1648 Fix debug-attach and profiling for Python 3.6
  • Loading branch information
@zooba
zooba committed Sep 22, 2016
2 parents 60a2bac + 881ec1e commit 120e450
Show file tree
Hide file tree
Showing 8 changed files with 167 additions and 32 deletions.
16 changes: 16 additions & 0 deletions Python/Product/Core/Core.csproj
View file
Expand Up @@ -77,6 +77,14 @@
<Project>{89d51398-a003-44ba-b1b2-cfc6f8396d7e}</Project>
<Name>Microsoft.PythonTools.BuildTasks</Name>
</ProjectReference>
<ProjectReference Include="..\DebuggerHelper\DebuggerHelper.vcxproj">
<Project>{25956dfa-17a2-4109-b9e5-d46cce1ed52f}</Project>
<Name>DebuggerHelper</Name>
</ProjectReference>
<ProjectReference Include="..\DebuggerHelper\DebuggerHelperX86.vcxproj">
<Project>{a2a795f7-27d0-4801-88da-95b368f070ad}</Project>
<Name>DebuggerHelperX86</Name>
</ProjectReference>
<ProjectReference Include="..\Debugger\Debugger.csproj">
<Project>{DECC7971-FA58-4DB0-9561-BFFADD393BBD}</Project>
<Name>Microsoft.PythonTools.Debugger</Name>
Expand All @@ -93,6 +101,14 @@
<Project>{3814d9db-10e6-4478-bd98-6c5840612af8}</Project>
<Name>Microsoft.PythonTools.ProjectWizards</Name>
</ProjectReference>
<ProjectReference Include="..\PyDebugAttach\PyDebugAttach.vcxproj">
<Project>{ac19caa0-5c69-4b20-8a18-d4b6b65f22b8}</Project>
<Name>PyDebugAttach</Name>
</ProjectReference>
<ProjectReference Include="..\PyDebugAttach\PyDebugAttachX86.vcxproj">
<Project>{70e7eb43-81d3-4aa0-9870-0b304732aff2}</Project>
<Name>PyDebugAttachX86</Name>
</ProjectReference>
<ProjectReference Include="..\PythonTools\PythonTools.csproj">
<Project>{fa7be5f5-e04f-4613-b7ac-70ce10d1bb68}</Project>
<Name>Microsoft.PythonTools</Name>
Expand Down
25 changes: 11 additions & 14 deletions Python/Product/PyDebugAttach/PyDebugAttach.cpp
View file
Expand Up @@ -722,8 +722,8 @@ long GetPythonThreadId(PythonVersion version, PyThreadState* curThread) {
threadId = ((PyThreadState_25_27*)curThread)->thread_id;
} else if (PyThreadState_30_33::IsFor(version)) {
threadId = ((PyThreadState_30_33*)curThread)->thread_id;
} else if (PyThreadState_34_35::IsFor(version)) {
threadId = ((PyThreadState_34_35*)curThread)->thread_id;
} else if (PyThreadState_34_36::IsFor(version)) {
threadId = ((PyThreadState_34_36*)curThread)->thread_id;
}
return threadId;
}
Expand Down Expand Up @@ -855,17 +855,14 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) {
auto getThreadTls = (PyThread_get_key_value*)GetProcAddress(module, "PyThread_get_key_value");
auto setThreadTls = (PyThread_set_key_value*)GetProcAddress(module, "PyThread_set_key_value");
auto delThreadTls = (PyThread_delete_key_value*)GetProcAddress(module, "PyThread_delete_key_value");
auto pyGilStateEnsure = (PyGILState_EnsureFunc*)GetProcAddress(module, "PyGILState_Ensure");
auto pyGilStateRelease = (PyGILState_ReleaseFunc*)GetProcAddress(module, "PyGILState_Release");
auto PyCFrame_Type = (PyTypeObject*)GetProcAddress(module, "PyCFrame_Type");

if (addPendingCall == nullptr || curPythonThread == nullptr || interpHead == nullptr || gilEnsure == nullptr || gilRelease == nullptr || threadHead == nullptr ||
initThreads == nullptr || releaseLock == nullptr || threadsInited == nullptr || threadNext == nullptr || threadSwap == nullptr ||
pyDictNew == nullptr || pyCompileString == nullptr || pyEvalCode == nullptr || getDictItem == nullptr || call == nullptr ||
getBuiltins == nullptr || dictSetItem == nullptr || intFromLong == nullptr || pyErrRestore == nullptr || pyErrFetch == nullptr ||
errOccurred == nullptr || pyImportMod == nullptr || pyGetAttr == nullptr || pyNone == nullptr || pySetAttr == nullptr || boolFromLong == nullptr ||
getThreadTls == nullptr || setThreadTls == nullptr || delThreadTls == nullptr ||
pyGilStateEnsure == nullptr || pyGilStateRelease == nullptr) {
getThreadTls == nullptr || setThreadTls == nullptr || delThreadTls == nullptr) {
// we're missing some APIs, we cannot attach.
connInfo.ReportError(ConnError_PythonNotFound);
return false;
Expand Down Expand Up @@ -975,13 +972,13 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) {
// Py_InitThreads to bring up multi-threading.
// Some context here: http://bugs.python.org/issue11329
// http://pytools.codeplex.com/workitem/834
gilState = pyGilStateEnsure();
gilState = gilEnsure();
}
initThreads();

if (version >= PythonVersion_32) {
// we will release the GIL here
pyGilStateRelease(gilState);
gilRelease(gilState);
} else {
releaseLock();
}
Expand Down Expand Up @@ -1149,8 +1146,8 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) {
frame = ((PyThreadState_25_27*)curThread)->frame;
} else if (PyThreadState_30_33::IsFor(version)) {
frame = ((PyThreadState_30_33*)curThread)->frame;
} else if (PyThreadState_34_35::IsFor(version)) {
frame = ((PyThreadState_34_35*)curThread)->frame;
} else if (PyThreadState_34_36::IsFor(version)) {
frame = ((PyThreadState_34_36*)curThread)->frame;
}

auto threadObj = PyObjectHolder(isDebug, call(new_thread.ToPython(), pyThreadId.ToPython(), pyTrue, frame, NULL));
Expand Down Expand Up @@ -1342,8 +1339,8 @@ int TraceGeneral(int interpreterId, PyObject *obj, PyFrameObject *frame, int wha
((PyThreadState_25_27*)curThread)->c_tracefunc(((PyThreadState_25_27*)curThread)->c_traceobj, frame, what, arg);
} else if (PyThreadState_30_33::IsFor(version)) {
((PyThreadState_30_33*)curThread)->c_tracefunc(((PyThreadState_30_33*)curThread)->c_traceobj, frame, what, arg);
} else if (PyThreadState_34_35::IsFor(version)) {
((PyThreadState_34_35*)curThread)->c_tracefunc(((PyThreadState_34_35*)curThread)->c_traceobj, frame, what, arg);
} else if (PyThreadState_34_36::IsFor(version)) {
((PyThreadState_34_36*)curThread)->c_tracefunc(((PyThreadState_34_36*)curThread)->c_traceobj, frame, what, arg);
}
}
return 0;
Expand Down Expand Up @@ -1387,8 +1384,8 @@ void SetInitialTraceFunc(DWORD interpreterId, PyThreadState *thread) {
gilstate_counter = ((PyThreadState_25_27*)thread)->gilstate_counter;
} else if (PyThreadState_30_33::IsFor(version)) {
gilstate_counter = ((PyThreadState_30_33*)thread)->gilstate_counter;
} else if (PyThreadState_34_35::IsFor(version)) {
gilstate_counter = ((PyThreadState_34_35*)thread)->gilstate_counter;
} else if (PyThreadState_34_36::IsFor(version)) {
gilstate_counter = ((PyThreadState_34_36*)thread)->gilstate_counter;
}

if (gilstate_counter == 1) {
Expand Down
16 changes: 13 additions & 3 deletions Python/Product/VsPyProf/PythonApi.cpp
View file
Expand Up @@ -88,7 +88,7 @@ VsPyProf* VsPyProf::Create(HMODULE pythonModule) {
}

if ((major == 2 && (minor >= 4 && minor <= 7)) ||
(major == 3 && (minor >= 0 && minor <= 5))) {
(major == 3 && (minor >= 0 && minor <= 6))) {
return new VsPyProf(pythonModule,
major,
minor,
Expand Down Expand Up @@ -134,6 +134,8 @@ bool VsPyProf::GetUserToken(PyFrameObject* frameObj, DWORD_PTR& func, DWORD_PTR&
filename = ((PyCodeObject30_32*)codeObj)->co_filename;
} else if (PyCodeObject33_35::IsFor(MajorVersion, MinorVersion)) {
filename = ((PyCodeObject33_35*)codeObj)->co_filename;
} else if (PyCodeObject36::IsFor(MajorVersion, MinorVersion)) {
filename = ((PyCodeObject36*)codeObj)->co_filename;
}
module = (DWORD_PTR)filename;

Expand Down Expand Up @@ -210,6 +212,9 @@ bool VsPyProf::GetUserToken(PyFrameObject* frameObj, DWORD_PTR& func, DWORD_PTR&
} else if (PyCodeObject33_35::IsFor(MajorVersion, MinorVersion)) {
RegisterName(func, ((PyCodeObject33_35*)codeObj)->co_name, &moduleName);
lineno = ((PyCodeObject33_35*)codeObj)->co_firstlineno;
} else if (PyCodeObject36::IsFor(MajorVersion, MinorVersion)) {
RegisterName(func, ((PyCodeObject36*)codeObj)->co_name, &moduleName);
lineno = ((PyCodeObject36*)codeObj)->co_firstlineno;
}

// give the profiler the line number of this function
Expand Down Expand Up @@ -242,6 +247,9 @@ wstring VsPyProf::GetClassNameFromFrame(PyFrameObject* frameObj, PyObject *codeO
} else if (PyCodeObject33_35::IsFor(MajorVersion, MinorVersion)) {
argCount = ((PyCodeObject33_35*)codeObj)->co_argcount;
argNames = (PyTupleObject*)((PyCodeObject33_35*)codeObj)->co_varnames;
} else if (PyCodeObject36::IsFor(MajorVersion, MinorVersion)) {
argCount = ((PyCodeObject36*)codeObj)->co_argcount;
argNames = (PyTupleObject*)((PyCodeObject36*)codeObj)->co_varnames;
}

if (argCount != 0 && argNames->ob_type == PyTuple_Type) {
Expand All @@ -252,8 +260,8 @@ wstring VsPyProf::GetClassNameFromFrame(PyFrameObject* frameObj, PyObject *codeO
PyObject* self = nullptr;
if (PyFrameObject25_33::IsFor(MajorVersion, MinorVersion)) {
self = ((PyFrameObject25_33*)frameObj)->f_localsplus[0];
} else if (PyFrameObject34_35::IsFor(MajorVersion, MinorVersion)) {
self = ((PyFrameObject34_35*)frameObj)->f_localsplus[0];
} else if (PyFrameObject34_36::IsFor(MajorVersion, MinorVersion)) {
self = ((PyFrameObject34_36*)frameObj)->f_localsplus[0];
}
return GetClassNameFromSelf(self, codeObj);
}
Expand Down Expand Up @@ -281,6 +289,8 @@ wstring VsPyProf::GetClassNameFromSelf(PyObject* self, PyObject *codeObj) {
nameObj = ((PyCodeObject30_32*)codeObj)->co_name;
} else if (PyCodeObject33_35::IsFor(MajorVersion, MinorVersion)) {
nameObj = ((PyCodeObject33_35*)codeObj)->co_name;
} else if (PyCodeObject36::IsFor(MajorVersion, MinorVersion)) {
nameObj = ((PyCodeObject36*)codeObj)->co_name;
}
GetNameAscii(nameObj, codeName);

Expand Down
57 changes: 45 additions & 12 deletions Python/Product/VsPyProf/python.h
View file
Expand Up @@ -28,7 +28,8 @@ enum PythonVersion {
PythonVersion_32 = 0x0302,
PythonVersion_33 = 0x0303,
PythonVersion_34 = 0x0304,
PythonVersion_35 = 0x0305
PythonVersion_35 = 0x0305,
PythonVersion_36 = 0x0306
};


Expand Down Expand Up @@ -144,7 +145,38 @@ class PyCodeObject33_35 : public PyObject {
}
};

// 2.5 - 3.1
// 3.6
class PyCodeObject36 : public PyObject {
public:
int co_argcount; /* #arguments, except *args */
int co_kwonlyargcount; /* #keyword only arguments */
int co_nlocals; /* #local variables */
int co_stacksize; /* #entries needed for evaluation stack */
int co_flags; /* CO_..., see below */
int co_firstlineno; /* first source line number */
PyObject *co_code; /* instruction opcodes */
PyObject *co_consts; /* list (constants used) */
PyObject *co_names; /* list of strings (names used) */
PyObject *co_varnames; /* tuple of strings (local variable names) */
PyObject *co_freevars; /* tuple of strings (free variable names) */
PyObject *co_cellvars; /* tuple of strings (cell variable names) */
/* The rest doesn't count for hash or comparisons */
unsigned char *co_cell2arg; /* Maps cell vars which are arguments. */
PyObject *co_filename; /* unicode (where it was loaded from) */
PyObject *co_name; /* unicode (name, for reference) */
PyObject *co_lnotab; /* string (encoding addr<->lineno mapping) */
void *co_zombieframe; /* for optimization only (see frameobject.c) */

static bool IsFor(int majorVersion, int minorVersion) {
return majorVersion == 3 && minorVersion >= 6;
}

static bool IsFor(PythonVersion version) {
return version >= PythonVersion_36;
}
};

// 2.5 - 3.6
class PyFunctionObject : public PyObject {
public:
PyObject *func_code; /* A code object */
Expand Down Expand Up @@ -175,7 +207,7 @@ typedef struct {
long hash; /* Hash value; -1 if not set */
} PyUnicodeObject;

// 2.4 - 3.5 compatible
// 2.4 - 3.6 compatible
class PyFrameObject : public PyVarObject {
public:
PyFrameObject *f_back; /* previous frame, or NULL */
Expand Down Expand Up @@ -216,7 +248,7 @@ class PyFrameObject25_33 : public PyFrameObject {
}
};

class PyFrameObject34_35 : public PyFrameObject {
class PyFrameObject34_36 : public PyFrameObject {
public:
/* Borrowed reference to a generator, or NULL */
PyObject *f_gen;
Expand All @@ -231,14 +263,14 @@ class PyFrameObject34_35 : public PyFrameObject {
PyObject *f_localsplus[1]; /* locals+stack, dynamically sized */

static bool IsFor(int majorVersion, int minorVersion) {
return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 5;
return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 6;
}
};


typedef void (*destructor)(PyObject *);

// 2.4 - 3.5
// 2.4 - 3.6
class PyMethodDef {
public:
char *ml_name; /* The name of the built-in function/method */
Expand All @@ -261,7 +293,7 @@ class PyTypeObject : public PyVarObject {
void* tp_setattr;
union {
void* tp_compare; /* 2.4 - 3.4 */
void* tp_as_async; /* 3.5 */
void* tp_as_async; /* 3.5 - 3.6 */
};
void* tp_repr;

Expand Down Expand Up @@ -331,7 +363,7 @@ class PyTypeObject : public PyVarObject {
unsigned int tp_version_tag;
};

// 2.4 - 3.5
// 2.4 - 3.6
class PyTupleObject : public PyVarObject {
public:
PyObject *ob_item[1];
Expand All @@ -342,7 +374,7 @@ class PyTupleObject : public PyVarObject {
*/
};

// 2.4 - 3.5
// 2.4 - 3.6
class PyCFunctionObject : public PyObject {
public:
PyMethodDef *m_ml; /* Description of the C function to call */
Expand Down Expand Up @@ -473,7 +505,7 @@ class PyThreadState_30_33 : public PyThreadState {
}
};

class PyThreadState_34_35 : public PyThreadState {
class PyThreadState_34_36 : public PyThreadState {
public:
PyThreadState *prev;
PyThreadState *next;
Expand Down Expand Up @@ -513,11 +545,11 @@ class PyThreadState_34_35 : public PyThreadState {

/* XXX signal handlers should also be here */
static bool IsFor(int majorVersion, int minorVersion) {
return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 5;
return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 6;
}

static bool IsFor(PythonVersion version) {
return version >= PythonVersion_34 && version <= PythonVersion_35;
return version >= PythonVersion_34 && version <= PythonVersion_36;
}
};

Expand Down Expand Up @@ -570,6 +602,7 @@ static PythonVersion GetPythonVersion(HMODULE hMod) {
case '3': return PythonVersion_33;
case '4': return PythonVersion_34;
case '5': return PythonVersion_35;
case '6': return PythonVersion_36;
}
}
}
Expand Down
17 changes: 16 additions & 1 deletion Python/Tests/Core/DebugReplEvaluatorTests.cs
View file
Expand Up @@ -116,7 +116,7 @@ public class DebugReplEvaluatorTests {
Assert.AreEqual(@"Traceback (most recent call last):
File ""<debug input>"", line 1, in <module>
NameError: name 'does_not_exist' is not defined
", _window.Error);
".Replace("\r\n", "\n"), _window.Error.Replace("\r\n", "\n"));
}

[TestMethod, Priority(3)]
Expand Down Expand Up @@ -434,6 +434,21 @@ public class DebugReplEvaluatorTests35 : DebugReplEvaluatorTests {
}
}

[TestClass]
public class DebugReplEvaluatorTests36 : DebugReplEvaluatorTests {
[ClassInitialize]
public static new void DoDeployment(TestContext context) {
AssertListener.Initialize();
PythonTestData.Deploy();
}

internal override PythonVersion Version {
get {
return PythonPaths.Python36 ?? PythonPaths.Python36_x64;
}
}
}

[TestClass]
public class DebugReplEvaluatorTests27 : DebugReplEvaluatorTests {
[ClassInitialize]
Expand Down
31 changes: 29 additions & 2 deletions Python/Tests/DebuggerTests/AttachTests.cs
View file
Expand Up @@ -807,7 +807,10 @@ void main()
string script = TestData.GetPath(@"TestData\DebuggerProject\AttachPtvsd.py");
var psi = new ProcessStartInfo(Version.InterpreterPath, PtvsdInterpreterArguments + " \"" + script + "\"") {
WorkingDirectory = TestData.GetPath(),
UseShellExecute = false
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true
};

var p = Process.Start(psi);
Expand All @@ -822,7 +825,7 @@ void main()
break;
} catch (SocketException) {
// Failed to connect - the process might have not started yet, so keep trying a few more times.
if (i >= 5) {
if (i >= 5 || p.HasExited) {
throw;
}
}
Expand Down Expand Up @@ -862,6 +865,8 @@ void main()
DetachProcess(proc);
}
} finally {
Console.WriteLine(p.StandardOutput.ReadToEnd());
Console.WriteLine(p.StandardError.ReadToEnd());
DisposeProcess(p);
}
}
Expand Down Expand Up @@ -1046,6 +1051,28 @@ public class AttachTests35_x64 : AttachTests35 {
}
}

[TestClass]
public class AttachTests36 : AttachTests {
internal override PythonVersion Version {
get {
return PythonPaths.Python36;
}
}

public override void AttachNewThread_PyThreadState_New() {
// PyEval_AcquireLock deprecated in 3.2
}
}

[TestClass]
public class AttachTests36_x64 : AttachTests35 {
internal override PythonVersion Version {
get {
return PythonPaths.Python36_x64;
}
}
}

[TestClass]
public class AttachTests25 : AttachTests {
internal override PythonVersion Version {
Expand Down

0 comments on commit 120e450

Please sign in to comment.