diff --git a/tests/extension.prof b/tests/extension.prof new file mode 100644 index 00000000..4398cb13 Binary files /dev/null and b/tests/extension.prof differ diff --git a/tests/run_slow.py b/tests/run_slow.py new file mode 100644 index 00000000..0c5602d9 --- /dev/null +++ b/tests/run_slow.py @@ -0,0 +1,17 @@ + +# Test profiling of mixed Python / native code stacks + +from ctypes import * + +h = cdll.LoadLibrary("./slow_ext.so") + +def more_slow(): + h.slow2(10000) + +def slow_wrap(): + h.slow2(20000) + more_slow() + +if __name__ == '__main__': + slow_wrap() + diff --git a/tests/slow_ext.cpp b/tests/slow_ext.cpp new file mode 100644 index 00000000..809862c1 --- /dev/null +++ b/tests/slow_ext.cpp @@ -0,0 +1,33 @@ + +// Extension module with slow code +// Test profiling of native code +// +// Compile with: +// g++ -fPIC -g -O0 --shared -o slow_ext.so slow_ext.cpp + +#include + +extern "C" { +int slow1(int n); +int slow2(int n); +} + +int slow1(int n) +{ + int sum = 0; + for (int i = 0; i < n; i++) { + for (int j = 0; j < 100000; j++) { + sum += i*j; + } + } + + return sum; +} + +int slow2(int n) +{ + return slow1(n); +} + + + diff --git a/vmprof/addrspace.py b/vmprof/addrspace.py index 9a315657..612cb884 100644 --- a/vmprof/addrspace.py +++ b/vmprof/addrspace.py @@ -38,6 +38,9 @@ class JittedVirtual(Hashable): class VirtualFrame(Hashable): pass +class NativeFrame(Hashable): + pass + class BaseMetaFrame(Hashable): def add_to_meta(self, meta): meta[self.name] = meta.get(self.name, 0) + 1 @@ -57,6 +60,16 @@ class WarmupFrame(BaseMetaFrame): class BlackholeWarmupFrame(WarmupFrame): name = 'blackhole' +def is_python_runtime(lib): + lib_path = [] + if lib and lib.name: + lib_path = lib.name.split('/') + + if lib_path and 'libpython' in lib_path[-1]: + return True + + return False + class AddressSpace(object): def __init__(self, libs): all = [(lib.start, lib) for lib in libs] @@ -114,7 +127,7 @@ def filter(self, profiles): filtered_profiles.append((current, prof[1])) return filtered_profiles - def _next_profile(self, lst, jit_frames, addr_set, interp_name, + def _filter_stack(self, lst, jit_frames, addr_set, interp_name, only_virtual): current = [] jitting = False @@ -136,17 +149,28 @@ def _next_profile(self, lst, jit_frames, addr_set, interp_name, current.append(JitAddr(jit_addr)) jitting = False continue + if addr in self.meta_data: current.append(self.meta_data[addr](addr)) - elif is_virtual or not only_virtual: + continue + + cls = None + if not only_virtual: + if not is_python_runtime(lib): + cls = NativeFrame + + if is_virtual: if jitting: cls = JittedVirtual else: cls = VirtualFrame + + if cls: if previous_virtual != addr: current.append(cls(addr)) previous_virtual = current[-1] addr_set.add(addr) + return current def filter_addr(self, profiles, only_virtual=True, @@ -159,7 +183,7 @@ def filter_addr(self, profiles, only_virtual=True, if len(prof[0]) < 5: skipped += 1 continue # broken profile - current = self._next_profile(prof[0], jit_frames, addr_set, + current = self._filter_stack(prof[0], jit_frames, addr_set, interp_name, only_virtual) if current: current.reverse() diff --git a/vmprof/profiler.py b/vmprof/profiler.py index b66d1688..c9f4cfcf 100644 --- a/vmprof/profiler.py +++ b/vmprof/profiler.py @@ -30,6 +30,9 @@ def read_profile(prof_filename, lib_cache={}, extra_libs=None, period, profiles, virtual_symbols, libs, interp_name = read_prof(prof) + if interp_name == 'pypy': + virtual_only = True + if not virtual_only or include_extra_info: exe_name = libs[0].name for lib in libs: diff --git a/vmprof/show.py b/vmprof/show.py index 5482d2a6..18fc5402 100644 --- a/vmprof/show.py +++ b/vmprof/show.py @@ -31,15 +31,17 @@ def __init__(self, prune_percent=None, prune_level=None, indent=None): self._prune_level = prune_level or 1000 self._indent = indent or 2 - def show(self, profile): + def show(self, profile, virtual_only): """ Read and display a vmprof profile file. :param profile: The filename of the vmprof profile file to display. :type profile: str + :param virtual_only: Display only Python frames + :type profile: bool """ try: - stats = vmprof.read_profile(profile, virtual_only=True, include_extra_info=True) + stats = vmprof.read_profile(profile, virtual_only=virtual_only, include_extra_info=True) except Exception as e: print("Fatal: could not read vmprof profile file '{}': {}".format(profile, e)) return @@ -91,6 +93,13 @@ def print_node(parent, node, level): p3 = click.style(funname, fg='white', bold=False) p5 = click.style("{:>2}".format(level), fg='red', bold=False) + elif parts == 0: + # Native code frame + funname = node.name + p2 = click.style(funname, fg='green', bold=True) + p2b = click.style(('.' * level * self._indent), fg='green', bold=False) + p3 = "" + else: raise Exception("fail!") @@ -108,9 +117,10 @@ def print_node(parent, node, level): @click.option('--prune_percent', type=float, default=0, help='The indention per level within the call graph.') @click.option('--prune_level', type=int, default=None, help='Prune output of a profile stats node when CPU.') @click.option('--indent', type=int, default=2, help='The indention per level within the call graph.') -def main(profile, prune_percent, prune_level, indent): +@click.option('--python_only', is_flag=True, help='Show only Python frames.') +def main(profile, prune_percent, prune_level, indent, python_only): pp = PrettyPrinter(prune_percent=prune_percent, prune_level=prune_level, indent=indent) - pp.show(profile) + pp.show(profile, virtual_only=python_only) if __name__ == '__main__': diff --git a/vmprof/stats.py b/vmprof/stats.py index 7d57c11d..69cae3fe 100644 --- a/vmprof/stats.py +++ b/vmprof/stats.py @@ -1,6 +1,6 @@ import six from vmprof.addrspace import JittedVirtual, JitAddr, VirtualFrame,\ - BaseMetaFrame + BaseMetaFrame, NativeFrame class EmptyProfileFile(Exception): pass @@ -106,7 +106,7 @@ def get_tree(self): for i in range(1, len(profile[0])): addr = profile[0][i] name = self._get_name(addr) - if not isinstance(addr, (VirtualFrame, JittedVirtual)): + if not isinstance(addr, (VirtualFrame, JittedVirtual, NativeFrame)): continue last_virtual = addr last_virtual_pos = i