diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c6e5085d..23e4d1c5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,7 +11,7 @@ Changes * FIX: Fixed explicit profiling of class methods; added handling for profiling static, bound, and partial methods, ``functools.partial`` objects, (cached) properties, and async generator functions * FIX: Fixed namespace bug when running ``kernprof -m`` on certain modules (e.g. ``calendar`` on Python 3.12+). * FIX: Fixed ``@contextlib.contextmanager`` bug where the cleanup code (e.g. restoration of ``sys`` attributes) is not run if exceptions occurred inside the context -* ENH: Added CLI arguments ``-c`` to ``kernprof`` for (auto-)profiling inline-script execution instead of that of script files; passing ``'-'`` as the script-file name now also reads from and profiles ``stdin`` +* ENH: Added CLI arguments ``-c`` to ``kernprof`` for (auto-)profiling module/package/inline-script execution instead of that of script files; passing ``'-'`` as the script-file name now also reads from and profiles ``stdin`` * ENH: In Python >=3.11, profiled objects are reported using their qualified name. * ENH: Highlight final summary using rich if enabled * ENH: Made it possible to use multiple profiler instances simultaneously @@ -24,6 +24,7 @@ Changes * On-import profiling is now more aggressive so that it doesn't miss entities like class methods and properties * ``LineProfiler`` can now be used as a class decorator +* ENH: Added capability to parse TOML config files for defaults for ``kernprof`` and ``python -m line_profiler`` CLI options, ``GlobalProfiler`` configurations, and profiler output (e.g. ``LineProfiler.print_stats()``) formatting #335 4.2.0 ~~~~~ diff --git a/MANIFEST.in b/MANIFEST.in index c9d9793e..93aba83b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ +graft line_profiler/rc include *.md include *.rst include *.py diff --git a/kernprof.py b/kernprof.py index a94f8dc2..0699945c 100755 --- a/kernprof.py +++ b/kernprof.py @@ -73,11 +73,20 @@ def main(): .. code:: - usage: kernprof [-h] [-V] [-l] [-b] [-o OUTFILE] [-s SETUP] [-v] [-q] [-r] [-u UNIT] [-z] [-i [OUTPUT_INTERVAL]] [-p {path/to/script | object.dotted.path}[,...]] - [--no-preimports] [--prof-imports] + usage: kernprof [-h] [-V] [--config CONFIG] [--no-config] + [--line-by-line [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0]] + [--builtin [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0]] + [-s SETUP] [-p {path/to/script | object.dotted.path}[,...]] + [--preimports [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0]] + [--prof-imports [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0]] + [-o OUTFILE] [-v] [-q] + [--rich [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0]] + [-u UNIT] + [--skip-zero [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0]] + [-i [OUTPUT_INTERVAL]] {path/to/script | -m path.to.module | -c "literal code"} ... - Run and profile a python script. + Run and profile a python script or module. positional arguments: {path/to/script | -m path.to.module | -c "literal code"} @@ -87,28 +96,73 @@ def main(): options: -h, --help show this help message and exit -V, --version show program's version number and exit - -l, --line-by-line Use the line-by-line profiler instead of cProfile. Implies --builtin. - -b, --builtin Put 'profile' in the builtins. Use 'profile.enable()'/'.disable()', '@profile' to decorate functions, or 'with profile:' to profile a section of code. + --config CONFIG Path to the TOML file, from the `tool.line_profiler.kernprof` + table of which to load defaults for the options. (Default: + 'pyproject.toml') + --no-config Disable the loading of configuration files other than the + default one + + profiling options: + --line-by-line [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0] + Use the line-by-line profiler instead of cProfile. Implies + `--builtin`. (Default: False; short form: -l) + --builtin [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0] + Put `profile` in the builtins. Use + `profile.enable()`/`.disable()` to toggle profiling, + `@profile` to decorate functions, or `with profile:` to + profile a section of code. (Default: False; short form: -b) + -s, --setup SETUP Path to the Python source file containing setup code to + execute before the code to profile. (Default: N/A) + -p, --prof-mod PROF_MOD + List of modules, functions and/or classes to profile specified + by their name or path. These profiling targets can be supplied + both as comma-separated items, or separately with multiple + copies of this flag. Packages are automatically recursed into + unless they are specified with `.__init__`. Adding the + current script/module profiles the entirety of it. Only works + with line profiling (`-l`/`--line-by-line`). (Default: N/A; + pass an empty string to clear the defaults (or any `-p` target + specified earlier) + ---preimports [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0] + Instead of eagerly importing all profiling targets specified + via `-p` and profiling them, only profile those that are + directly imported in the profiled code. Only works with + line profiling (`-l`/`--line-by-line`). (Default: False) + Eagerly import all profiling targets specified via `-p` and + profile them, instead of only profiling those that are + directly imported in the profiled code. Only works with line + profiling (`-l`/`--line-by-line`). (Default: True) + --prof-imports [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0] + If the script/module profiled is in `--prof-mod`, autoprofile + all its imports. Only works with line profiling (`-l`/`--line- + by-line`). (Default: False) + + output options: -o, --outfile OUTFILE - Save stats to (default: 'scriptname.lprof' with --line-by-line, 'scriptname.prof' without) - -s, --setup SETUP Code to execute before the code to profile + Save stats to OUTFILE. (Default: + '.lprof' in line-profiling mode + (`-l`/`--line-by-line`); '.prof' + otherwise) -v, --verbose, --view - Increase verbosity level. At level 1, view the profiling results in addition to saving them; at level 2, show other diagnostic info. - -q, --quiet Decrease verbosity level. At level -1, disable helpful messages (e.g. "Wrote profile results to <...>"); at level -2, silence the stdout; at level -3, - silence the stderr. - -r, --rich Use rich formatting if viewing output - -u, --unit UNIT Output unit (in seconds) in which the timing info is displayed (default: 1e-6) - -z, --skip-zero Hide functions which have not been called + Increase verbosity level (default: 0). At level 1, view the + profiling results in addition to saving them; at level 2, + show other diagnostic info. + -q, --quiet Decrease verbosity level (default: 0). At level -1, disable + helpful messages (e.g. "Wrote profile results to <...>"); at + level -2, silence the stdout; at level -3, silence the stderr. + --rich [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0] + Use rich formatting if viewing output. (Default: False; short + form: -r) + -u, --unit UNIT Output unit (in seconds) in which the timing info is + displayed. (Default: 1e-06 s) + --skip-zero [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0] + Hide functions which have not been called. (Default: False; + short form: -z) -i, --output-interval [OUTPUT_INTERVAL] - Enables outputting of cumulative profiling results to file every n seconds. Uses the threading module. Minimum value is 1 (second). Defaults to - disabled. - -p, --prof-mod {path/to/script | object.dotted.path}[,...] - List of modules, functions and/or classes to profile specified by their name or path. These profiling targets can be supplied both as comma-separated - items, or separately with multiple copies of this flag. Packages are automatically recursed into unless they are specified with `.__init__`. Adding - the current script/module profiles the entirety of it. Only works with line_profiler -l, --line-by-line. - --no-preimports Instead of eagerly importing all profiling targets specified via -p and profiling them, only profile those that are directly imported in the profiled - code. Only works with line_profiler -l, --line-by-line. - --prof-imports If specified, modules specified to `--prof-mod` will also autoprofile modules that they import. Only works with line_profiler -l, --line-by-line + Enables outputting of cumulative profiling results to OUTFILE + every OUTPUT_INTERVAL seconds. Uses the threading module. + Minimum value (and the value implied if the bare option is + given) is 1 s. (Default: 0 s (disabled)) NOTE: @@ -127,7 +181,7 @@ def main(): To restore the old behavior, pass the :option:`!--no-preimports` flag. -""" +""" # noqa: E501 import atexit import builtins import functools @@ -141,7 +195,7 @@ def main(): import tempfile import time import warnings -from argparse import ArgumentError, ArgumentParser +from argparse import ArgumentParser from io import StringIO from operator import methodcaller from runpy import run_module @@ -162,6 +216,11 @@ def main(): except ImportError: from profile import Profile # type: ignore[assignment,no-redef] +import line_profiler +from line_profiler.cli_utils import ( + add_argument, get_cli_config, + get_python_executable as _python_command, # Compatibility + positive_float, short_string_path) from line_profiler.profiler_mixin import ByCountProfilerMixin from line_profiler._logger import Logger from line_profiler import _diagnostics as diagnostics @@ -217,7 +276,7 @@ class RepeatedTimer: References: .. [SO474528] https://stackoverflow.com/questions/474528/execute-function-every-x-seconds/40965385#40965385 - """ + """ # noqa: E501 def __init__(self, interval, dump_func, outfile): self._timer = None self.interval = interval @@ -235,7 +294,8 @@ def _run(self): def start(self): if not self.is_running: self.next_call += self.interval - self._timer = threading.Timer(self.next_call - time.time(), self._run) + self._timer = threading.Timer(self.next_call - time.time(), + self._run) self._timer.start() self.is_running = True @@ -244,7 +304,7 @@ def stop(self): self.is_running = False -def find_module_script(module_name): +def find_module_script(module_name, *, exit_on_error=True): """Find the path to the executable script for a module or package.""" from line_profiler.autoprofile.util_static import modname_to_modpath @@ -253,11 +313,15 @@ def find_module_script(module_name): if fname: return fname - sys.stderr.write('Could not find module %s\n' % module_name) - raise SystemExit(1) + msg = f'Could not find module `{module_name}`' + if exit_on_error: + print(msg, file=sys.stderr) + raise SystemExit(1) + else: + raise ModuleNotFoundError(msg) -def find_script(script_name, exit_on_error=True): +def find_script(script_name, *, exit_on_error=True): """ Find the script. If the input is not a file, then :envvar:`PATH` will be searched. @@ -280,16 +344,6 @@ def find_script(script_name, exit_on_error=True): raise FileNotFoundError(msg) -def _python_command(): - """ - Return a command that corresponds to :py:data:`sys.executable`. - """ - for abbr in 'python', 'python3': - if os.path.samefile(shutil.which(abbr), sys.executable): - return abbr - return sys.executable - - def _normalize_profiling_targets(targets): """ Normalize the parsed :option:`!--prof-mod` by: @@ -298,6 +352,8 @@ def _normalize_profiling_targets(targets): subsequently to absolute paths. * Splitting non-file paths at commas into (presumably) file paths and/or dotted paths. + * Allowing paths specified earlier to be invalidated by an empty + string. * Removing duplicates. """ def find(path): @@ -309,6 +365,9 @@ def find(path): results = {} for chunk in targets: + if not chunk: + results.clear() + continue filename = find(chunk) if filename is not None: results.setdefault(filename) @@ -490,80 +549,120 @@ def pre_parse_single_arg_directive(args, flag, sep='--'): return args, thing, post_args -def positive_float(value): - val = float(value) - if val <= 0: - raise ArgumentError - return val - - def no_op(*_, **__) -> None: pass def _add_core_parser_arguments(parser): """ - Add the core kernprof args to a ArgumentParser + Add the core kernprof args to a + :py:class:`~argparse.ArgumentParser`. """ - parser.add_argument('-V', '--version', action='version', version=__version__) - parser.add_argument('-l', '--line-by-line', action='store_true', - help='Use the line-by-line profiler instead of cProfile. Implies --builtin.') - parser.add_argument('-b', '--builtin', action='store_true', - help="Put 'profile' in the builtins. Use 'profile.enable()'/'.disable()', " - "'@profile' to decorate functions, or 'with profile:' to profile a " - 'section of code.') - parser.add_argument('-o', '--outfile', - help="Save stats to (default: 'scriptname.lprof' with " - "--line-by-line, 'scriptname.prof' without)") - parser.add_argument('-s', '--setup', - help='Code to execute before the code to profile') - parser.add_argument('-v', '--verbose', '--view', - action='count', default=0, - help='Increase verbosity level. ' - 'At level 1, view the profiling results ' - 'in addition to saving them; ' - 'at level 2, show other diagnostic info.') - parser.add_argument('-q', '--quiet', - action='count', default=0, - help='Decrease verbosity level. ' - 'At level -1, disable helpful messages ' - '(e.g. "Wrote profile results to <...>"); ' - 'at level -2, silence the stdout; ' - 'at level -3, silence the stderr.') - parser.add_argument('-r', '--rich', action='store_true', - help='Use rich formatting if viewing output') - parser.add_argument('-u', '--unit', default='1e-6', type=positive_float, - help='Output unit (in seconds) in which the timing info is ' - 'displayed (default: 1e-6)') - parser.add_argument('-z', '--skip-zero', action='store_true', - help="Hide functions which have not been called") - parser.add_argument('-i', '--output-interval', type=int, default=0, const=0, nargs='?', - help="Enables outputting of cumulative profiling results to file every n seconds. Uses the threading module. " - "Minimum value is 1 (second). Defaults to disabled.") - parser.add_argument('-p', '--prof-mod', - action='append', - metavar=("{path/to/script | object.dotted.path}" - "[,...]"), - help="List of modules, functions and/or classes " - "to profile specified by their name or path. " - "These profiling targets can be supplied both as " - "comma-separated items, or separately with " - "multiple copies of this flag. " - "Packages are automatically recursed into unless " - "they are specified with `.__init__`. " - "Adding the current script/module profiles the " - "entirety of it. " - "Only works with line_profiler -l, --line-by-line.") - parser.add_argument('--no-preimports', - action='store_true', - help="Instead of eagerly importing all profiling " - "targets specified via -p and profiling them, " - "only profile those that are directly imported in " - "the profiled code. " - "Only works with line_profiler -l, --line-by-line.") - parser.add_argument('--prof-imports', action='store_true', - help="If specified, modules specified to `--prof-mod` will also autoprofile modules that they import. " - "Only works with line_profiler -l, --line-by-line") + defaults, default_source = get_cli_config('kernprof') + add_argument(parser, '-V', '--version', + action='version', version=__version__) + add_argument(parser, '--config', + help='Path to the TOML file, from the ' + '`tool.line_profiler.kernprof` table of which to load ' + 'defaults for the options. ' + f'(Default: {short_string_path(default_source)!r})') + add_argument(parser, '--no-config', + action='store_const', dest='config', const=False, + help='Disable the loading of configuration files other ' + 'than the default one') + prof_opts = parser.add_argument_group('profiling options') + add_argument(prof_opts, '-l', '--line-by-line', action='store_true', + help='Use the line-by-line profiler instead of cProfile. ' + 'Implies `--builtin`. ' + f'(Default: {defaults["line_by_line"]})') + add_argument(prof_opts, '-b', '--builtin', action='store_true', + help="Put `profile` in the builtins. " + "Use `profile.enable()`/`.disable()` to " + "toggle profiling, " + "`@profile` to decorate functions, " + "or `with profile:` to profile a section of code. " + f"(Default: {defaults['builtin']})") + if defaults['setup']: + def_setupfile = repr(defaults['setup']) + else: + def_setupfile = 'N/A' + add_argument(prof_opts, '-s', '--setup', + help='Path to the Python source file containing setup ' + 'code to execute before the code to profile. ' + f'(Default: {def_setupfile})') + if defaults['prof_mod']: + def_prof_mod = repr(defaults['prof_mod']) + else: + def_prof_mod = 'N/A' + add_argument(prof_opts, '-p', '--prof-mod', action='append', + help="List of modules, functions and/or classes to profile " + "specified by their name or path. These profiling targets " + "can be supplied both as comma-separated items, or " + "separately with multiple copies of this flag. Packages " + "are automatically recursed into unless they are specified " + "with `.__init__`. Adding the current script/module " + "profiles the entirety of it. Only works with line " + "profiling (`-l`/`--line-by-line`). " + f"(Default: {def_prof_mod}; " + "pass an empty string to clear the defaults (or any `-p` " + "target specified earlier))") + add_argument(prof_opts, '--preimports', action='store_true', + help="Eagerly import all profiling targets specified via " + "`-p` and profile them, instead of only profiling those " + "that are directly imported in the profiled code. " + "Only works with line profiling (`-l`/`--line-by-line`). " + f"(Default: {defaults['preimports']})") + add_argument(prof_opts, '--prof-imports', action='store_true', + help="If the script/module profiled is in `--prof-mod`, " + "autoprofile all its imports. " + "Only works with line profiling (`-l`/`--line-by-line`). " + f"(Default: {defaults['prof_imports']})") + out_opts = parser.add_argument_group('output options') + if defaults['outfile']: + def_outfile = repr(defaults['outfile']) + else: + def_outfile = ( + "'.lprof' in line-profiling mode " + "(`-l`/`--line-by-line`); " + "'.prof' otherwise") + add_argument(out_opts, '-o', '--outfile', + help=f'Save stats to OUTFILE. (Default: {def_outfile})') + add_argument(out_opts, '-v', '--verbose', '--view', + action='count', default=defaults['verbose'], + help="Increase verbosity level " + f"(default: {defaults['verbose']}). " + "At level 1, view the profiling results in addition to " + "saving them; " + "at level 2, show other diagnostic info.") + add_argument(out_opts, '-q', '--quiet', + action='count', default=0, + help='Decrease verbosity level ' + f"(default: {defaults['verbose']}). " + 'At level -1, disable ' + 'helpful messages (e.g. "Wrote profile results to <...>"); ' + 'at level -2, silence the stdout; ' + 'at level -3, silence the stderr.') + add_argument(out_opts, '-r', '--rich', action='store_true', + help='Use rich formatting if viewing output. ' + f'(Default: {defaults["rich"]})') + add_argument(out_opts, '-u', '--unit', type=positive_float, + help='Output unit (in seconds) in which ' + 'the timing info is displayed. ' + f'(Default: {defaults["unit"]} s)') + add_argument(out_opts, '-z', '--skip-zero', action='store_true', + help="Hide functions which have not been called. " + f"(Default: {defaults['skip_zero']})") + if defaults['output_interval']: + def_out_int = f'{defaults["output_interval"]} s' + else: + def_out_int = '0 s (disabled)' + add_argument(out_opts, '-i', '--output-interval', + type=int, const=1, nargs='?', + help="Enables outputting of cumulative profiling results " + "to OUTFILE every OUTPUT_INTERVAL seconds. " + "Uses the threading module. " + "Minimum value (and the value implied if the bare option " + f"is given) is 1 s. (Default: {def_out_int})") def _build_parsers(args=None): @@ -606,12 +705,13 @@ def _build_parsers(args=None): _add_core_parser_arguments(parser) if parser is help_parser or module is literal_code is None: - parser.add_argument('script', - metavar='{path/to/script' - ' | -m path.to.module | -c "literal code"}', - help='The python script file, module, or ' - 'literal code to run') - parser.add_argument('args', nargs='...', help='Optional script arguments') + add_argument(parser, 'script', + metavar='{path/to/script' + ' | -m path.to.module | -c "literal code"}', + help='The python script file, module, or ' + 'literal code to run') + add_argument(parser, 'args', + nargs='...', help='Optional script arguments') special_info = { 'module': module, 'literal_code': literal_code, @@ -621,7 +721,8 @@ def _build_parsers(args=None): return real_parser, help_parser, special_info -def _parse_arguments(real_parser, help_parser, special_info, args): +def _parse_arguments( + real_parser, help_parser, special_info, args, exit_on_error): module = special_info['module'] literal_code = special_info['literal_code'] @@ -629,17 +730,41 @@ def _parse_arguments(real_parser, help_parser, special_info, args): # Hand off to the dummy parser if necessary to generate the help # text - options = SimpleNamespace(**vars(real_parser.parse_args(args))) + try: + options = SimpleNamespace(**vars(real_parser.parse_args(args))) + except SystemExit as e: + # If `exit_on_error` is true, let `SystemExit` bubble up and + # kill the interpretor; + # else, catch and handle it more gracefully + # (Note: can't use `ArgumentParser(exit_on_error=False)` in + # Python 3.8) + if exit_on_error: + raise + elif e.code: + raise RuntimeError from None + else: + return # TODO: make flags later where appropriate options.dryrun = diagnostics.NO_EXEC options.static = diagnostics.STATIC_ANALYSIS if help_parser and getattr(options, 'help', False): help_parser.print_help() - exit() + if exit_on_error: + raise SystemExit(0) + else: + return + + # Parse the provided config file (if any), and resolve the values + # of the un-specified options try: del options.help except AttributeError: pass + defaults, options.config = get_cli_config('kernprof', options.config) + for key, default in defaults.items(): + if getattr(options, key, None) is None: + setattr(options, key, default) + # Add in the pre-partitioned arguments cut off by `-m ` or # `-c