From 77e6bab23cf7f79888aadc080724979cf29fbea7 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 19 Apr 2025 07:49:33 +0200 Subject: [PATCH 01/37] Basic TOML facilities line_profiler/line_profiler_rc.toml Default configuration file line_profiler/toml_config.py[i] New module for reading TOML config files --- line_profiler/line_profiler_rc.toml | 143 ++++++++++++++++++++++++ line_profiler/toml_config.py | 166 ++++++++++++++++++++++++++++ line_profiler/toml_config.pyi | 34 ++++++ 3 files changed, 343 insertions(+) create mode 100644 line_profiler/line_profiler_rc.toml create mode 100644 line_profiler/toml_config.py create mode 100644 line_profiler/toml_config.pyi diff --git a/line_profiler/line_profiler_rc.toml b/line_profiler/line_profiler_rc.toml new file mode 100644 index 00000000..5b8b2804 --- /dev/null +++ b/line_profiler/line_profiler_rc.toml @@ -0,0 +1,143 @@ +######################################################################## +# +# Sample configuration file +# ========================= +# +# This file provides defaults for the behaviors of the CLI application +# `kernprof`, the executable module `line_profiler`, and the class +# `line_profiler.GlobalProfiler`. +# Users can customize said behaviors by writing a suitable configuation +# file in the TOML syntax (https://toml.io/en/latest). +# +# Namespacing +# ----------- +# +# All options read by `line_profiler` live under the +# `tool.line_profiler` namespace. The same applies no matter if reading +# from a `line_profiler_rc.toml`, `pyproject.toml`, or other +# user-specified file. +# +# Naming scheme +# ------------- +# +# - `tool.line_profiler.kernprof` items: +# Same as the corresponding CLI options, minus the leading '--'. +# - `tool.line_profiler.setup` (resp. `.write`, `.show`) items: +# Same as the corresponding `GlobalProfiler.setup_config` (resp. +# `.write_config`, `.show_config`) key-value pairs. +# +# Resolution order +# ---------------- +# +# - Lookup for either of these two files starts from the current +# directory: `line_profiler_rc.toml` or `pyproject.toml`. In case when +# both are present, the former takes priority. +# - If the current directory +# - Doesn't contain such a file; or if +# - None of them is a (1) valid and (2) readable TOML config file, +# we look in the parent directory for a config file, and so on. +# - If we have reached the file-system root without finding a readable +# config file, this default file is used. +# +# ### Notes +# +# - For `kernprof` and `python -m line_profiler`, it is also possible to +# explicitly supply the config file and skip the lookup. +# - The first looked-up file that is readable TOML (possibly empty) is +# used; +# it doesn't matter if it contains the `tool.line_profiler` namespace. +# - If any option is missing from the specified or looked-up file, its +# default value (as defined here) is used. +# +######################################################################## + +# `kernprof` options + +[tool.line_profiler.kernprof] + +# - Boolean flags +# - `line-by-line` (bool): +# `-l`/`--line-by-line` (true) or `--no-line-by-line` (false) +line-by-line = false +# - `builtin` (bool): +# `-b`/`--builtin` (true) or `--no-builtin` (false) +builtin = false +# - `view` (bool): +# `-v`/`--view` (true) or `--no-view` (false) +view = false +# - `rich` (bool): +# `-r`/`--rich` (true) or `--no-rich` (false) +rich = false +# - `skip-zero` (bool): +# `-z`/`--skip-zero` (true) or `--no-skip-zero` (false) +skip-zero = false +# - `prof-imports` (bool): +# `--prof-imports` (true) or `--no-prof-imports` (false) +prof-imports = false + +# - Misc flags +# - `outfile` (str): +# `--outfile=...`; filename to which to `LineProfiler.dump_stats()` +# the profiling results (use an empty string to fall back to the +# default, which depends on the script/module profiled) +outfile = "" +# - `setup` (str): +# `--setup=...`: filename from which to read setup code (if any) +# before running the main script/module +setup = "" +# - `unit` (float): +# `--unit=...`: timer unit for the displayed profiling results when +# `view` is true +unit = 1e-6 +# - `output-interval` (int): +# `--output-interval=...`: interval in which partial profiling +# results are written to `outfile` (set to 0 to only write when the +# code finishes running/errors out) +output-interval = 0 +# - `prof-mod` (list[str]): +# `--prof-mod`: filenames and dotted import paths (modules, +# functions, or classes) to be profiled if imported in the profiled +# script/module +prof-mod = [] + +# `line_profiler.GlobalProfiler` options + +[tool.line_profiler.setup] + +# - `GlobalProfiler.setup_config` key-value pairs +# - `environ_flags` (list[str]): +# If any of these environment variables is set to a "non-falsy" +# value, the `GlobalProfiler` is `.enable()`-ed +environ_flags = ["LINE_PROFILE"] +# - `cli_flags` (list[str]): +# If any of these strings is present verbatim as a positional +# argument, the `GlobalProfiler` is `.enable()`-ed +cli_flags = ["--line-profile", "--line_profile"] + +[tool.line_profiler.write] + +# - `GlobalProfiler.write_config` key-value pairs +# - `lprof` (bool): +# Whether to `LineProfiler.dump_stats()` to an `.lprof` file +lprof = true +# - `text` (bool): +# Whether to `LineProfiler.print_stats()` to a `.txt` file +text = true +# - `timestamped_text` (bool): +# Whether to `LineProfiler.print_stats()` to a `.txt` file with a +# timestamp in the filename +timestamped_text = true +# - `stdout` (bool): +# Whether to `LineProfiler.print_stats()` to the stdout +stdout = true + +[tool.line_profiler.show] + +# - `GlobalProfiler.show_config` key-value pairs +# (booleans; refer to the synonymous arguments of +# `LineProfiler.print_stats()`) +sort = true +stripzeros = true +rich = true +details = false +summarize = true diff --git a/line_profiler/toml_config.py b/line_profiler/toml_config.py new file mode 100644 index 00000000..6c5a8499 --- /dev/null +++ b/line_profiler/toml_config.py @@ -0,0 +1,166 @@ +""" +Read and resolve user-supplied TOML files and combine them with the +default to generate configurations. +""" +import copy +import importlib.resources +import itertools +import pathlib +try: + import tomllib +except ImportError: # Python < 3.11 + import tomli as tomllib + + +__all__ = ['get_config', 'get_default_config'] + +namespace = 'tool', 'line_profiler' +targets = 'line_profiler_rc.toml', 'pyproject.toml' +_defaults = None + + +def find_and_read_config_file(*, config=None, targets=targets): + """ + Arguments: + config (Union[str | PurePath | None]): + Optional path to a specific TOML file; + if provided, skip lookup and just try to read from that file + targets (Sequence[str | PurePath]): + Optional filenames among which TOML files are looked up + + Return: + If the provied/looked-up file is readable and is valid TOML: + content, path (tuple[dict, Path]): + - `content`: parsed content of the file as a dictionary + - `path`: absolute path to the file + Otherwise + None + """ + def iter_configs(dir_path): + for dpath in itertools.chain((dir_path,), dir_path.parents): + for target in targets: + cfg = dpath / target + try: + if cfg.is_file(): + yield cfg + except OSError: # E.g. permission errors + pass + + if config is None: + pwd = pathlib.Path.cwd().absolute() + configs = iter_configs(pwd) + else: + configs = pathlib.Path(config).absolute(), + for config in configs: + try: + with config.open(mode='rb') as fobj: + return tomllib.load(fobj), config + except (OSError, tomllib.TOMLDecodeError): + pass + return None + + +def get_subtable(table, keys, *, allow_absence=True): + """ + Arguments + table (Mapping): + (Nested) Mapping + keys (Sequence): + Sequence of keys for item access on `table` and its + descendant tables + allow_absence (bool): + If true, allow for the keys to be absent; + otherwise, raise a `KeyError` + """ + for key in keys: + if allow_absence: + table = table.get(key, {}) + else: + table = table[key] + return table + + +def get_config(config=None): + """ + Arguments: + config (Union[str | PurePath | None]): + Optional path to a specific TOML file; + if provided, skip lookup and just try to read from that file + + Return: + conf_dict, path (tuple[dict, Path]): + - `conf_dict`: the combination of the `tool.line_profiler` + tables of the provided/looked-up config file (if any) and + the default as a dictionary + - `path`: absolute path to the config file whence the + config options are loaded + + Notes: + - For the config TOML file, it is required that each of the + following keys either is absent or maps to a table: + - `tool` and `tool.line_profiler` + - `tool.line_profiler.kernprof`, `.setup`, `.write`, and + `.show` + If this is not the case: + - If `config` is provided, a `ValueError` is raised. + - Otherwise, the looked-up file is considered invalid and + ignored. + """ + def merge(template, supplied): + if not (isinstance(template, dict) and isinstance(supplied, dict)): + return supplied + result = {} + for key, default in template.items(): + if key in supplied: + result[key] = merge(default, supplied[key]) + else: + result[key] = default + return result + + default_conf, default_source = get_default_config() + try: + content, source = find_and_read_config_file(config=config) + except TypeError: # Got `None` + return default_conf, default_source + conf = {} + try: + for key in 'kernprof', 'setup', 'write', 'show': + conf[key] = subtable = get_subtable( + content, [*namespace, key]) + if not isinstance(subtable, dict): + raise TypeError + except (TypeError, AttributeError): + if config is None: + # No file explicitly provided and the looked-up file is + # invalid, just fall back to the default configs + return default_conf, default_source + else: + # The explicitly provided config file is invalid, raise an + # error + raise ValueError( + f'config = {config!r}: expected each of these keys to either ' + 'be nonexistent or map to a table: ' + '`tool`, `tool.line_profiler`, and ' + '`tool.line_profiler.kernprof`, `.setup`, `.write`, ' + 'and `.show`') + # Filter the content of `conf` down to just the key-value pairs + # pairs present in the default configs + return merge(default_conf, conf), source + + +def get_default_config(): + """ + Return: + conf_dict, path (tuple[dict, Path]) + - `conf_dict`: the default config file's + `tool.line_profiler` table as a dictionary + - `path`: absolute path to the default config file + """ + if _defaults is None: + with importlib.resources.path(__spec__.name.rpartition('.')[0], + 'line_profiler_rc.toml') as path: + conf_dict, source = find_and_read_config_file(config=path) + globals()['_defaults'] = (get_subtable(conf_dict, namespace, + allow_absence=False), + source) + return copy.deepcopy(_defaults[0]), _defaults[1] diff --git a/line_profiler/toml_config.pyi b/line_profiler/toml_config.pyi new file mode 100644 index 00000000..cdcda646 --- /dev/null +++ b/line_profiler/toml_config.pyi @@ -0,0 +1,34 @@ +import pathlib +try: + import tomllib +except ImportError: # Python < 3.11 + import tomli as tomllib +from typing import Any, Dict, Mapping, Sequence, Tuple, TypeVar, Union + + +targets = 'line_profiler_rc.toml', 'pyproject.toml' + +K = TypeVar('K') +V = TypeVar('V') +Config = Tuple[Dict[str, Dict[str, Any]], pathlib.Path] +NestedTable = Mapping[K, Union['NestedTable[K, V]', V]] + + +def find_and_read_config_file( + *, + config: Union[str, pathlib.PurePath, None] = None, + targets: Sequence[Union[str, pathlib.PurePath]] = targets) -> Config: + ... + + +def get_subtable(table: NestedTable[K, V], keys: Sequence[K], *, + allow_absence: bool = True) -> NestedTable[K, V]: + ... + + +def get_config(config: Union[str, pathlib.PurePath, None] = None) -> Config: + ... + + +def get_default_config() -> Config: + ... From 4afce8c572647af018a10aa054bb4994ccef2aea Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 19 Apr 2025 08:50:26 +0200 Subject: [PATCH 02/37] Packaging updates MANIFEST.in, setup.py Configured to include `line_profiler/line_profiler_rc.toml` in source and wheel distributions requirements/runtime.txt Added dependency `tomli` for Python < 3.11 (to stand in for `tomllib`) --- MANIFEST.in | 1 + requirements/runtime.txt | 1 + setup.py | 5 ++++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index c9d9793e..3a646e47 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,6 +4,7 @@ include *.py include *.txt include *.toml include run_tests.sh +include line_profiler *.toml recursive-include requirements *.txt recursive-include tests *.py recursive-include line_profiler *.txt diff --git a/requirements/runtime.txt b/requirements/runtime.txt index e69de29b..fe0df55e 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -0,0 +1 @@ +tomli; python_version < '3.11' diff --git a/setup.py b/setup.py index cec98395..85c4a92e 100755 --- a/setup.py +++ b/setup.py @@ -284,7 +284,10 @@ def run_cythonize(force=False): setupkw["py_modules"] = ['kernprof', 'line_profiler'] setupkw["python_requires"] = ">=3.8" setupkw['license_files'] = ['LICENSE.txt', 'LICENSE_Python.txt'] - setupkw["package_data"] = {"line_profiler": ["py.typed", "*.pyi"]} + setupkw["package_data"] = {"line_profiler": ["py.typed", "*.pyi", "*.toml"]} + # `include_package_data` is needed to put `line_profiler_rc.toml` in + # the wheel + setupkw["include_package_data"] = True setupkw['keywords'] = ['timing', 'timer', 'profiling', 'profiler', 'line_profiler'] setupkw["classifiers"] = [ 'Development Status :: 5 - Production/Stable', From db3e0b095448078d4bf1a7ab2eae97f5b9bc1eb4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 19 Apr 2025 09:01:19 +0200 Subject: [PATCH 03/37] WIP: `kernprof` refactoring kernprof.py - Made all `line_profiler` imports unconditional (the ship has sailed, there's already an unconditional import for `line_profiler.profiler_mixin.ByCountProfilerMixin`) - For each boolean option (e.g. `--view`): - Added a corresponding negative option (e.g. `--no-view`) - Changed the default value from `False` to `None`, so that we can distinguish between cases where the negative option is passed and no option is passed (and in that case read from the config (TODO)) main(), find_module_script(), find_script() Added argument `exit_on_error` to optionally prevent parsing errors from killing the interpretor --- kernprof.py | 221 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 147 insertions(+), 74 deletions(-) diff --git a/kernprof.py b/kernprof.py index 39bbfde5..baee0d1a 100755 --- a/kernprof.py +++ b/kernprof.py @@ -93,6 +93,7 @@ def main(): the full script. Multiple copies of this flag can be supplied and the.list is extended. Only works with line_profiler -l, --line-by-line --prof-imports If specified, modules specified to `--prof-mod` will also autoprofile modules that they import. Only works with line_profiler -l, --line-by-line """ +import argparse import builtins import functools import os @@ -102,7 +103,6 @@ def main(): import concurrent.futures # NOQA import tempfile import time -from argparse import ArgumentError, ArgumentParser from runpy import run_module # NOTE: This version needs to be manually maintained in @@ -116,6 +116,7 @@ def main(): except ImportError: from profile import Profile # type: ignore[assignment,no-redef] +import line_profiler from line_profiler.profiler_mixin import ByCountProfilerMixin @@ -193,7 +194,7 @@ def stop(self): self.is_running = False -def find_module_script(module_name): +def find_module_script(module_name, *, exit_on_error=True): """Find the path to the executable script for a module or package.""" from line_profiler.autoprofile.util_static import modname_to_modpath @@ -202,11 +203,15 @@ def find_module_script(module_name): if fname: return fname - sys.stderr.write('Could not find module %s\n' % module_name) - raise SystemExit(1) + msg = f'Could not find module `{module_name}`' + if exit_on_error: + print(msg, file=sys.stderr) + raise SystemExit(1) + else: + raise ModuleNotFoundError(msg) -def find_script(script_name): +def find_script(script_name, *, exit_on_error=True): """ Find the script. If the input is not a file, then $PATH will be searched. @@ -221,8 +226,12 @@ def find_script(script_name): if os.path.isfile(fn): return fn - sys.stderr.write('Could not find script %s\n' % script_name) - raise SystemExit(1) + msg = f'Could not find script {script_name!r}' + if exit_on_error: + print(msg, file=sys.stderr) + raise SystemExit(1) + else: + raise FileNotFoundError(msg) def _python_command(): @@ -341,18 +350,60 @@ def pre_parse_single_arg_directive(args, flag, sep='--'): @_restore_list(sys.argv) @_restore_list(sys.path) -def main(args=None): +def main(args=None, exit_on_error=True): """ Runs the command line interface """ + # Make the absent value a list so that the `append` and `extend` + # action still works def positive_float(value): val = float(value) if val <= 0: - raise ArgumentError + raise argparse.ArgumentError return val + def add_argument(parser_like, *args, + hide_complementary_flag=True, **kwargs): + """ + Override the 'store_true' and 'store_false' actions so that they + are turned into 'store_const' options which don't set the + default to the opposite boolean, thus allowing us to later + distinguish between cases where the flag has been passed or not. + + Also automatically generates complementary boolean options for + `action='store_true'` options. If `hide_complementary_flag` is + true, the auto-generated option (all the long flags prefixed + with 'no-', e.g. '--foo' is negated by '--no-foo') is hidden + from the help text. + """ + if kwargs.get('action') not in ('store_true', 'store_false'): + return parser_like.add_argument(*args, **kwargs) + kwargs['const'] = kwargs['action'] == 'store_true' + kwargs['action'] = 'store_const' + kwargs.setdefault('default', None) + if kwargs['action'] == 'store_false': + return parser_like.add_argument(*args, **kwargs) + # Automatically generate a complementary option for a boolean + # option; + # for convenience, turn it into a `store_const` action + # (in Python 3.9+ one can use `argparse.BooleanOptionalAction`, + # but we want to maintain compatibility with Python 3.8) + action = parser_like.add_argument(*args, **kwargs) + long_flags = [arg for arg in args if arg.startswith('--')] + assert long_flags + if hide_complementary_flag: + falsy_help_text = argparse.SUPPRESS + else: + falsy_help_text = 'Negate these flags: ' + ', '.join(args) + parser_like.add_argument(*('--no-' + flag[2:] for flag in long_flags), + **{**kwargs, + 'const': False, + 'dest': action.dest, + 'help': falsy_help_text}) + return action + create_parser = functools.partial( - ArgumentParser, + argparse.ArgumentParser, description='Run and profile a python script.') if args is None: @@ -387,53 +438,82 @@ def positive_float(value): help_parser = create_parser() parsers = [real_parser, help_parser] for parser in parsers: - parser.add_argument('-V', '--version', action='version', version=__version__) - parser.add_argument('-l', '--line-by-line', action='store_true', - help='Use the line-by-line profiler instead of cProfile. Implies --builtin.') - parser.add_argument('-b', '--builtin', action='store_true', - help="Put 'profile' in the builtins. Use 'profile.enable()'/'.disable()', " - "'@profile' to decorate functions, or 'with profile:' to profile a " - 'section of code.') - parser.add_argument('-o', '--outfile', - help="Save stats to (default: 'scriptname.lprof' with " - "--line-by-line, 'scriptname.prof' without)") - parser.add_argument('-s', '--setup', - help='Code to execute before the code to profile') - parser.add_argument('-v', '--view', action='store_true', - help='View the results of the profile in addition to saving it') - parser.add_argument('-r', '--rich', action='store_true', - help='Use rich formatting if viewing output') - parser.add_argument('-u', '--unit', default='1e-6', type=positive_float, - help='Output unit (in seconds) in which the timing info is ' - 'displayed (default: 1e-6)') - parser.add_argument('-z', '--skip-zero', action='store_true', - help="Hide functions which have not been called") - parser.add_argument('-i', '--output-interval', type=int, default=0, const=0, nargs='?', - help="Enables outputting of cumulative profiling results to file every n seconds. Uses the threading module. " - "Minimum value is 1 (second). Defaults to disabled.") - parser.add_argument('-p', '--prof-mod', action='append', type=str, - help="List of modules, functions and/or classes to profile specified by their name or path. " - "List is comma separated, adding the current script path profiles the full script. " - "Multiple copies of this flag can be supplied and the.list is extended. " - "Only works with line_profiler -l, --line-by-line") - parser.add_argument('--prof-imports', action='store_true', - help="If specified, modules specified to `--prof-mod` will also autoprofile modules that they import. " - "Only works with line_profiler -l, --line-by-line") + add_argument(parser, '-V', '--version', + action='version', version=__version__) + add_argument(parser, '-l', '--line-by-line', action='store_true', + help='Use the line-by-line profiler instead of cProfile. ' + 'Implies --builtin.') + add_argument(parser, '-b', '--builtin', action='store_true', + help="Put 'profile' in the builtins. " + "Use 'profile.enable()'/'.disable()', " + "'@profile' to decorate functions, " + "or 'with profile:' to profile a section of code.") + add_argument(parser, '-o', '--outfile', + help='Save stats to ' + "(default: 'scriptname.lprof' with --line-by-line, " + "'scriptname.prof' without)") + add_argument(parser, '-s', '--setup', + help='Code to execute before the code to profile') + add_argument(parser, '-v', '--view', action='store_true', + help='View the results of the profile ' + 'in addition to saving it') + add_argument(parser, '-r', '--rich', action='store_true', + help='Use rich formatting if viewing output') + add_argument(parser, '-u', '--unit', default='1e-6', type=positive_float, + help='Output unit (in seconds) in which ' + 'the timing info is displayed (default: %(default)s)') + add_argument(parser, '-z', '--skip-zero', action='store_true', + help="Hide functions which have not been called") + add_argument(parser, '-i', '--output-interval', + type=int, default=0, const=0, nargs='?', + help="Enables outputting of cumulative profiling results " + "to file every n seconds. Uses the threading module. " + "Minimum value is 1 (second). Defaults to disabled.") + add_argument(parser, '-p', '--prof-mod', action='append', type=str, + help="List of modules, functions and/or classes " + "to profile specified by their name or path. " + "List is comma separated, adding the current script path " + "profiles the full script. " + "Multiple copies of this flag can be supplied and " + "the list is extended. " + "Only works with line_profiler -l, --line-by-line") + add_argument(parser, '--prof-imports', action='store_true', + help="If specified, modules specified to `--prof-mod` " + "will also autoprofile modules that they import. " + "Only works with line_profiler -l, --line-by-line") if parser is help_parser or module is literal_code is None: - parser.add_argument('script', - metavar='{path/to/script' - ' | -m path.to.module | -c "literal code"}', - help='The python script file, module, or ' - 'literal code to run') - parser.add_argument('args', nargs='...', help='Optional script arguments') + add_argument(parser, 'script', + metavar='{path/to/script' + ' | -m path.to.module | -c "literal code"}', + help='The python script file, module, or ' + 'literal code to run') + add_argument(parser, 'args', + nargs='...', help='Optional script arguments') # Hand off to the dummy parser if necessary to generate the help # text - options = real_parser.parse_args(args) + try: + options = real_parser.parse_args(args) + except SystemExit as e: + # If `exit_on_error` is true, let `SystemExit` bubble up and + # kill the interpretor; + # else, catch and handle it more gracefully + # (Note: can't use `ArgumentParser(exit_on_error=False)` in + # Python 3.8) + if exit_on_error: + raise + elif e.code: + raise RuntimeError from None + else: + return if help_parser and getattr(options, 'help', False): help_parser.print_help() - exit() + if exit_on_error: + raise SystemExit(0) + else: + return + try: del options.help except AttributeError: @@ -453,9 +533,9 @@ def positive_float(value): if tempfile_source_and_content: with tempfile.TemporaryDirectory() as tmpdir: _write_tempfile(*tempfile_source_and_content, options, tmpdir) - return _main(options, module) + return _main(options, module, exit_on_error) else: - return _main(options, module) + return _main(options, module, exit_on_error) def _write_tempfile(source, content, options, tmpdir): @@ -487,7 +567,7 @@ def _write_tempfile(source, content, options, tmpdir): suffix='.' + extension) -def _main(options, module=False): +def _main(options, module=False, exit_on_error=True): """ Called by ``main()`` for the actual execution and profiling of code; not to be invoked on its own. @@ -503,10 +583,10 @@ def _main(options, module=False): # Note: this NEEDS to happen here, before the setup script (or # any other code) has a chance to `os.chdir()` sys.path.insert(0, os.path.abspath(os.curdir)) - if options.setup is not None: - # Run some setup code outside of the profiler. This is good for large - # imports. - setup_file = find_script(options.setup) + if options.setup: + # Run some setup code outside of the profiler. This is good for + # large imports. + setup_file = find_script(options.setup, exit_on_error=exit_on_error) __file__ = setup_file __name__ = '__main__' # Make sure the script's directory is on sys.path instead of just @@ -516,7 +596,6 @@ def _main(options, module=False): execfile(setup_file, ns, ns) if options.line_by_line: - import line_profiler prof = line_profiler.LineProfiler() options.builtin = True elif Profile.__module__ == 'profile': @@ -525,25 +604,19 @@ def _main(options, module=False): else: prof = ContextualProfile() - # If line_profiler is installed, then overwrite the explicit decorator - try: - import line_profiler - except ImportError: # Shouldn't happen - install_profiler = global_profiler = None - else: - global_profiler = line_profiler.profile - install_profiler = global_profiler._kernprof_overwrite - - if global_profiler: - install_profiler(prof) + # Overwrite the explicit decorator + global_profiler = line_profiler.profile + install_profiler = global_profiler._kernprof_overwrite + install_profiler(prof) if options.builtin: builtins.__dict__['profile'] = prof if module: - script_file = find_module_script(options.script) + script_file = find_module_script(options.script, + exit_on_error=exit_on_error) else: - script_file = find_script(options.script) + script_file = find_script(options.script, exit_on_error=exit_on_error) # Make sure the script's directory is on sys.path instead of # just kernprof.py's. sys.path.insert(0, os.path.dirname(script_file)) @@ -551,6 +624,7 @@ def _main(options, module=False): __name__ = '__main__' if options.output_interval: + # XXX: why are we doing this here (5a38626) and again below? rt = RepeatedTimer(max(options.output_interval, 1), prof.dump_stats, options.outfile) original_stdout = sys.stdout if options.output_interval: @@ -606,9 +680,8 @@ def _main(options, module=False): else: print(f'{py_exe} -m line_profiler -rmt "{options.outfile}"') # Restore the state of the global `@line_profiler.profile` - if global_profiler: - install_profiler(None) + install_profiler(None) if __name__ == '__main__': - main(sys.argv[1:]) + main() From 0707290092650bfd6e07d837e9579bef3635ed6a Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 19 Apr 2025 11:27:40 +0200 Subject: [PATCH 04/37] `kernprof` refactoring (reading configs) kernprof.py __doc__ Updated with newest `kernprof --help` output short_string_path() New helper function for abbreviating paths _python_command() - Replaced string comparison between paths with `os.path.samefile()` - Updated to use abbreviated paths where possible main() - Updated description to include documentation for the negative options - Added option `--config` for loading config from a specific file instead of going through lookup - Updated `const` value for the bare `-i`/`--output-intereval` option (the old value 0, equivalent to not specifying the option, doesn't really make sense) - Grouped options into argument groups for better organization - Updated help texts for options to be more stylistically consistent and to show the default values - Updated help texts for the `-p`/`--prof-mod` option to show an example - Updated help texts for the `--prof-imports` to be more in line with what it actually does (see docstring of `line_profiler.autoprofile.ast_tree_profiler.AstTreeProfiler`) - Added option resolution: values of un-specified flags now taken from the specified/looked-up config file --- kernprof.py | 218 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 156 insertions(+), 62 deletions(-) diff --git a/kernprof.py b/kernprof.py index baee0d1a..903ac6d7 100755 --- a/kernprof.py +++ b/kernprof.py @@ -63,35 +63,44 @@ def main(): .. code:: - usage: kernprof [-h] [-V] [-l] [-b] [-o OUTFILE] [-s SETUP] [-v] [-r] [-u UNIT] [-z] [-i [OUTPUT_INTERVAL]] [-p PROF_MOD] [--prof-imports] + usage: kernprof [-h] [-V] [--config CONFIG] [-l] [-b] [-s SETUP] [-p PROF_MOD] [--prof-imports] [-o OUTFILE] [-v] [-r] [-u UNIT] [-z] [-i [OUTPUT_INTERVAL]] {path/to/script | -m path.to.module | -c "literal code"} ... - - Run and profile a python script. - + + Run and profile a python script or module.Boolean options can be negated by passing the corresponding flag (e.g. `--no-view` for `--view`). + positional arguments: {path/to/script | -m path.to.module | -c "literal code"} The python script file, module, or literal code to run args Optional script arguments - + options: -h, --help show this help message and exit -V, --version show program's version number and exit - -l, --line-by-line Use the line-by-line profiler instead of cProfile. Implies --builtin. - -b, --builtin Put 'profile' in the builtins. Use 'profile.enable()'/'.disable()', '@profile' to decorate functions, or 'with profile:' to profile a section of code. - -o, --outfile OUTFILE - Save stats to (default: 'scriptname.lprof' with --line-by-line, 'scriptname.prof' without) - -s, --setup SETUP Code to execute before the code to profile - -v, --view View the results of the profile in addition to saving it - -r, --rich Use rich formatting if viewing output - -u, --unit UNIT Output unit (in seconds) in which the timing info is displayed (default: 1e-6) - -z, --skip-zero Hide functions which have not been called - -i, --output-interval [OUTPUT_INTERVAL] - Enables outputting of cumulative profiling results to file every n seconds. Uses the threading module. Minimum value is 1 (second). Defaults to - disabled. + --config CONFIG Path to the TOML file, from the `tool.line_profiler.kernprof` table of which to load defaults for the options. (Default: 'pyproject.toml') + + profiling options: + -l, --line-by-line Use the line-by-line profiler instead of cProfile. Implies `--builtin`. (Boolean option; default: False) + -b, --builtin Put `profile` in the builtins. Use `profile.enable()`/`.disable()` to toggle profiling, `@profile` to decorate functions, or `with profile:` to profile + a section of code. (Boolean option; default: False) + -s, --setup SETUP Path to the Python source file containing setup code to execute before the code to profile. (Default: N/A) -p, --prof-mod PROF_MOD List of modules, functions and/or classes to profile specified by their name or path. List is comma separated, adding the current script path profiles - the full script. Multiple copies of this flag can be supplied and the.list is extended. Only works with line_profiler -l, --line-by-line - --prof-imports If specified, modules specified to `--prof-mod` will also autoprofile modules that they import. Only works with line_profiler -l, --line-by-line + the full script. Multiple copies of this flag can be supplied and the list is extended (e.g. `-p this.module,another.module -p some.func`). Only works + with line profiling (`-l`/`--line-by-line`). (Default: N/A) + --prof-imports If the script/module profiled is in `--prof-mod`, autoprofile all its imports. Only works with line profiling (`-l`/`--line-by-line`). (Boolean option; + default: False) + + output options: + -o, --outfile OUTFILE + Save stats to OUTFILE. (Default: '.lprof' in line-profiling mode (`-l`/`--line-by-line`); '.prof' + otherwise) + -v, --view View the results of the profile in addition to saving it. (Boolean option; default: False) + -r, --rich Use rich formatting if viewing output. (Boolean option; default: False) + -u, --unit UNIT Output unit (in seconds) in which the timing info is displayed. (Default: 1e-06 s) + -z, --skip-zero Hide functions which have not been called. (Boolean option; default: False) + -i, --output-interval [OUTPUT_INTERVAL] + Enables outputting of cumulative profiling results to OUTFILE every OUTPUT_INTERVAL seconds. Uses the threading module. Minimum value (and the value + implied if the bare option is given) is 1 s. (Default: 0 s (disabled)) """ import argparse import builtins @@ -102,6 +111,7 @@ def main(): import asyncio # NOQA import concurrent.futures # NOQA import tempfile +import pathlib import time from runpy import run_module @@ -118,6 +128,7 @@ def main(): import line_profiler from line_profiler.profiler_mixin import ByCountProfilerMixin +from line_profiler.toml_config import get_config def execfile(filename, globals=None, locals=None): @@ -234,15 +245,34 @@ def find_script(script_name, *, exit_on_error=True): raise FileNotFoundError(msg) +def short_string_path(path): + """ + Get the shortest formatted `path` among the provided path, the + corresponding absolute path, and its relative path to the current + directory. + """ + path = pathlib.Path(path) + paths = {str(path)} + abspath = path.absolute() + paths.add(str(abspath)) + try: + paths.add(str(abspath.relative_to(path.cwd().absolute()))) + except ValueError: # Not relative to the curdir + pass + return min(paths, key=len) + + def _python_command(): """ Return a command that corresponds to :py:obj:`sys.executable`. """ import shutil - for abbr in 'python', 'python3': - if os.path.samefile(shutil.which(abbr), sys.executable): - return abbr - return sys.executable + if os.path.samefile(shutil.which('python'), sys.executable): + return 'python' + elif os.path.samefile(shutil.which('python3'), sys.executable): + return 'python3' + else: + return short_string_path(sys.executable) class _restore_list: @@ -402,9 +432,22 @@ def add_argument(parser_like, *args, 'help': falsy_help_text}) return action + def get_kernprof_config(*args, **kwargs): + """ + Get the `tool.line_profiler.kernprof` configs and normalize its + keys. + """ + conf, source = get_config(*args, **kwargs) + kernprof_conf = {key.replace('-', '_'): value + for key, value in conf['kernprof'].items()} + return kernprof_conf, source + create_parser = functools.partial( argparse.ArgumentParser, - description='Run and profile a python script.') + description='Run and profile a python script or module.' + 'Boolean options can be negated by passing the corresponding flag ' + '(e.g. `--no-view` for `--view`).') + defaults, default_source = get_kernprof_config() if args is None: args = sys.argv[1:] @@ -440,47 +483,84 @@ def add_argument(parser_like, *args, for parser in parsers: add_argument(parser, '-V', '--version', action='version', version=__version__) - add_argument(parser, '-l', '--line-by-line', action='store_true', + add_argument(parser, '--config', + help='Path to the TOML file, from the ' + '`tool.line_profiler.kernprof` table of which to load ' + 'defaults for the options. ' + f'(Default: {short_string_path(default_source)!r})') + prof_opts = parser.add_argument_group('profiling options') + add_argument(prof_opts, '-l', '--line-by-line', action='store_true', help='Use the line-by-line profiler instead of cProfile. ' - 'Implies --builtin.') - add_argument(parser, '-b', '--builtin', action='store_true', - help="Put 'profile' in the builtins. " - "Use 'profile.enable()'/'.disable()', " - "'@profile' to decorate functions, " - "or 'with profile:' to profile a section of code.") - add_argument(parser, '-o', '--outfile', - help='Save stats to ' - "(default: 'scriptname.lprof' with --line-by-line, " - "'scriptname.prof' without)") - add_argument(parser, '-s', '--setup', - help='Code to execute before the code to profile') - add_argument(parser, '-v', '--view', action='store_true', - help='View the results of the profile ' - 'in addition to saving it') - add_argument(parser, '-r', '--rich', action='store_true', - help='Use rich formatting if viewing output') - add_argument(parser, '-u', '--unit', default='1e-6', type=positive_float, - help='Output unit (in seconds) in which ' - 'the timing info is displayed (default: %(default)s)') - add_argument(parser, '-z', '--skip-zero', action='store_true', - help="Hide functions which have not been called") - add_argument(parser, '-i', '--output-interval', - type=int, default=0, const=0, nargs='?', - help="Enables outputting of cumulative profiling results " - "to file every n seconds. Uses the threading module. " - "Minimum value is 1 (second). Defaults to disabled.") - add_argument(parser, '-p', '--prof-mod', action='append', type=str, + 'Implies `--builtin`. ' + f'(Boolean option; default: {defaults["line_by_line"]})') + add_argument(prof_opts, '-b', '--builtin', action='store_true', + help="Put `profile` in the builtins. " + "Use `profile.enable()`/`.disable()` to toggle profiling, " + "`@profile` to decorate functions, " + "or `with profile:` to profile a section of code. " + f"(Boolean option; default: {defaults['builtin']})") + if defaults['setup']: + def_setupfile = repr(defaults['setup']) + else: + def_setupfile = 'N/A' + add_argument(prof_opts, '-s', '--setup', + help='Path to the Python source file containing setup ' + 'code to execute before the code to profile. ' + f'(Default: {def_setupfile})') + if defaults['prof_mod']: + def_prof_mod = repr(defaults['prof_mod']) + else: + def_prof_mod = 'N/A' + add_argument(prof_opts, '-p', '--prof-mod', action='append', help="List of modules, functions and/or classes " "to profile specified by their name or path. " "List is comma separated, adding the current script path " "profiles the full script. " "Multiple copies of this flag can be supplied and " - "the list is extended. " - "Only works with line_profiler -l, --line-by-line") - add_argument(parser, '--prof-imports', action='store_true', - help="If specified, modules specified to `--prof-mod` " - "will also autoprofile modules that they import. " - "Only works with line_profiler -l, --line-by-line") + "the list is extended " + "(e.g. `-p this.module,another.module -p some.func`). " + "Only works with line profiling (`-l`/`--line-by-line`). " + f"(Default: {def_prof_mod})") + add_argument(prof_opts, '--prof-imports', action='store_true', + help="If the script/module profiled is in `--prof-mod`, " + "autoprofile all its imports. " + "Only works with line profiling (`-l`/`--line-by-line`). " + f"(Boolean option; default: {defaults['prof_imports']})") + out_opts = parser.add_argument_group('output options') + if defaults['outfile']: + def_outfile = repr(defaults['outfile']) + else: + def_outfile = ( + "'.lprof' in line-profiling mode " + "(`-l`/`--line-by-line`); " + "'.prof' otherwise") + add_argument(out_opts, '-o', '--outfile', + help=f'Save stats to OUTFILE. (Default: {def_outfile})') + add_argument(out_opts, '-v', '--view', action='store_true', + help='View the results of the profile ' + 'in addition to saving it. ' + f'(Boolean option; default: {defaults["view"]})') + add_argument(out_opts, '-r', '--rich', action='store_true', + help='Use rich formatting if viewing output. ' + f'(Boolean option; default: {defaults["rich"]})') + add_argument(out_opts, '-u', '--unit', type=positive_float, + help='Output unit (in seconds) in which ' + 'the timing info is displayed. ' + f'(Default: {defaults["unit"]} s)') + add_argument(out_opts, '-z', '--skip-zero', action='store_true', + help="Hide functions which have not been called. " + f"(Boolean option; default: {defaults['skip_zero']})") + if defaults['output_interval']: + def_out_int = f'{defaults["output_interval"]} s' + else: + def_out_int = '0 s (disabled)' + add_argument(out_opts, '-i', '--output-interval', + type=int, const=1, nargs='?', + help="Enables outputting of cumulative profiling results " + "to OUTFILE every OUTPUT_INTERVAL seconds. " + "Uses the threading module. " + "Minimum value (and the value implied if the bare option " + f"is given) is 1 s. (Default: {def_out_int})") if parser is help_parser or module is literal_code is None: add_argument(parser, 'script', @@ -514,10 +594,23 @@ def add_argument(parser_like, *args, else: return + # Parse the provided config file (if any), and resolve the values + # of the un-specified options + config = options.config try: del options.help except AttributeError: pass + try: + del options.config + except AttributeError: + pass + if config: + defaults, _ = get_kernprof_config(config) + for key, default in defaults.items(): + if getattr(options, key, None) is None: + setattr(options, key, default) + # Add in the pre-partitioned arguments cut off by `-m ` or # `-c